From 73c9d7aa7f9c0346d339dc5d0002335b81aac00d Mon Sep 17 00:00:00 2001 From: Abdulrahman Date: Fri, 31 Jul 2026 21:45:02 +0300 Subject: [PATCH 1/9] Add gitignored runtime layout and pinned whisper.cpp setup scripts Everything the local transcription runtime needs lives outside Git: backend/runtime/whisper/ for the cloned source and built binary, backend/storage/models/ for weights, backend/storage/processing/ for per-job decoded audio, backend/storage/benchmarks/ for benchmark input and output. setup_whisper.sh clones and builds the pinned v1.9.1 release (commit f049fff95a089aa9969deb009cdd4892b3e74916), downloads a model with upstream's own downloader, verifies both, and records the tag, commit, build options, binary version and model SHA-256 in a manifest the application reads. It never installs system packages, never touches git or shell configuration: a missing cmake/ffmpeg is reported with the exact command to run, and the script exits. Core ML, quantization and VAD are deliberately left off. They change output and need measuring before adoption. check_whisper_runtime.sh diagnoses the same set-up read-only. The public-safety check now rejects runtime binaries, models, processing and benchmark artifacts, audio of any kind, and transcript dumps; it also fails if the local username appears in tracked content, or if anything under the runtime directories is neither tracked nor ignored. --- .gitignore | 32 +++++ backend/runtime/.gitkeep | 0 backend/storage/benchmarks/.gitkeep | 0 backend/storage/processing/.gitkeep | 0 scripts/check_public_safety.sh | 58 +++++++- scripts/check_whisper_runtime.sh | 98 ++++++++++++++ scripts/setup_whisper.sh | 201 ++++++++++++++++++++++++++++ 7 files changed, 388 insertions(+), 1 deletion(-) create mode 100644 backend/runtime/.gitkeep create mode 100644 backend/storage/benchmarks/.gitkeep create mode 100644 backend/storage/processing/.gitkeep create mode 100755 scripts/check_whisper_runtime.sh create mode 100755 scripts/setup_whisper.sh diff --git a/.gitignore b/.gitignore index e7b41c1..ef31815 100644 --- a/.gitignore +++ b/.gitignore @@ -25,6 +25,38 @@ __pycache__/ backend/storage/recordings/** !backend/storage/recordings/.gitkeep +# --- Phase 2A: external whisper.cpp runtime (source, build tree, binaries) --- +# Cloned and built locally by scripts/setup_whisper.sh. Never vendored: see +# docs/adr/0009-pinned-whisper-cpp-runtime.md. +backend/runtime/** +!backend/runtime/.gitkeep + +# --- Phase 2A: model weights (multi-hundred-MB, never committed) --- +backend/storage/models/** +!backend/storage/models/.gitkeep + +# --- Phase 2A: per-job temporary decoded PCM (your speech in the clear) --- +backend/storage/processing/** +!backend/storage/processing/.gitkeep + +# --- Phase 2A: local benchmark inputs and output (real recordings) --- +backend/storage/benchmarks/** +!backend/storage/benchmarks/.gitkeep + +# --- Audio, anywhere (no real recording is ever a tracked fixture) --- +*.wav +*.webm +*.ogg +*.oga +*.m4a +*.mp3 +*.flac + +# --- Core ML / quantized model artifacts (generated, never committed) --- +*.mlmodel +*.mlmodelc/ +*.mlpackage/ + # --- Local application logs --- *.log logs/ diff --git a/backend/runtime/.gitkeep b/backend/runtime/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/backend/storage/benchmarks/.gitkeep b/backend/storage/benchmarks/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/backend/storage/processing/.gitkeep b/backend/storage/processing/.gitkeep new file mode 100644 index 0000000..e69de29 diff --git a/scripts/check_public_safety.sh b/scripts/check_public_safety.sh index 9c05434..9dd53d8 100755 --- a/scripts/check_public_safety.sh +++ b/scripts/check_public_safety.sh @@ -20,6 +20,22 @@ prohibited_patterns=( '\.(gguf|bin|pt|onnx)$' '(^|/)models/' '\.log$' + # --- Phase 2A additions --- + # External whisper.cpp runtime: source tree, build output, binaries. + '^backend/runtime/.+' + # Model weights and per-job / benchmark runtime artifacts. + '^backend/storage/models/.+' + '^backend/storage/processing/.+' + '^backend/storage/benchmarks/.+' + # Audio of any kind: no real recording is ever a tracked fixture, and + # the intermediate 16 kHz PCM WAV is decoded speech in the clear. + '\.(wav|webm|ogg|oga|m4a|mp3|flac|aac|opus)$' + # Core ML / quantization artifacts generated from a model. + '\.(mlmodel)$' + '\.(mlmodelc|mlpackage)/' + # Transcript / benchmark dumps. + '(^|/)transcripts?/.+\.(txt|json|srt|vtt)$' + '\.(srt|vtt)$' ) tracked_files=$(git ls-files) @@ -35,7 +51,7 @@ done echo "== Checking tracked file contents for obvious local-path / secret leakage ==" # Excludes this script itself (it necessarily contains these patterns). -content_matches=$(git grep -nIE '(/Users/[A-Za-z0-9_.-]+|AKIA[0-9A-Z]{16}|-----BEGIN [A-Z ]*PRIVATE KEY-----)' \ +content_matches=$(git grep -nIE '(/Users/[A-Za-z0-9_.-]+|/home/[A-Za-z0-9_.-]+|AKIA[0-9A-Z]{16}|-----BEGIN [A-Z ]*PRIVATE KEY-----)' \ -- . ':(exclude)scripts/check_public_safety.sh' || true) if [ -n "$content_matches" ]; then echo "FAIL: possible local path or secret leakage in tracked content:" @@ -43,6 +59,46 @@ if [ -n "$content_matches" ]; then fail=1 fi +# The local username, wherever it appears, is a leak even outside a /Users +# path (e.g. baked into a benchmark report or a pasted command line). +local_user="$(id -un 2>/dev/null || true)" +if [ -n "$local_user" ] && [ "$local_user" != "root" ]; then + user_matches=$(git grep -nIw -- "$local_user" -- . ':(exclude)scripts/check_public_safety.sh' || true) + if [ -n "$user_matches" ]; then + echo "FAIL: the local username appears in tracked content:" + echo "$user_matches" | sed 's/^/ /' + fail=1 + fi +fi + +echo "== Confirming Phase 2A runtime artifacts are untracked and ignored ==" +# These directories legitimately exist on a set-up machine. The check is not +# "do they exist" but "is anything inside them visible to git" -- which +# covers both accidental `git add -f` and a missing .gitignore rule. +runtime_dirs=( + backend/runtime + backend/storage/models + backend/storage/processing + backend/storage/benchmarks +) +for dir in "${runtime_dirs[@]}"; do + [ -d "$dir" ] || continue + # Tracked files under the directory (excluding the .gitkeep placeholder). + tracked_here=$(git ls-files -- "$dir" | grep -v '\.gitkeep$' || true) + if [ -n "$tracked_here" ]; then + echo "FAIL: tracked file(s) under runtime directory '$dir':" + echo "$tracked_here" | sed 's/^/ /' + fail=1 + fi + # Untracked-and-not-ignored files: these would be swept up by `git add -A`. + exposed=$(git ls-files --others --exclude-standard -- "$dir" | grep -v '\.gitkeep$' || true) + if [ -n "$exposed" ]; then + echo "FAIL: file(s) under '$dir' are neither tracked nor ignored (a 'git add -A' would commit them):" + echo "$exposed" | sed 's/^/ /' + fail=1 + fi +done + if [ "$fail" -eq 0 ]; then echo "OK: no prohibited tracked paths or obvious secret/path leakage found." else diff --git a/scripts/check_whisper_runtime.sh b/scripts/check_whisper_runtime.sh new file mode 100755 index 0000000..1dd6f20 --- /dev/null +++ b/scripts/check_whisper_runtime.sh @@ -0,0 +1,98 @@ +#!/usr/bin/env bash +# Diagnose the local transcription runtime, without changing anything. +# +# Read-only: it installs nothing, builds nothing, downloads nothing, and +# touches no configuration. Every problem it reports comes with the exact +# command to fix it. +set -uo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" +RUNTIME_DIR="$REPO_ROOT/backend/runtime/whisper" +MODELS_DIR="$REPO_ROOT/backend/storage/models" +MODEL_NAME="${1:-base.en}" + +ok() { printf ' \033[32mOK\033[0m %s\n' "$*"; } +bad() { printf ' \033[31mMISS\033[0m %s\n' "$*"; } +note() { printf ' %s\n' "$*"; } + +problems=0 + +echo "SpeakLab transcription runtime check" +echo "====================================" +echo +echo "Tools:" +for tool in cmake ffmpeg git; do + if command -v "$tool" >/dev/null 2>&1; then + ok "$tool found" + else + bad "$tool not found" + case "$tool" in + cmake) note "Install with: brew install cmake" ;; + ffmpeg) note "Install with: brew install ffmpeg" ;; + git) note "Install with: xcode-select --install" ;; + esac + problems=$((problems + 1)) + fi +done + +echo +echo "whisper.cpp runtime:" +if [ -x "$RUNTIME_DIR/bin/whisper-cli" ]; then + ok "whisper-cli binary present" + version_line="$("$RUNTIME_DIR/bin/whisper-cli" --version 2>&1 | head -1)" + note "reports: ${version_line:-}" +else + bad "whisper-cli binary not built" + note "Build with: scripts/setup_whisper.sh" + problems=$((problems + 1)) +fi + +if [ -f "$RUNTIME_DIR/manifest.json" ]; then + ok "runtime manifest present" + # Plain grep rather than a jq dependency for a diagnostic script. + for key in release_tag commit build_options; do + value="$(grep -o "\"$key\": *\"[^\"]*\"" "$RUNTIME_DIR/manifest.json" | head -1 | sed 's/.*: *"//; s/"$//')" + [ -n "$value" ] && note "$key: $value" + done +else + bad "runtime manifest missing" + note "Rerun: scripts/setup_whisper.sh (it records the built version)" + problems=$((problems + 1)) +fi + +echo +echo "Model:" +MODEL_FILE="$MODELS_DIR/ggml-${MODEL_NAME}.bin" +if [ -f "$MODEL_FILE" ]; then + size="$(wc -c <"$MODEL_FILE" | tr -d ' ')" + ok "ggml-${MODEL_NAME}.bin present (${size} bytes)" + if command -v shasum >/dev/null 2>&1; then + note "sha256: $(shasum -a 256 "$MODEL_FILE" | awk '{print $1}')" + fi +else + bad "ggml-${MODEL_NAME}.bin not downloaded" + note "Download with: scripts/setup_whisper.sh --model ${MODEL_NAME}" + problems=$((problems + 1)) +fi + +echo +echo "Artifacts are gitignored:" +cd "$REPO_ROOT" +exposed="$(git ls-files --others --exclude-standard -- backend/runtime backend/storage/models backend/storage/processing backend/storage/benchmarks 2>/dev/null | grep -v '\.gitkeep$' || true)" +if [ -z "$exposed" ]; then + ok "no runtime artifact is visible to git" +else + bad "these runtime files are NOT ignored by git:" + echo "$exposed" | sed 's/^/ /' + problems=$((problems + 1)) +fi + +echo +if [ "$problems" -eq 0 ]; then + echo "All checks passed. Start the worker with:" + echo " cd backend && source .venv/bin/activate && python -m app.worker" + exit 0 +fi +echo "$problems problem(s) found -- see the suggested commands above." +echo "Nothing was installed or modified by this script." +exit 1 diff --git a/scripts/setup_whisper.sh b/scripts/setup_whisper.sh new file mode 100755 index 0000000..6e3032e --- /dev/null +++ b/scripts/setup_whisper.sh @@ -0,0 +1,201 @@ +#!/usr/bin/env bash +# Build the pinned whisper.cpp runtime and download a model, locally. +# +# What this script will NOT do, deliberately: +# - install anything with Homebrew, apt, or any other package manager +# - modify your PATH, shell profile, git config, or any global setting +# - commit, stage, or otherwise touch git state +# - vendor whisper.cpp source or model weights into this repository +# +# Missing dependencies are REPORTED with the exact command to run, and the +# script exits. Installing system software is the owner's decision, not a +# side effect of running a setup script. +# +# Everything it creates lives in gitignored directories: +# backend/runtime/whisper/ cloned source, build tree, and the binary +# backend/storage/models/ downloaded model weights +# +# Usage: +# scripts/setup_whisper.sh [--model base.en] +set -euo pipefail + +REPO_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)" + +# --- Pinned upstream version ------------------------------------------------- +# Verified against https://github.com/ggml-org/whisper.cpp/releases/latest. +# A pinned tag, never `master`: an inference runtime that changes underneath +# you silently changes your transcripts. See +# docs/adr/0009-pinned-whisper-cpp-runtime.md. +WHISPER_REPO="https://github.com/ggml-org/whisper.cpp.git" +WHISPER_TAG="v1.9.1" +WHISPER_COMMIT="f049fff95a089aa9969deb009cdd4892b3e74916" + +MODEL_NAME="base.en" +while [ $# -gt 0 ]; do + case "$1" in + --model) + MODEL_NAME="${2:-}" + shift 2 + ;; + --model=*) + MODEL_NAME="${1#*=}" + shift + ;; + -h | --help) + sed -n '2,20p' "$0" + exit 0 + ;; + *) + echo "Unknown argument: $1" >&2 + exit 2 + ;; + esac +done + +RUNTIME_DIR="$REPO_ROOT/backend/runtime/whisper" +SRC_DIR="$RUNTIME_DIR/src" +BUILD_DIR="$RUNTIME_DIR/build" +BIN_DIR="$RUNTIME_DIR/bin" +MODELS_DIR="$REPO_ROOT/backend/storage/models" +MANIFEST="$RUNTIME_DIR/manifest.json" + +# CMAKE_BUILD_TYPE=Release matters a lot here: a debug build of whisper.cpp +# is several times slower. Core ML, quantization and VAD are deliberately +# NOT enabled -- they are real optimizations that change output and must be +# measured before being adopted (see docs/AI_MODEL_STRATEGY.md). Metal on +# Apple Silicon is on by default upstream and needs no flag. +BUILD_OPTIONS="-DCMAKE_BUILD_TYPE=Release -DWHISPER_BUILD_TESTS=OFF -DWHISPER_BUILD_EXAMPLES=ON" + +info() { printf '\033[1m==>\033[0m %s\n' "$*"; } +warn() { printf '\033[33mWARN:\033[0m %s\n' "$*" >&2; } +die() { + printf '\033[31mERROR:\033[0m %s\n' "$*" >&2 + exit 1 +} + +# --- Dependency checks ------------------------------------------------------- + +missing=0 +require_tool() { + local tool="$1" install_hint="$2" + if ! command -v "$tool" >/dev/null 2>&1; then + printf '\033[31mMissing:\033[0m %s\n' "$tool" >&2 + printf ' Install it with: %s\n' "$install_hint" >&2 + missing=1 + fi +} + +info "Checking required tools" +require_tool git "xcode-select --install (macOS) or your distro's git package" +require_tool cmake "brew install cmake (macOS) or your distro's cmake package" +require_tool ffmpeg "brew install ffmpeg (macOS) or your distro's ffmpeg package" +require_tool curl "already present on macOS; otherwise your distro's curl package" + +if [ "$missing" -ne 0 ]; then + echo >&2 + die "One or more dependencies are missing. Install them yourself (commands above), then re-run this script. +This script will not install system packages for you." +fi +info "All required tools found." + +# --- Fetch the pinned source ------------------------------------------------- + +mkdir -p "$RUNTIME_DIR" "$MODELS_DIR" + +if [ -d "$SRC_DIR/.git" ]; then + info "Reusing existing checkout in backend/runtime/whisper/src" + git -C "$SRC_DIR" fetch --tags --depth 1 origin "$WHISPER_TAG" >/dev/null 2>&1 || true +else + info "Cloning whisper.cpp $WHISPER_TAG" + rm -rf "$SRC_DIR" + git clone --depth 1 --branch "$WHISPER_TAG" "$WHISPER_REPO" "$SRC_DIR" +fi + +ACTUAL_COMMIT="$(git -C "$SRC_DIR" rev-parse HEAD)" +if [ "$ACTUAL_COMMIT" != "$WHISPER_COMMIT" ]; then + warn "Checked-out commit $ACTUAL_COMMIT does not match the pinned commit $WHISPER_COMMIT." + warn "The tag may have been moved upstream. Continuing, but the manifest will record what was ACTUALLY built." +fi + +# --- Build ------------------------------------------------------------------- + +info "Configuring build (Release; no Core ML, no quantization, no VAD)" +# shellcheck disable=SC2086 # BUILD_OPTIONS is intentionally word-split +cmake -S "$SRC_DIR" -B "$BUILD_DIR" $BUILD_OPTIONS + +info "Building whisper-cli" +cmake --build "$BUILD_DIR" --config Release --target whisper-cli -j "$(getconf _NPROCESSORS_ONLN 2>/dev/null || echo 4)" + +BUILT_BINARY="$(find "$BUILD_DIR" -type f -name 'whisper-cli' -perm -u+x 2>/dev/null | head -1 || true)" +[ -n "$BUILT_BINARY" ] || die "Build finished but no whisper-cli binary was found under backend/runtime/whisper/build." + +mkdir -p "$BIN_DIR" +cp "$BUILT_BINARY" "$BIN_DIR/whisper-cli" +chmod +x "$BIN_DIR/whisper-cli" +info "Installed binary at backend/runtime/whisper/bin/whisper-cli" + +# --- Verify the binary ------------------------------------------------------- + +BINARY_VERSION="$("$BIN_DIR/whisper-cli" --version 2>&1 | head -1 || true)" +if [ -z "$BINARY_VERSION" ]; then + BINARY_VERSION="$("$BIN_DIR/whisper-cli" --help 2>&1 | head -1 || true)" +fi +info "Binary reports: ${BINARY_VERSION:-}" + +# --- Download the model ------------------------------------------------------ + +MODEL_FILE="$MODELS_DIR/ggml-${MODEL_NAME}.bin" +if [ -f "$MODEL_FILE" ]; then + info "Model ggml-${MODEL_NAME}.bin already present; not re-downloading." +else + info "Downloading model $MODEL_NAME (this is a large file)" + # Uses upstream's own downloader so no URL is hard-coded here. + bash "$SRC_DIR/models/download-ggml-model.sh" "$MODEL_NAME" "$MODELS_DIR" +fi +[ -f "$MODEL_FILE" ] || die "Model download did not produce ggml-${MODEL_NAME}.bin in backend/storage/models." + +if command -v shasum >/dev/null 2>&1; then + MODEL_SHA="$(shasum -a 256 "$MODEL_FILE" | awk '{print $1}')" +elif command -v sha256sum >/dev/null 2>&1; then + MODEL_SHA="$(sha256sum "$MODEL_FILE" | awk '{print $1}')" +else + MODEL_SHA="" + warn "No shasum/sha256sum available; model hash will not be recorded." +fi +MODEL_SIZE="$(wc -c <"$MODEL_FILE" | tr -d ' ')" + +# --- Record what was actually built ----------------------------------------- +# The application reads this manifest to stamp transcripts with the runtime +# version, rather than guessing from the binary. + +cat >"$MANIFEST" <}" +echo +echo "Next: run the worker in its own terminal:" +echo " cd backend && source .venv/bin/activate && python -m app.worker" +echo +echo "Verify at any time with: scripts/check_whisper_runtime.sh" From 6a42d5208c07a8e53e1b22b804a78c573d67cd2f Mon Sep 17 00:00:00 2001 From: Abdulrahman Date: Fri, 31 Jul 2026 21:45:13 +0300 Subject: [PATCH 2/9] Add processing_jobs table, transcript provenance, and segment storage Migration 0002 adds the durable job queue and everything a real transcript needs to stay identifiable years later. processing_jobs carries status, stage, attempt/max-attempt counts, a safe error code and message, worker id, timestamps and runtime provenance. A partial unique index on (session_id, job_type) WHERE status IN ('PENDING','RUNNING') enforces one active job per session in the database, rather than via a check-then-insert that would race. transcripts gains detected language, runtime version, model name and SHA-256, parameters, audio and processing durations, and real-time factor. transcript_segments stores ordered segment timestamps with a UNIQUE (transcript_id, segment_index) constraint, CHECKs for ordering, and cascading deletion. Word-level timestamps are deliberately out of scope for this phase. sessions gains audio_duration_seconds: the authoritative duration decoded from the media, kept separate from client_reported_duration_seconds so the browser's own timer stays available for comparison rather than being overwritten. Every added column is nullable or defaulted, so this applies cleanly to an existing Phase 1 database without rewriting a single row. Job claiming uses BEGIN IMMEDIATE on an autocommit connection, so get_connection now takes an autocommit flag. The claim's UPDATE re-asserts status = 'PENDING' so a lost race updates zero rows instead of double-claiming, and a rejected duplicate rolls back rather than leaving its connection holding a write lock. --- backend/app/config.py | 63 +++ backend/app/db.py | 17 +- ...2_processing_jobs_and_real_transcripts.sql | 95 ++++ backend/app/repositories/job_repository.py | 454 ++++++++++++++++++ .../app/repositories/session_repository.py | 136 +++++- backend/app/schemas.py | 120 +++++ 6 files changed, 878 insertions(+), 7 deletions(-) create mode 100644 backend/app/migrations/0002_processing_jobs_and_real_transcripts.sql create mode 100644 backend/app/repositories/job_repository.py diff --git a/backend/app/config.py b/backend/app/config.py index 5a0a983..8032d60 100644 --- a/backend/app/config.py +++ b/backend/app/config.py @@ -25,6 +25,21 @@ } +# The whisper.cpp release this project is pinned to. Recorded here (not read +# from whatever happens to be checked out in backend/runtime/) so that a +# runtime built from a different tag is *detectable* rather than silently +# accepted -- see scripts/setup_whisper.sh, which writes the actual built +# tag/commit into the runtime manifest, and docs/adr/0009-pinned-whisper-cpp-runtime.md. +WHISPER_PINNED_RELEASE_TAG = "v1.9.1" +WHISPER_PINNED_COMMIT = "f049fff95a089aa9969deb009cdd4892b3e74916" + +# Models the setup script and benchmark utility know how to fetch. This is a +# convenience list, NOT a constraint on the data model: transcripts store +# whatever model name was actually used, so a non-English model added later +# needs no schema change (see docs/DATA_MODEL.md). +KNOWN_MODEL_NAMES: tuple[str, ...] = ("base.en", "small.en") + + @dataclass class Settings: storage_dir: Path @@ -33,6 +48,12 @@ class Settings: migrations_dir: Path seed_file: Path + # Phase 2A runtime locations. All three are gitignored -- see the root + # .gitignore and scripts/check_public_safety.sh. + runtime_dir: Path = BACKEND_DIR / "runtime" + models_dir: Path = BACKEND_DIR / "storage" / "models" + processing_dir: Path = BACKEND_DIR / "storage" / "processing" + host: str = "127.0.0.1" port: int = 8000 cors_origins: tuple[str, ...] = ( @@ -57,6 +78,45 @@ class Settings: history_default_limit: int = 20 history_max_limit: int = 100 + # --- Transcription runtime ------------------------------------------- + # Which TranscriptionProvider the worker uses. "whisper-cpp" is the + # default; "mock" exists for tests and for deliberately running the + # Phase 1 simulated pipeline. There is no silent fallback from one to the + # other: if whisper.cpp is selected and unavailable, the job FAILS with a + # clear error code rather than quietly producing a fake transcript. + transcription_provider: str = "whisper-cpp" + whisper_model_name: str = "base.en" + # None => let whisper.cpp auto-detect. Deliberately not hard-coded to + # English even though the default model is English-only. + whisper_language: str | None = "en" + whisper_threads: int | None = None + whisper_timeout_seconds: int = 900 + ffmpeg_binary: str = "ffmpeg" + ffmpeg_timeout_seconds: int = 300 + + # --- Worker / job queue ---------------------------------------------- + worker_poll_interval_seconds: float = 1.0 + worker_max_attempts: int = 3 + # A RUNNING job whose worker died is reclaimed after this long. Must be + # comfortably longer than the slowest expected transcription. + job_stale_after_seconds: int = 1800 + + # Keep per-job temporary PCM files instead of deleting them. Local + # debugging only -- never enable for ordinary use (the decoded WAV is + # your speech in the clear; see docs/PRIVACY_AND_SECURITY.md). + keep_processing_files: bool = False + + @property + def whisper_binary_path(self) -> Path: + return self.runtime_dir / "whisper" / "bin" / "whisper-cli" + + @property + def whisper_manifest_path(self) -> Path: + return self.runtime_dir / "whisper" / "manifest.json" + + def model_path(self, model_name: str) -> Path: + return self.models_dir / f"ggml-{model_name}.bin" + def default_settings(base_dir: Path | None = None) -> Settings: """Build the default Settings. `base_dir` overrides the backend root @@ -69,4 +129,7 @@ def default_settings(base_dir: Path | None = None) -> Settings: recordings_dir=storage_dir / "recordings", migrations_dir=APP_DIR / "migrations", seed_file=APP_DIR / "data" / "topics.seed.json", + runtime_dir=root / "runtime", + models_dir=storage_dir / "models", + processing_dir=storage_dir / "processing", ) diff --git a/backend/app/db.py b/backend/app/db.py index 2614d7a..db95bea 100644 --- a/backend/app/db.py +++ b/backend/app/db.py @@ -19,7 +19,7 @@ from app.timeutil import utc_now_iso -def get_connection(db_path: Path) -> sqlite3.Connection: +def get_connection(db_path: Path, *, autocommit: bool = False) -> sqlite3.Connection: db_path.parent.mkdir(parents=True, exist_ok=True) # check_same_thread=False: FastAPI resolves sync dependencies (like this # connection) in a worker thread but may run an async route handler that @@ -27,7 +27,20 @@ def get_connection(db_path: Path) -> sqlite3.Connection: # uses this connection sequentially, never from two threads at once, so # disabling sqlite3's same-thread guard here is safe -- this is the # pattern FastAPI's own docs recommend for this exact situation. - conn = sqlite3.connect(db_path, timeout=5, check_same_thread=False) + # + # autocommit=True sets isolation_level=None, which stops sqlite3 from + # opening implicit transactions on our behalf. That is required for the + # worker's `BEGIN IMMEDIATE` job claim: with the default (implicit) + # handling, an explicit BEGIN can land inside a transaction sqlite3 + # already started, and the claim would not get the write lock it needs. + # Request-scoped connections keep the default behavior -- see + # app/repositories/job_repository.py. + conn = sqlite3.connect( + db_path, + timeout=5, + check_same_thread=False, + isolation_level=None if autocommit else "", + ) conn.row_factory = sqlite3.Row conn.execute("PRAGMA foreign_keys = ON") conn.execute("PRAGMA journal_mode = WAL") diff --git a/backend/app/migrations/0002_processing_jobs_and_real_transcripts.sql b/backend/app/migrations/0002_processing_jobs_and_real_transcripts.sql new file mode 100644 index 0000000..b3130be --- /dev/null +++ b/backend/app/migrations/0002_processing_jobs_and_real_transcripts.sql @@ -0,0 +1,95 @@ +-- SpeakLab Phase 2A: durable processing jobs + real (non-mock) transcripts. +-- +-- This migration is written to apply cleanly on top of an existing Phase 1 +-- database that already contains real sessions -- every added column is +-- nullable or defaulted, and no existing row is rewritten. See +-- tests/test_migrations.py, which applies 0001 (populating it with Phase 1 +-- shaped rows) and only then applies 0002. + +-- Durable job queue. SQLite is the queue: a row here is the unit of work a +-- worker claims, and it survives a worker crash or a machine restart because +-- it is committed data rather than in-process state. See +-- docs/adr/0006-local-worker-sqlite-queue.md. +-- +-- `stage` is the coarse progress step shown to the user while a job runs; it +-- is deliberately free text constrained by the application rather than a +-- CHECK, because adding a new stage should not require a migration. +-- +-- error_code / error_message are the *safe* pair surfaced through the API: +-- a stable machine code plus an already-redacted human sentence. Raw +-- subprocess stderr and absolute paths never reach these columns -- see +-- app/safe_text.py. +CREATE TABLE IF NOT EXISTS processing_jobs ( + id TEXT PRIMARY KEY, + session_id TEXT NOT NULL REFERENCES sessions (id) ON DELETE CASCADE, + job_type TEXT NOT NULL CHECK (job_type IN ('TRANSCRIPTION')), + status TEXT NOT NULL CHECK (status IN ('PENDING', 'RUNNING', 'SUCCEEDED', 'FAILED')), + stage TEXT NOT NULL, + attempt_count INTEGER NOT NULL DEFAULT 0, + max_attempts INTEGER NOT NULL DEFAULT 3, + error_code TEXT, + error_message TEXT, + worker_id TEXT, + provider_name TEXT, + provider_version TEXT, + runtime_version TEXT, + model_name TEXT, + created_at TEXT NOT NULL, + started_at TEXT, + updated_at TEXT NOT NULL, + completed_at TEXT, + CHECK (attempt_count >= 0), + CHECK (max_attempts >= 1) +); + +-- One *active* transcription job per session, enforced by the database +-- rather than by an application-level check-then-insert (which would race). +-- Completed/failed jobs are excluded from the index, so a session can +-- accumulate a history of attempts while never having two in flight. +CREATE UNIQUE INDEX IF NOT EXISTS idx_jobs_one_active_per_session + ON processing_jobs (session_id, job_type) + WHERE status IN ('PENDING', 'RUNNING'); + +-- Claim scan: workers repeatedly ask for the oldest PENDING job. +CREATE INDEX IF NOT EXISTS idx_jobs_status_created + ON processing_jobs (status, created_at); + +-- Authoritative duration, decoded from the media itself by the audio +-- preprocessor. Kept SEPARATE from client_reported_duration_seconds (which +-- is preserved untouched for diagnostic comparison) exactly as +-- docs/DATA_MODEL.md anticipated in Phase 1. +ALTER TABLE sessions ADD COLUMN audio_duration_seconds REAL; + +-- Transcript provenance. A transcript row must be able to answer "what +-- produced this text, with which weights, under which parameters, and how +-- long did it take" -- otherwise transcripts made months apart with +-- different models silently look comparable. See +-- docs/adr/0009-pinned-whisper-cpp-runtime.md. +ALTER TABLE transcripts ADD COLUMN detected_language TEXT; +ALTER TABLE transcripts ADD COLUMN runtime_version TEXT; +ALTER TABLE transcripts ADD COLUMN model_name TEXT; +ALTER TABLE transcripts ADD COLUMN model_sha256 TEXT; +ALTER TABLE transcripts ADD COLUMN parameters_json TEXT; +ALTER TABLE transcripts ADD COLUMN audio_duration_ms INTEGER; +ALTER TABLE transcripts ADD COLUMN processing_duration_ms INTEGER; +ALTER TABLE transcripts ADD COLUMN real_time_factor REAL; + +-- Segment-level timestamps. Word-level timestamps are deliberately NOT +-- stored in this phase (see docs/SCORING_AND_LIMITATIONS.md): segments are +-- what whisper.cpp reports reliably in its JSON output, and are sufficient +-- for everything Phase 2A displays. +CREATE TABLE IF NOT EXISTS transcript_segments ( + id TEXT PRIMARY KEY, + transcript_id TEXT NOT NULL REFERENCES transcripts (id) ON DELETE CASCADE, + segment_index INTEGER NOT NULL, + start_ms INTEGER NOT NULL, + end_ms INTEGER NOT NULL, + text TEXT NOT NULL, + UNIQUE (transcript_id, segment_index), + CHECK (segment_index >= 0), + CHECK (start_ms >= 0), + CHECK (end_ms >= start_ms) +); + +CREATE INDEX IF NOT EXISTS idx_segments_transcript + ON transcript_segments (transcript_id, segment_index); diff --git a/backend/app/repositories/job_repository.py b/backend/app/repositories/job_repository.py new file mode 100644 index 0000000..c328103 --- /dev/null +++ b/backend/app/repositories/job_repository.py @@ -0,0 +1,454 @@ +"""The durable job queue, implemented on the database we already have. + +SQLite is doing real queue work here, not pretending to. A queue needs three +things: durable storage of pending work, exactly-one-consumer handoff, and +recovery when a consumer dies. SQLite provides all three -- rows survive a +crash, `BEGIN IMMEDIATE` gives a serialized write lock for the handoff, and a +timestamp column is enough to spot a job whose worker never came back. See +docs/adr/0006-local-worker-sqlite-queue.md for why no broker was added. + +## Atomic claiming + +The dangerous version of "claim a job" is: + + row = SELECT ... WHERE status = 'PENDING' LIMIT 1 + UPDATE ... SET status = 'RUNNING' WHERE id = row.id + +Between those two statements another worker can read the same row, and both +proceed to transcribe the same recording. `claim_next` closes that window +two ways at once: the whole read-then-write runs inside a `BEGIN IMMEDIATE` +transaction (which takes SQLite's write lock up front, so a second worker +blocks rather than reads), and the UPDATE re-asserts `status = 'PENDING'` in +its WHERE clause, so even a lost race updates zero rows and returns nothing +instead of double-claiming. + +The transaction covers *only* the claim. It is committed before any audio is +touched, because holding SQLite's write lock across a minute of inference +would block every unrelated write in the application -- see +`TranscriptionJobService`, which reloads the job by id afterwards. + +Claiming requires a connection in autocommit mode (`isolation_level=None`) +so that the explicit `BEGIN IMMEDIATE` is genuinely ours; see +`app/db.py: get_connection(autocommit=True)`. +""" + +from __future__ import annotations + +import sqlite3 +from abc import ABC, abstractmethod +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone + +from app.idutil import new_id +from app.schemas import JobStatus, JobType, ProcessingStage +from app.timeutil import utc_now_iso + + +@dataclass(frozen=True) +class ProcessingJob: + id: str + session_id: str + job_type: str + status: str + stage: str + attempt_count: int + max_attempts: int + error_code: str | None + error_message: str | None + worker_id: str | None + provider_name: str | None + provider_version: str | None + runtime_version: str | None + model_name: str | None + created_at: str + started_at: str | None + updated_at: str + completed_at: str | None + + @property + def attempts_remaining(self) -> int: + return max(0, self.max_attempts - self.attempt_count) + + +def _row_to_job(row: sqlite3.Row) -> ProcessingJob: + return ProcessingJob( + id=row["id"], + session_id=row["session_id"], + job_type=row["job_type"], + status=row["status"], + stage=row["stage"], + attempt_count=row["attempt_count"], + max_attempts=row["max_attempts"], + error_code=row["error_code"], + error_message=row["error_message"], + worker_id=row["worker_id"], + provider_name=row["provider_name"], + provider_version=row["provider_version"], + runtime_version=row["runtime_version"], + model_name=row["model_name"], + created_at=row["created_at"], + started_at=row["started_at"], + updated_at=row["updated_at"], + completed_at=row["completed_at"], + ) + + +class ActiveJobExistsError(Exception): + """A session already has a PENDING or RUNNING job of this type. + + Raised from the database's partial unique index rather than from a + check-then-insert in Python, so two concurrent uploads cannot both pass + the check. + """ + + +class JobRepository(ABC): + @abstractmethod + def create( + self, session_id: str, job_type: str, max_attempts: int + ) -> ProcessingJob: ... + + @abstractmethod + def get(self, job_id: str) -> ProcessingJob | None: ... + + @abstractmethod + def get_latest_for_session(self, session_id: str) -> ProcessingJob | None: ... + + @abstractmethod + def claim_next(self, worker_id: str) -> ProcessingJob | None: ... + + @abstractmethod + def set_stage(self, job_id: str, stage: str) -> None: ... + + @abstractmethod + def mark_succeeded(self, job_id: str, **provenance: str | None) -> None: ... + + @abstractmethod + def mark_failed( + self, job_id: str, error_code: str, error_message: str + ) -> None: ... + + @abstractmethod + def release_for_retry( + self, job_id: str, error_code: str, error_message: str + ) -> None: ... + + @abstractmethod + def recover_stale(self, stale_after_seconds: int) -> list[ProcessingJob]: ... + + +class SqliteJobRepository(JobRepository): + def __init__(self, conn: sqlite3.Connection): + self._conn = conn + + # -- creation ---------------------------------------------------------- + + def create( + self, + session_id: str, + job_type: str = JobType.TRANSCRIPTION.value, + max_attempts: int = 3, + ) -> ProcessingJob: + job_id = new_id() + now = utc_now_iso() + try: + self._conn.execute( + """ + INSERT INTO processing_jobs ( + id, session_id, job_type, status, stage, attempt_count, + max_attempts, created_at, updated_at + ) VALUES (?, ?, ?, ?, ?, 0, ?, ?, ?) + """, + ( + job_id, + session_id, + job_type, + JobStatus.PENDING.value, + ProcessingStage.QUEUED.value, + max_attempts, + now, + now, + ), + ) + self._conn.commit() + except sqlite3.IntegrityError as exc: + # Roll back before re-raising. A failed INSERT still leaves this + # connection inside the transaction sqlite3 opened implicitly, + # and that transaction holds a write lock -- which would make the + # *worker's* BEGIN IMMEDIATE fail with "database is locked" the + # moment a duplicate upload was rejected. (Found by a smoke test + # doing exactly that sequence.) + self._conn.rollback() + # The partial unique index is the enforcement point for "one + # active transcription job per session". + if "idx_jobs_one_active_per_session" in str(exc) or "UNIQUE" in str(exc): + raise ActiveJobExistsError( + f"Session {session_id!r} already has an active {job_type} job" + ) from exc + raise + except Exception: + self._conn.rollback() + raise + return self.get(job_id) # type: ignore[return-value] + + # -- reads ------------------------------------------------------------- + + def get(self, job_id: str) -> ProcessingJob | None: + row = self._conn.execute( + "SELECT * FROM processing_jobs WHERE id = ?", (job_id,) + ).fetchone() + return _row_to_job(row) if row else None + + def get_latest_for_session(self, session_id: str) -> ProcessingJob | None: + row = self._conn.execute( + """ + SELECT * FROM processing_jobs + WHERE session_id = ? + ORDER BY created_at DESC, rowid DESC + LIMIT 1 + """, + (session_id,), + ).fetchone() + return _row_to_job(row) if row else None + + def count_by_status(self, status: str) -> int: + return self._conn.execute( + "SELECT COUNT(*) AS c FROM processing_jobs WHERE status = ?", (status,) + ).fetchone()["c"] + + def count_touched_since(self, cutoff_iso: str) -> int: + """Jobs a worker has started or updated since `cutoff_iso`. + + The only available evidence that a worker process exists, since + nothing else in this system writes to these columns. + """ + return self._conn.execute( + """ + SELECT COUNT(*) AS c FROM processing_jobs + WHERE started_at IS NOT NULL AND updated_at >= ? + """, + (cutoff_iso,), + ).fetchone()["c"] + + # -- the claim --------------------------------------------------------- + + def claim_next(self, worker_id: str) -> ProcessingJob | None: + """Atomically take ownership of the oldest PENDING job. + + Returns None if there is nothing to do, or if another worker won the + race. Requires an autocommit connection (see module docstring). + """ + now = utc_now_iso() + self._conn.execute("BEGIN IMMEDIATE") + try: + row = self._conn.execute( + """ + SELECT * FROM processing_jobs + WHERE status = ? + ORDER BY created_at ASC, rowid ASC + LIMIT 1 + """, + (JobStatus.PENDING.value,), + ).fetchone() + if row is None: + self._conn.execute("ROLLBACK") + return None + + cursor = self._conn.execute( + """ + UPDATE processing_jobs + SET status = ?, + stage = ?, + worker_id = ?, + started_at = ?, + updated_at = ?, + attempt_count = attempt_count + 1 + WHERE id = ? AND status = ? + """, + ( + JobStatus.RUNNING.value, + ProcessingStage.CLAIMED.value, + worker_id, + now, + now, + row["id"], + JobStatus.PENDING.value, + ), + ) + if cursor.rowcount != 1: + # Belt and braces: the row changed underneath us. + self._conn.execute("ROLLBACK") + return None + job_id = row["id"] + self._conn.execute("COMMIT") + except Exception: + self._conn.execute("ROLLBACK") + raise + + return self.get(job_id) + + # -- progress and outcomes -------------------------------------------- + + def set_stage(self, job_id: str, stage: str) -> None: + self._conn.execute( + "UPDATE processing_jobs SET stage = ?, updated_at = ? WHERE id = ?", + (stage, utc_now_iso(), job_id), + ) + self._conn.commit() + + def mark_succeeded( + self, + job_id: str, + provider_name: str | None = None, + provider_version: str | None = None, + runtime_version: str | None = None, + model_name: str | None = None, + ) -> None: + now = utc_now_iso() + self._conn.execute( + """ + UPDATE processing_jobs + SET status = ?, stage = ?, error_code = NULL, error_message = NULL, + provider_name = ?, provider_version = ?, runtime_version = ?, + model_name = ?, updated_at = ?, completed_at = ? + WHERE id = ? + """, + ( + JobStatus.SUCCEEDED.value, + ProcessingStage.COMPLETED.value, + provider_name, + provider_version, + runtime_version, + model_name, + now, + now, + job_id, + ), + ) + self._conn.commit() + + def mark_failed(self, job_id: str, error_code: str, error_message: str) -> None: + """Terminal failure: this job will not be attempted again.""" + now = utc_now_iso() + self._conn.execute( + """ + UPDATE processing_jobs + SET status = ?, stage = ?, error_code = ?, error_message = ?, + updated_at = ?, completed_at = ? + WHERE id = ? + """, + ( + JobStatus.FAILED.value, + ProcessingStage.FAILED.value, + error_code, + error_message, + now, + now, + job_id, + ), + ) + self._conn.commit() + + def release_for_retry( + self, job_id: str, error_code: str, error_message: str + ) -> None: + """Hand the job back to the queue after a retryable failure. + + `attempt_count` is not reset -- it was already incremented at claim + time, which is what makes the retry budget finite. + """ + now = utc_now_iso() + self._conn.execute( + """ + UPDATE processing_jobs + SET status = ?, stage = ?, worker_id = NULL, started_at = NULL, + error_code = ?, error_message = ?, updated_at = ? + WHERE id = ? + """, + ( + JobStatus.PENDING.value, + ProcessingStage.QUEUED.value, + error_code, + error_message, + now, + job_id, + ), + ) + self._conn.commit() + + # -- crash recovery ---------------------------------------------------- + + def recover_stale(self, stale_after_seconds: int) -> list[ProcessingJob]: + """Reclaim RUNNING jobs whose worker is evidently gone. + + A worker that is SIGKILLed (or whose machine loses power) leaves a + row stuck in RUNNING with nobody working on it. Nothing else will + ever move that row, so on startup and periodically the worker sweeps + for RUNNING jobs older than a documented timeout and either requeues + them (if attempts remain) or fails them terminally. + + The timeout must exceed the slowest plausible transcription, or a + long-but-healthy job would be stolen from the worker still running + it -- see Settings.job_stale_after_seconds. + """ + cutoff = ( + datetime.now(timezone.utc) - timedelta(seconds=stale_after_seconds) + ).isoformat(timespec="seconds").replace("+00:00", "Z") + now = utc_now_iso() + + self._conn.execute("BEGIN IMMEDIATE") + try: + rows = self._conn.execute( + """ + SELECT * FROM processing_jobs + WHERE status = ? AND (started_at IS NULL OR started_at < ?) + """, + (JobStatus.RUNNING.value, cutoff), + ).fetchall() + + recovered: list[str] = [] + for row in rows: + job_id = row["id"] + recovered.append(job_id) + if row["attempt_count"] < row["max_attempts"]: + self._conn.execute( + """ + UPDATE processing_jobs + SET status = ?, stage = ?, worker_id = NULL, + started_at = NULL, error_code = ?, error_message = ?, + updated_at = ? + WHERE id = ? + """, + ( + JobStatus.PENDING.value, + ProcessingStage.QUEUED.value, + "WORKER_LOST", + "Processing was interrupted and has been queued again.", + now, + job_id, + ), + ) + else: + self._conn.execute( + """ + UPDATE processing_jobs + SET status = ?, stage = ?, error_code = ?, + error_message = ?, updated_at = ?, completed_at = ? + WHERE id = ? + """, + ( + JobStatus.FAILED.value, + ProcessingStage.FAILED.value, + "WORKER_LOST", + "Processing was interrupted too many times.", + now, + now, + job_id, + ), + ) + self._conn.execute("COMMIT") + except Exception: + self._conn.execute("ROLLBACK") + raise + + return [job for job in (self.get(jid) for jid in recovered) if job] diff --git a/backend/app/repositories/session_repository.py b/backend/app/repositories/session_repository.py index 610667d..bb4cfe0 100644 --- a/backend/app/repositories/session_repository.py +++ b/backend/app/repositories/session_repository.py @@ -31,6 +31,7 @@ SessionListItem, SessionStatus, Transcript, + TranscriptSegmentOut, ) from app.timeutil import utc_now_iso @@ -52,6 +53,7 @@ def _row_to_session(row: sqlite3.Row) -> Session: prep_seconds_planned=row["prep_seconds_planned"], speaking_seconds_planned=row["speaking_seconds_planned"], client_reported_duration_seconds=row["client_reported_duration_seconds"], + audio_duration_seconds=row["audio_duration_seconds"], status=row["status"], failure_reason=row["failure_reason"], created_at=row["created_at"], @@ -80,7 +82,14 @@ def _row_to_reflection(row: sqlite3.Row) -> Reflection: ) -def _row_to_transcript(row: sqlite3.Row) -> Transcript: +def _row_to_transcript( + row: sqlite3.Row, segments: list[TranscriptSegmentOut] | None = None +) -> Transcript: + raw_parameters = row["parameters_json"] + try: + parameters = json.loads(raw_parameters) if raw_parameters else None + except json.JSONDecodeError: + parameters = None return Transcript( id=row["id"], session_id=row["session_id"], @@ -89,6 +98,15 @@ def _row_to_transcript(row: sqlite3.Row) -> Transcript: text=row["text"], is_mock=bool(row["is_mock"]), created_at=row["created_at"], + detected_language=row["detected_language"], + runtime_version=row["runtime_version"], + model_name=row["model_name"], + model_sha256=row["model_sha256"], + parameters=parameters, + audio_duration_ms=row["audio_duration_ms"], + processing_duration_ms=row["processing_duration_ms"], + real_time_factor=row["real_time_factor"], + segments=segments or [], ) @@ -168,6 +186,17 @@ def get_reflection(self, session_id: str) -> Reflection | None: ... @abstractmethod def save_transcript(self, session_id: str, result: TranscriptResult) -> Transcript: ... + @abstractmethod + def set_audio_duration(self, session_id: str, duration_seconds: float) -> None: ... + + @abstractmethod + def delete_analysis_output(self, session_id: str) -> None: ... + + @abstractmethod + def get_transcript_segments( + self, transcript_id: str + ) -> list[TranscriptSegmentOut]: ... + @abstractmethod def save_metrics(self, session_id: str, result: MetricsResult) -> Metrics: ... @@ -341,15 +370,110 @@ def get_reflection(self, session_id: str) -> Reflection | None: return _row_to_reflection(row) if row else None def save_transcript(self, session_id: str, result: TranscriptResult) -> Transcript: + """Insert the transcript and its segments in ONE transaction. + + A transcript whose segments are missing (because the process died + between two commits) would render as a wall of untimed text and look + like a successful result. Committing both together means the + transcript either exists complete or does not exist at all -- which + is also why the worker only transitions the session to COMPLETED + after this returns (see docs/DATA_FLOW.md). + """ now = utc_now_iso() + transcript_id = new_id() self._conn.execute( - "INSERT INTO transcripts (id, session_id, provider_name, provider_version, text, is_mock, created_at) " - "VALUES (?, ?, ?, ?, ?, ?, ?)", - (new_id(), session_id, result.provider_name, result.provider_version, result.text, int(result.is_mock), now), + """ + INSERT INTO transcripts ( + id, session_id, provider_name, provider_version, text, is_mock, + created_at, detected_language, runtime_version, model_name, + model_sha256, parameters_json, audio_duration_ms, + processing_duration_ms, real_time_factor + ) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?) + """, + ( + transcript_id, + session_id, + result.provider_name, + result.provider_version, + result.text, + int(result.is_mock), + now, + result.detected_language, + result.runtime_version, + result.model_name, + result.model_sha256, + json.dumps(result.parameters) if result.parameters else None, + result.audio_duration_ms, + result.processing_duration_ms, + result.real_time_factor, + ), ) + for segment in result.segments: + self._conn.execute( + """ + INSERT INTO transcript_segments ( + id, transcript_id, segment_index, start_ms, end_ms, text + ) VALUES (?, ?, ?, ?, ?, ?) + """, + ( + new_id(), + transcript_id, + segment.index, + segment.start_ms, + segment.end_ms, + segment.text, + ), + ) self._conn.commit() return self.get_transcript(session_id) # type: ignore[return-value] + def delete_analysis_output(self, session_id: str) -> None: + """Clear transcript/metrics/feedback rows for a session. + + Called before re-saving on a retry. Without it, a job that died + after committing the transcript but before completing would hit the + UNIQUE(session_id) constraint on its second attempt and fail + permanently for a reason that has nothing to do with the audio. + Transcript segments go with the transcript via ON DELETE CASCADE. + """ + self._conn.execute("DELETE FROM transcripts WHERE session_id = ?", (session_id,)) + self._conn.execute("DELETE FROM metrics WHERE session_id = ?", (session_id,)) + self._conn.execute("DELETE FROM feedback WHERE session_id = ?", (session_id,)) + self._conn.commit() + + def set_audio_duration(self, session_id: str, duration_seconds: float) -> None: + """Record the authoritative, decoded duration. + + Deliberately a separate column from client_reported_duration_seconds + rather than an overwrite: keeping both is what lets a browser timer + bug be noticed later instead of silently becoming the truth. + """ + self._conn.execute( + "UPDATE sessions SET audio_duration_seconds = ?, updated_at = ? WHERE id = ?", + (duration_seconds, utc_now_iso(), session_id), + ) + self._conn.commit() + + def get_transcript_segments(self, transcript_id: str) -> list[TranscriptSegmentOut]: + rows = self._conn.execute( + """ + SELECT segment_index, start_ms, end_ms, text + FROM transcript_segments + WHERE transcript_id = ? + ORDER BY segment_index ASC + """, + (transcript_id,), + ).fetchall() + return [ + TranscriptSegmentOut( + segment_index=r["segment_index"], + start_ms=r["start_ms"], + end_ms=r["end_ms"], + text=r["text"], + ) + for r in rows + ] + def save_metrics(self, session_id: str, result: MetricsResult) -> Metrics: now = utc_now_iso() self._conn.execute( @@ -405,7 +529,9 @@ def get_transcript(self, session_id: str) -> Transcript | None: row = self._conn.execute( "SELECT * FROM transcripts WHERE session_id = ?", (session_id,) ).fetchone() - return _row_to_transcript(row) if row else None + if row is None: + return None + return _row_to_transcript(row, self.get_transcript_segments(row["id"])) def get_metrics(self, session_id: str) -> Metrics | None: row = self._conn.execute( diff --git a/backend/app/schemas.py b/backend/app/schemas.py index dcea7aa..87a8912 100644 --- a/backend/app/schemas.py +++ b/backend/app/schemas.py @@ -41,6 +41,35 @@ class SessionStatus(str, Enum): FAILED = "FAILED" +class JobType(str, Enum): + TRANSCRIPTION = "TRANSCRIPTION" + + +class JobStatus(str, Enum): + PENDING = "PENDING" + RUNNING = "RUNNING" + SUCCEEDED = "SUCCEEDED" + FAILED = "FAILED" + + +class ProcessingStage(str, Enum): + """Coarse progress, shown to the user while a job runs. + + Deliberately coarse: these are the steps a person waiting for a + transcript can act on ("it's still converting" vs "the model is + running"), not an internal trace. + """ + + QUEUED = "QUEUED" + CLAIMED = "CLAIMED" + PREPARING_AUDIO = "PREPARING_AUDIO" + TRANSCRIBING = "TRANSCRIBING" + ANALYZING = "ANALYZING" + SAVING = "SAVING" + COMPLETED = "COMPLETED" + FAILED = "FAILED" + + # --- Topics --------------------------------------------------------------- @@ -81,7 +110,12 @@ class Session(BaseModel): mode: SpeakingMode prep_seconds_planned: int speaking_seconds_planned: int + # The browser's own stopwatch. Kept for diagnostic comparison against + # the decoded value below -- see docs/DATA_MODEL.md. client_reported_duration_seconds: int | None + # Measured by decoding the recording. Authoritative; null until the + # worker has decoded the audio. + audio_duration_seconds: float | None = None status: SessionStatus failure_reason: str | None created_at: str @@ -137,6 +171,13 @@ class Reflection(BaseModel): # --- Mock analysis output --------------------------------------------------- +class TranscriptSegmentOut(BaseModel): + segment_index: int + start_ms: int + end_ms: int + text: str + + class Transcript(BaseModel): id: str session_id: str @@ -145,6 +186,17 @@ class Transcript(BaseModel): text: str is_mock: bool created_at: str + # Phase 2A provenance. All optional so Phase 1 rows (which have none of + # it) still deserialize rather than 500-ing when viewed in history. + detected_language: str | None = None + runtime_version: str | None = None + model_name: str | None = None + model_sha256: str | None = None + parameters: dict | None = None + audio_duration_ms: int | None = None + processing_duration_ms: int | None = None + real_time_factor: float | None = None + segments: list[TranscriptSegmentOut] = [] class Metrics(BaseModel): @@ -173,6 +225,41 @@ class Feedback(BaseModel): created_at: str +# --- Processing jobs --------------------------------------------------------- + + +class ProcessingState(BaseModel): + """The narrow, pollable view of "what is happening to this session". + + Everything here is safe to expose: no paths, no worker hostnames, no + subprocess output. `error_code` is a stable identifier the frontend maps + to setup guidance; `error_message` is a pre-redacted sentence. + """ + + session_id: str + session_status: SessionStatus + job_id: str | None = None + job_status: JobStatus | None = None + stage: ProcessingStage | None = None + attempt_count: int = 0 + max_attempts: int = 0 + error_code: str | None = None + error_message: str | None = None + failure_reason: str | None = None + updated_at: str | None = None + + @property + def is_terminal(self) -> bool: + return self.session_status in (SessionStatus.COMPLETED, SessionStatus.FAILED) + + +class RecordingAcceptedResponse(BaseModel): + """The 202 body returned when a recording is accepted for processing.""" + + session: Session + processing: ProcessingState + + class SessionDetail(BaseModel): session: Session recording: RecordingMeta | None @@ -180,6 +267,39 @@ class SessionDetail(BaseModel): transcript: Transcript | None metrics: Metrics | None feedback: Feedback | None + processing: ProcessingState | None = None + + +# --- Runtime diagnostics ----------------------------------------------------- + + +class RuntimeComponent(BaseModel): + available: bool + version: str | None = None + detail: str | None = None + + +class RuntimeCapabilities(BaseModel): + """Non-sensitive local capability report. + + Deliberately excludes absolute paths, usernames, home directories, + environment variables, transcript content, and anything secret -- see + docs/PRIVACY_AND_SECURITY.md. + """ + + transcription_provider: str + worker: RuntimeComponent + ffmpeg: RuntimeComponent + whisper_binary: RuntimeComponent + model: RuntimeComponent + model_name: str + model_size_bytes: int | None = None + platform: str + architecture: str + pinned_release_tag: str + ready: bool + pending_jobs: int = 0 + running_jobs: int = 0 # --- Settings --------------------------------------------------------------- From 45a3cdaea1373fb92fa6f50132c7bde4a9fa63f7 Mon Sep 17 00:00:00 2001 From: Abdulrahman Date: Fri, 31 Jul 2026 21:45:22 +0300 Subject: [PATCH 3/9] Add the ffmpeg audio preprocessing boundary and safe subprocess handling Browsers record Opus-in-WebM or AAC-in-MP4; whisper.cpp reads neither. The decode step between them is a real boundary, so it gets an interface: AudioPreprocessor, with FfmpegAudioPreprocessor behind it. The preprocessor produces 16 kHz mono signed-16-bit PCM WAV, measures the authoritative duration from the decoded WAV header rather than trusting the client, writes only inside a per-job temporary directory, and removes that directory on success and on failure alike. The original recording is opened read-only and never modified. A decode that silently ignores the conversion flags is treated as a failure rather than handed to the model. run_command is now the only place this application starts a process. It takes argv arrays and rejects strings outright, so there is no shell and no quoting class of bug; a timeout is mandatory; output is captured rather than inherited; stdin is closed. ffmpeg and whisper stderr are full of absolute paths, which on a personal machine contain the owner's username. safe_text redacts absolute paths and home references before anything is logged, and raw subprocess output never reaches an exception message or the API. --- backend/app/audio/__init__.py | 0 backend/app/audio/preprocessor.py | 261 ++++++++++++++++++++++++++++++ backend/app/hashing.py | 32 ++++ backend/app/safe_text.py | 61 +++++++ backend/app/subprocess_util.py | 133 +++++++++++++++ 5 files changed, 487 insertions(+) create mode 100644 backend/app/audio/__init__.py create mode 100644 backend/app/audio/preprocessor.py create mode 100644 backend/app/hashing.py create mode 100644 backend/app/safe_text.py create mode 100644 backend/app/subprocess_util.py diff --git a/backend/app/audio/__init__.py b/backend/app/audio/__init__.py new file mode 100644 index 0000000..e69de29 diff --git a/backend/app/audio/preprocessor.py b/backend/app/audio/preprocessor.py new file mode 100644 index 0000000..8867a2a --- /dev/null +++ b/backend/app/audio/preprocessor.py @@ -0,0 +1,261 @@ +"""The audio preprocessing boundary. + +Browsers do not record what a speech model can read. Chrome and Firefox +produce Opus inside a WebM container; Safari produces AAC inside MP4. +whisper.cpp reads none of those -- it wants raw 16 kHz mono signed-16-bit +PCM. Something has to decode one into the other, and that "something" is a +real architectural boundary rather than a utility function, because it is +where an untrusted-ish binary blob from a browser becomes a known-shaped +array of samples (see docs/adr/0007-ffmpeg-preprocessing-boundary.md). + +Responsibilities, all enforced here rather than by the caller: + +- Decode WebM / OGG / MP4-M4A / WAV / MP3 into 16 kHz mono PCM WAV. +- Report the *authoritative* duration, measured from the decoded stream -- + not the browser's stopwatch (see docs/DATA_MODEL.md). +- Write only inside a per-job temporary directory, and remove it afterwards + on success and on failure alike. +- Never mutate the original recording. It is opened read-only and is the + durable artifact; the PCM file is disposable scratch. +- Invoke ffmpeg via an argv array with a timeout, and never surface its raw + stderr (which is full of absolute paths) to the API. +""" + +from __future__ import annotations + +import shutil +import wave +from abc import ABC, abstractmethod +from contextlib import contextmanager +from dataclasses import dataclass +from pathlib import Path +from typing import Iterator + +from app.logging_config import get_logger +from app.subprocess_util import ( + CommandNotFound, + CommandTimeout, + probe_version, + run_command, +) + +logger = get_logger() + +TARGET_SAMPLE_RATE = 16_000 +TARGET_CHANNELS = 1 +TARGET_SAMPLE_WIDTH_BYTES = 2 # signed 16-bit + + +class AudioPreprocessingError(Exception): + """Decoding failed in a way the caller should treat as terminal. + + Carries a stable `code` and an already-safe `message`; the raw ffmpeg + stderr is logged (redacted) but never attached to the exception. + """ + + code = "AUDIO_DECODE_FAILED" + + def __init__(self, message: str, *, code: str | None = None): + super().__init__(message) + if code: + self.code = code + + +class FfmpegNotAvailable(AudioPreprocessingError): + code = "FFMPEG_NOT_FOUND" + + +class AudioPreprocessingTimeout(AudioPreprocessingError): + code = "FFMPEG_TIMEOUT" + + +@dataclass(frozen=True) +class PreparedAudio: + """A decoded, model-ready copy of a recording. + + `path` points inside the per-job temporary directory and is valid only + for the lifetime of the `prepare()` context. + """ + + path: Path + duration_seconds: float + sample_rate: int + channels: int + sample_width_bytes: int + + @property + def duration_ms(self) -> int: + return int(round(self.duration_seconds * 1000)) + + +class AudioPreprocessor(ABC): + """Turns a stored recording into model-ready PCM.""" + + @abstractmethod + def prepare(self, source: Path, work_dir: Path) -> PreparedAudio: + """Decode `source` into a PCM WAV written under `work_dir`.""" + + @abstractmethod + def is_available(self) -> bool: + """Whether the underlying decoder can actually be run right now.""" + + @abstractmethod + def version(self) -> str | None: + """A short version string for diagnostics, or None if unavailable.""" + + +class FfmpegAudioPreprocessor(AudioPreprocessor): + """ffmpeg-backed implementation. + + ffmpeg is an external system dependency, deliberately not vendored and + deliberately not auto-installed -- see scripts/setup_whisper.sh, which + prints the install command rather than running it. + """ + + def __init__(self, ffmpeg_binary: str = "ffmpeg", timeout_seconds: int = 300): + self._binary = ffmpeg_binary + self._timeout = timeout_seconds + + def is_available(self) -> bool: + return shutil.which(self._binary) is not None + + def version(self) -> str | None: + if not self.is_available(): + return None + return probe_version([self._binary, "-version"]) + + def prepare(self, source: Path, work_dir: Path) -> PreparedAudio: + if not self.is_available(): + raise FfmpegNotAvailable( + "ffmpeg is not installed or not on PATH, so the recording " + "cannot be decoded for transcription." + ) + if not source.is_file(): + raise AudioPreprocessingError( + "The stored recording file is missing.", + code="RECORDING_FILE_MISSING", + ) + + work_dir.mkdir(parents=True, exist_ok=True) + destination = work_dir / "audio-16k-mono.wav" + + argv = [ + self._binary, + "-nostdin", # never read the parent's stdin + "-hide_banner", + "-loglevel", "error", + "-y", # overwrite: destination is our own temp file + "-i", str(source), + "-vn", # ignore any video/cover-art stream + "-map", "0:a:0", # first audio stream only + "-ac", str(TARGET_CHANNELS), + "-ar", str(TARGET_SAMPLE_RATE), + "-c:a", "pcm_s16le", + "-f", "wav", + str(destination), + ] + + try: + result = run_command(argv, timeout_seconds=self._timeout) + except CommandNotFound as exc: + raise FfmpegNotAvailable( + "ffmpeg is not installed or not on PATH, so the recording " + "cannot be decoded for transcription." + ) from exc + except CommandTimeout as exc: + destination.unlink(missing_ok=True) + raise AudioPreprocessingTimeout( + "Decoding the recording took too long and was stopped." + ) from exc + + if not result.ok: + # Logged redacted; deliberately not attached to the exception, + # which is what the API may end up echoing. + logger.error( + "ffmpeg failed (exit %s) for command %s: %s", + result.returncode, + result.safe_argv(), + result.safe_stderr(), + ) + destination.unlink(missing_ok=True) + raise AudioPreprocessingError( + "The recording could not be decoded. It may be corrupt or " + "in an unsupported format." + ) + + if not destination.is_file() or destination.stat().st_size == 0: + raise AudioPreprocessingError( + "Decoding produced no audio. The recording may be empty." + ) + + return self._describe(destination) + + def _describe(self, wav_path: Path) -> PreparedAudio: + """Read the authoritative duration back out of the decoded file. + + Using the WAV header (frame count / sample rate) rather than a + second `ffprobe` call keeps this to one subprocess and measures the + exact bytes whisper.cpp will read, which is the number we actually + want to store and to divide inference time by for a real-time + factor. + """ + try: + with wave.open(str(wav_path), "rb") as handle: + channels = handle.getnchannels() + sample_rate = handle.getframerate() + sample_width = handle.getsampwidth() + frames = handle.getnframes() + except (wave.Error, EOFError, OSError) as exc: + raise AudioPreprocessingError( + "The decoded audio file could not be read back." + ) from exc + + if sample_rate <= 0 or frames <= 0: + raise AudioPreprocessingError( + "Decoding produced no audio. The recording may be empty." + ) + if ( + sample_rate != TARGET_SAMPLE_RATE + or channels != TARGET_CHANNELS + or sample_width != TARGET_SAMPLE_WIDTH_BYTES + ): + # A mismatch means ffmpeg silently ignored our conversion flags, + # which would hand whisper.cpp audio it will misread. Fail loudly. + raise AudioPreprocessingError( + "The decoded audio is not in the required 16 kHz mono 16-bit " + "format required for transcription." + ) + + return PreparedAudio( + path=wav_path, + duration_seconds=frames / float(sample_rate), + sample_rate=sample_rate, + channels=channels, + sample_width_bytes=sample_width, + ) + + +@contextmanager +def job_workspace( + processing_dir: Path, job_id: str, *, keep: bool = False +) -> Iterator[Path]: + """A per-job scratch directory, removed on the way out. + + Cleanup runs on the success path and on every failure path, because the + thing being cleaned up is decoded speech -- leaving it behind after a + crash would quietly turn a temp file into retained personal data. `keep` + exists solely for local debugging and is off by default. + """ + work_dir = processing_dir / job_id + work_dir.mkdir(parents=True, exist_ok=True) + try: + yield work_dir + finally: + if keep: + logger.warning( + "Keeping processing workspace for job %s because the debug " + "flag is enabled; it contains decoded audio.", + job_id, + ) + else: + shutil.rmtree(work_dir, ignore_errors=True) diff --git a/backend/app/hashing.py b/backend/app/hashing.py new file mode 100644 index 0000000..fb15e93 --- /dev/null +++ b/backend/app/hashing.py @@ -0,0 +1,32 @@ +"""Content hashing for model files. + +A transcript is only reproducible if you know which weights produced it, and +a model *name* is not enough: `ggml-base.en.bin` has been re-uploaded and +re-quantized upstream more than once. The SHA-256 of the actual bytes is the +thing that pins it. + +Hashing a multi-hundred-megabyte file is not free, so callers cache the +result rather than recomputing it per job -- see app/worker/runner.py, which +computes it once when building the service. +""" + +from __future__ import annotations + +import hashlib +from pathlib import Path + +CHUNK_SIZE = 1024 * 1024 + + +def sha256_of_file(path: Path) -> str | None: + """Return the lowercase hex SHA-256 of `path`, or None if unreadable.""" + if not path.is_file(): + return None + digest = hashlib.sha256() + try: + with path.open("rb") as handle: + while chunk := handle.read(CHUNK_SIZE): + digest.update(chunk) + except OSError: + return None + return digest.hexdigest() diff --git a/backend/app/safe_text.py b/backend/app/safe_text.py new file mode 100644 index 0000000..7055815 --- /dev/null +++ b/backend/app/safe_text.py @@ -0,0 +1,61 @@ +"""Redaction helpers for anything derived from a subprocess or an exception. + +Phase 2A runs two external programs (ffmpeg and whisper-cli) whose stderr +routinely contains absolute paths -- and those paths contain the operating +system username. That text must never reach the API, the job table, or the +application log, because all three are things the owner might reasonably +paste into a public issue. + +The rule enforced here is narrow and mechanical: strip anything that looks +like a filesystem path, cap the length, and collapse whitespace. Callers are +still expected to prefer a stable, human-written error message over relaying +subprocess output at all -- see app/services/transcription_job_service.py, +where the redacted text is used only as a log-side diagnostic detail. +""" + +from __future__ import annotations + +import re + +MAX_DETAIL_CHARS = 400 + +# POSIX absolute paths (/Users/..., /opt/..., /private/var/...) and Windows +# drive paths. Deliberately greedy about what counts as a path character so +# that a partially-quoted path doesn't leak its tail. +_ABSOLUTE_PATH = re.compile(r"(?:[A-Za-z]:)?(?:/|\\\\)[^\s'\"<>|]{2,}") + +# A bare `~` or `~user` home reference. +_HOME_REFERENCE = re.compile(r"~[A-Za-z0-9_.-]*(?:/[^\s'\"<>|]*)?") + +_WHITESPACE = re.compile(r"\s+") + + +def redact_paths(text: str | None) -> str: + """Replace absolute paths and home references with a placeholder. + + Relative fragments (e.g. `ggml-base.en.bin`) survive, because a bare + filename is useful for diagnosis and reveals nothing about the machine. + """ + if not text: + return "" + cleaned = _ABSOLUTE_PATH.sub("", text) + cleaned = _HOME_REFERENCE.sub("", cleaned) + cleaned = _WHITESPACE.sub(" ", cleaned).strip() + return cleaned + + +def safe_detail(text: str | None, *, limit: int = MAX_DETAIL_CHARS) -> str: + """Redact, then truncate, subprocess output intended for a log line.""" + cleaned = redact_paths(text) + if len(cleaned) <= limit: + return cleaned + return cleaned[: limit - 1].rstrip() + "…" + + +def safe_command(argv: list[str] | tuple[str, ...]) -> str: + """Render an argv array for logging with every path argument redacted. + + Used so that "what did we actually run" stays answerable from a log + without publishing the layout of the owner's home directory. + """ + return " ".join(redact_paths(str(arg)) or "" for arg in argv) diff --git a/backend/app/subprocess_util.py b/backend/app/subprocess_util.py new file mode 100644 index 0000000..c2a6679 --- /dev/null +++ b/backend/app/subprocess_util.py @@ -0,0 +1,133 @@ +"""The single place this application starts an external process. + +Every external invocation in Phase 2A (ffmpeg, whisper-cli, and the version +probes for both) goes through `run_command`. Concentrating it here is what +makes the safety properties checkable in one test file rather than argued +about per call site: + +- **argv arrays only, never `shell=True`.** There is no shell, so there is + no quoting/metacharacter class of bug at all. `run_command` rejects a + string command outright rather than helpfully splitting it. +- **A timeout is mandatory.** A wedged inference process must not pin a + worker forever; the caller always states how long it is willing to wait. +- **Output is captured, never inherited.** Subprocess stderr is data to be + redacted (see app/safe_text.py), not something that lands in the terminal + interleaved with application logs. +- **stdin is closed.** ffmpeg in particular will happily consume the + parent's stdin and block; `stdin=DEVNULL` removes that failure mode. +""" + +from __future__ import annotations + +import subprocess +from dataclasses import dataclass +from pathlib import Path + +from app.safe_text import safe_command, safe_detail + + +class CommandNotFound(Exception): + """The executable does not exist or is not executable.""" + + +class CommandTimeout(Exception): + """The command exceeded its timeout and was killed.""" + + +@dataclass(frozen=True) +class CommandResult: + argv: tuple[str, ...] + returncode: int + stdout: str + stderr: str + duration_seconds: float + + @property + def ok(self) -> bool: + return self.returncode == 0 + + def safe_stderr(self, limit: int = 400) -> str: + """stderr with absolute paths stripped, safe to log.""" + return safe_detail(self.stderr, limit=limit) + + def safe_argv(self) -> str: + return safe_command(self.argv) + + +def run_command( + argv: list[str] | tuple[str, ...], + *, + timeout_seconds: float, + cwd: Path | None = None, +) -> CommandResult: + """Run `argv` with a hard timeout, capturing output as text. + + Raises `CommandNotFound` if the executable is missing and + `CommandTimeout` if it outlives `timeout_seconds`. A non-zero exit is + NOT an exception -- it is returned, because callers distinguish + retryable from terminal failures by inspecting the result. + """ + if isinstance(argv, str): + raise TypeError( + "run_command takes an argv array, not a string -- passing a string " + "would require shell interpretation, which is never used here." + ) + args = [str(a) for a in argv] + if not args: + raise ValueError("run_command requires at least the executable path") + if timeout_seconds <= 0: + raise ValueError("run_command requires a positive timeout") + + import time + + started = time.monotonic() + try: + completed = subprocess.run( # noqa: S603 - argv array, shell=False by construction + args, + capture_output=True, + text=True, + errors="replace", + timeout=timeout_seconds, + cwd=str(cwd) if cwd else None, + stdin=subprocess.DEVNULL, + shell=False, + ) + except FileNotFoundError as exc: + raise CommandNotFound(f"Executable not found: {args[0]!r}") from exc + except PermissionError as exc: + raise CommandNotFound(f"Executable is not runnable: {args[0]!r}") from exc + except subprocess.TimeoutExpired as exc: + raise CommandTimeout( + f"Command timed out after {timeout_seconds:g}s" + ) from exc + + return CommandResult( + argv=tuple(args), + returncode=completed.returncode, + stdout=completed.stdout or "", + stderr=completed.stderr or "", + duration_seconds=time.monotonic() - started, + ) + + +def probe_version(argv: list[str], *, timeout_seconds: float = 15.0) -> str | None: + """Best-effort "what version is this binary" probe. + + Returns the first non-empty line of output, redacted, or None if the + command is missing/fails. Version reporting differs between tools (and + between whisper.cpp releases), so this never treats a failed probe as a + fatal condition -- the authoritative pinned version comes from the + runtime manifest written at setup time instead. + """ + try: + result = run_command(argv, timeout_seconds=timeout_seconds) + except (CommandNotFound, CommandTimeout, ValueError): + return None + combined = f"{result.stdout}\n{result.stderr}".strip() + if not combined: + return None + for line in combined.splitlines(): + cleaned = safe_detail(line, limit=200) + if cleaned: + return cleaned + return None From f5513d4688a5fc4d9551b6c9fad45fb5e00726c3 Mon Sep 17 00:00:00 2001 From: Abdulrahman Date: Fri, 31 Jul 2026 21:45:31 +0300 Subject: [PATCH 4/9] Revise the TranscriptionProvider contract for real speech-to-text The Phase 1 docs called this a one-line dependency swap. The provider boundary was right; the signature was not, and three things had to change. transcribe() received session metadata and no audio at all -- a mock can write a sentence from a topic title, a speech model cannot. It now takes a TranscriptionRequest carrying the decoded audio path, the measured duration, the requested language, and explicit model and runtime configuration, rather than reaching for global settings itself. TranscriptResult carried a name, a version, text and is_mock. It now also carries the detected language, ordered segments with millisecond timestamps, model name and SHA-256, runtime version, inference parameters, processing duration and real-time factor. Without those, two transcripts produced by different models are indistinguishable after the fact. WhisperCppTranscriptionProvider parses --output-json rather than scraping console text, so a formatting change upstream cannot silently corrupt a transcript. Missing, unreadable, or structurally wrong output is a provider failure; an empty transcription list is a legitimate "no speech" result and is not. Each failure carries a stable code and says whether a retry could plausibly help. MockTranscriptionProvider stays, implements the new contract, and reports no model or runtime provenance -- but it is no longer the default and is never a silent fallback. If whisper.cpp is unavailable the job fails visibly, because simulated text under a "real transcript" label is worse than an error. --- .../app/providers/transcription_provider.py | 179 +++++++-- backend/app/providers/whisper_cpp_provider.py | 367 ++++++++++++++++++ 2 files changed, 517 insertions(+), 29 deletions(-) create mode 100644 backend/app/providers/whisper_cpp_provider.py diff --git a/backend/app/providers/transcription_provider.py b/backend/app/providers/transcription_provider.py index 484f875..be19965 100644 --- a/backend/app/providers/transcription_provider.py +++ b/backend/app/providers/transcription_provider.py @@ -1,60 +1,181 @@ -"""TranscriptionProvider interface + a deterministic mock implementation. +"""TranscriptionProvider: the contract, revised for real speech-to-text. -The mock never claims to have listened to the audio -- its output is always -prefixed with a simulation notice. Phase 2 swaps `MockTranscriptionProvider` -for a whisper.cpp-backed implementation behind the same interface; nothing -above this layer needs to change (see docs/AI_MODEL_STRATEGY.md). +## Why this interface changed in Phase 2A + +Phase 1 documented the eventual Whisper integration as "write a new class +against the same interface and change one line in dependencies.py". The +*boundary* was right; the *signature* was not, and pretending otherwise +would have meant smuggling real requirements in through the back door. + +The Phase 1 signature was: + + transcribe(session_id, topic_title, mode, duration_seconds) -> TranscriptResult + +Three things were wrong with it for a real provider: + +1. **No audio.** It received metadata about a session, never the recording. + A mock can write a sentence from a topic title; a speech model cannot. + The input had to become "here is a decoded audio file". +2. **No provenance in the output.** `TranscriptResult` carried a name, a + version, some text, and `is_mock`. Real transcription also produces a + detected language, timestamped segments, the model identity and hash, + the runtime version, the parameters used, and how long it took. Without + those, two transcripts made with different models are indistinguishable + after the fact. +3. **Nothing about where it runs.** A provider that shells out to a pinned + binary needs to be told which binary and which weights, rather than + reaching for global configuration itself. + +So the input is now a `TranscriptionRequest` object and the output is a much +richer `TranscriptResult`. This is a deliberate, breaking contract revision. +See docs/AI_MODEL_STRATEGY.md for the corrected description and +docs/adr/0009-pinned-whisper-cpp-runtime.md for the runtime side. + +`MockTranscriptionProvider` still exists and still implements this +interface, but it is no longer the default and is never used as a silent +fallback: if the real provider cannot run, the job fails visibly rather than +producing simulated text that would be mistaken for a transcript. """ from __future__ import annotations from abc import ABC, abstractmethod -from dataclasses import dataclass +from dataclasses import dataclass, field +from pathlib import Path + + +class TranscriptionError(Exception): + """A provider could not produce a transcript. + + `code` is a stable, machine-readable reason. `retryable` says whether + running the same job again could plausibly succeed -- a timeout might, + a missing model file never will (see + app/services/transcription_job_service.py). + """ + + code = "TRANSCRIPTION_FAILED" + retryable = False + + def __init__( + self, + message: str, + *, + code: str | None = None, + retryable: bool | None = None, + ): + super().__init__(message) + if code: + self.code = code + if retryable is not None: + self.retryable = retryable + + +@dataclass(frozen=True) +class ModelConfig: + """Which weights to use, and how to identify them afterwards.""" + + name: str + path: Path + sha256: str | None = None -@dataclass +@dataclass(frozen=True) +class RuntimeConfig: + """Which inference binary to use, and the limits on running it.""" + + binary_path: Path + timeout_seconds: int = 900 + threads: int | None = None + version: str | None = None + + +@dataclass(frozen=True) +class TranscriptionRequest: + """Everything a transcription provider genuinely needs. + + `audio_path` is the *decoded* 16 kHz mono PCM WAV produced by the + AudioPreprocessor, never the original browser recording -- keeping the + decode step outside the provider is what lets the provider stay a thin, + testable wrapper around one subprocess call. + """ + + session_id: str + audio_path: Path + audio_duration_seconds: float + model: ModelConfig + runtime: RuntimeConfig + # None means "let the model detect the language". Not defaulted to + # English: the data model is language-agnostic even though the Phase 2A + # benchmark models are English-only. + language: str | None = None + + +@dataclass(frozen=True) +class TranscriptSegment: + index: int + start_ms: int + end_ms: int + text: str + + +@dataclass(frozen=True) class TranscriptResult: provider_name: str provider_version: str text: str is_mock: bool + detected_language: str | None = None + segments: tuple[TranscriptSegment, ...] = () + model_name: str | None = None + model_sha256: str | None = None + runtime_version: str | None = None + parameters: dict = field(default_factory=dict) + audio_duration_ms: int | None = None + processing_duration_ms: int | None = None + real_time_factor: float | None = None class TranscriptionProvider(ABC): @abstractmethod - def transcribe( - self, - *, - session_id: str, - topic_title: str, - mode: str, - duration_seconds: int | None, - ) -> TranscriptResult: ... + def transcribe(self, request: TranscriptionRequest) -> TranscriptResult: ... class MockTranscriptionProvider(TranscriptionProvider): + """The Phase 1 simulated provider, kept for tests and explicit opt-in. + + It never reads the audio and always reports `is_mock=True` with an + unmissable prefix. Selecting it is a deliberate configuration choice + (`Settings.transcription_provider = "mock"`), never an automatic + fallback from a broken whisper.cpp setup -- a fallback like that would + put simulated text where the UI now says "real transcript". + """ + PROVIDER_NAME = "mock-transcription" - PROVIDER_VERSION = "0.1.0" + PROVIDER_VERSION = "0.2.0" - def transcribe( - self, - *, - session_id: str, - topic_title: str, - mode: str, - duration_seconds: int | None, - ) -> TranscriptResult: - approx_duration = duration_seconds if duration_seconds else "an unknown number of" + def transcribe(self, request: TranscriptionRequest) -> TranscriptResult: + duration = request.audio_duration_seconds text = ( "[SIMULATED TRANSCRIPT -- no real speech-to-text was performed] " - f"This placeholder stands in for a {mode} response about " - f'"{topic_title}", lasting approximately {approx_duration} seconds. ' - "A real transcript will appear here once local Whisper transcription " - "is wired up in a later phase." + f"This placeholder stands in for roughly {duration:.0f} seconds of " + "speech. It is produced without reading the audio and must never " + "be treated as a transcript." ) + duration_ms = int(round(duration * 1000)) return TranscriptResult( provider_name=self.PROVIDER_NAME, provider_version=self.PROVIDER_VERSION, text=text, is_mock=True, + detected_language=None, + segments=( + TranscriptSegment(index=0, start_ms=0, end_ms=duration_ms, text=text), + ), + model_name=None, + model_sha256=None, + runtime_version=None, + parameters={"simulated": True}, + audio_duration_ms=duration_ms, + processing_duration_ms=0, + real_time_factor=0.0, ) diff --git a/backend/app/providers/whisper_cpp_provider.py b/backend/app/providers/whisper_cpp_provider.py new file mode 100644 index 0000000..2662db3 --- /dev/null +++ b/backend/app/providers/whisper_cpp_provider.py @@ -0,0 +1,367 @@ +"""Real local transcription via a pinned whisper.cpp `whisper-cli` binary. + +This provider is a thin, deliberately boring wrapper around one subprocess +call. Everything that could make it interesting has been pushed out: +decoding lives in the AudioPreprocessor, queueing lives in the worker, and +the binary itself is built and pinned by scripts/setup_whisper.sh. + +Two decisions here are load-bearing: + +**Structured output, not screen-scraping.** whisper-cli can print a +human-readable transcript to stdout, and parsing that would work right up +until a release changes its formatting or a progress line interleaves with +the text. Instead the provider asks for `--output-json` and reads the file +whisper.cpp writes. Malformed, missing, or incomplete JSON is treated as a +provider failure -- never as an empty-but-successful transcript, because +"the model heard nothing" and "we failed to read the model's output" must +not collapse into the same result. + +**The runtime is described, not assumed.** The release tag, commit, build +options and model hash come from the manifest that the setup script writes +next to the binary. If the manifest is missing, the provider still runs but +records what it can and says so, rather than inventing a version. +""" + +from __future__ import annotations + +import json +import time +from dataclasses import dataclass +from pathlib import Path + +from app.logging_config import get_logger +from app.providers.transcription_provider import ( + ModelConfig, + RuntimeConfig, + TranscriptionError, + TranscriptionProvider, + TranscriptionRequest, + TranscriptResult, + TranscriptSegment, +) +from app.subprocess_util import ( + CommandNotFound, + CommandTimeout, + probe_version, + run_command, +) + +logger = get_logger() + +PROVIDER_NAME = "whisper-cpp" +PROVIDER_VERSION = "0.1.0" + +# Output basename whisper-cli is told to use inside the job workspace. +_OUTPUT_STEM = "transcript" + + +@dataclass(frozen=True) +class RuntimeManifest: + """What scripts/setup_whisper.sh recorded about this runtime build.""" + + release_tag: str | None = None + commit: str | None = None + build_options: str | None = None + binary_version: str | None = None + + @property + def version_string(self) -> str | None: + """A single human string identifying the runtime build.""" + if self.release_tag and self.commit: + return f"whisper.cpp {self.release_tag} ({self.commit[:12]})" + if self.release_tag: + return f"whisper.cpp {self.release_tag}" + return self.binary_version + + +def load_runtime_manifest(path: Path) -> RuntimeManifest: + """Read the setup-written manifest, tolerating its absence.""" + try: + raw = json.loads(path.read_text()) + except (OSError, json.JSONDecodeError): + return RuntimeManifest() + if not isinstance(raw, dict): + return RuntimeManifest() + return RuntimeManifest( + release_tag=_opt_str(raw.get("release_tag")), + commit=_opt_str(raw.get("commit")), + build_options=_opt_str(raw.get("build_options")), + binary_version=_opt_str(raw.get("binary_version")), + ) + + +def _opt_str(value: object) -> str | None: + return value if isinstance(value, str) and value.strip() else None + + +def probe_binary_version(binary_path: Path) -> str | None: + """Ask the binary what it is. + + whisper-cli's version-reporting flags have not been stable across + releases, so this tries the obvious one and falls back to the first + line of help output. A failure here is informational, not fatal -- the + manifest is the authoritative record. + """ + if not binary_path.is_file(): + return None + return probe_version([str(binary_path), "--version"]) or probe_version( + [str(binary_path), "--help"] + ) + + +class WhisperCppTranscriptionProvider(TranscriptionProvider): + def __init__(self, manifest: RuntimeManifest | None = None): + self._manifest = manifest or RuntimeManifest() + + # -- preflight --------------------------------------------------------- + + def _check_preconditions(self, model: ModelConfig, runtime: RuntimeConfig) -> None: + """Fail fast, terminally, on the two setup mistakes we can detect. + + Neither of these gets better by retrying, so they are marked + non-retryable: a worker that kept re-running a job with no model + file would spin forever and bury the actual instruction the owner + needs to see. + """ + if not runtime.binary_path.is_file(): + raise TranscriptionError( + "The whisper.cpp runtime has not been built yet.", + code="WHISPER_BINARY_MISSING", + retryable=False, + ) + if not model.path.is_file(): + raise TranscriptionError( + f"The transcription model {model.name!r} has not been downloaded yet.", + code="MODEL_MISSING", + retryable=False, + ) + + # -- main entry point -------------------------------------------------- + + def transcribe(self, request: TranscriptionRequest) -> TranscriptResult: + self._check_preconditions(request.model, request.runtime) + + if not request.audio_path.is_file(): + raise TranscriptionError( + "The decoded audio file is missing.", + code="PROCESSED_AUDIO_MISSING", + retryable=False, + ) + + work_dir = request.audio_path.parent + output_stem = work_dir / _OUTPUT_STEM + argv = self._build_argv(request, output_stem) + + started = time.monotonic() + try: + result = run_command(argv, timeout_seconds=request.runtime.timeout_seconds) + except CommandNotFound as exc: + raise TranscriptionError( + "The whisper.cpp runtime has not been built yet.", + code="WHISPER_BINARY_MISSING", + retryable=False, + ) from exc + except CommandTimeout as exc: + # Genuinely retryable: a machine under momentary load can time + # out on a job that would otherwise complete. + raise TranscriptionError( + "Transcription took too long and was stopped.", + code="WHISPER_TIMEOUT", + retryable=True, + ) from exc + processing_seconds = time.monotonic() - started + + if not result.ok: + logger.error( + "whisper-cli failed (exit %s) for command %s: %s", + result.returncode, + result.safe_argv(), + result.safe_stderr(), + ) + raise TranscriptionError( + "The transcription engine exited with an error.", + code="WHISPER_FAILED", + retryable=True, + ) + + payload = self._read_output_json(output_stem) + text, segments, detected_language = self._parse_payload(payload) + + audio_ms = int(round(request.audio_duration_seconds * 1000)) + processing_ms = int(round(processing_seconds * 1000)) + rtf = ( + processing_seconds / request.audio_duration_seconds + if request.audio_duration_seconds > 0 + else None + ) + + return TranscriptResult( + provider_name=PROVIDER_NAME, + provider_version=PROVIDER_VERSION, + text=text, + is_mock=False, + detected_language=detected_language, + segments=segments, + model_name=request.model.name, + model_sha256=request.model.sha256, + runtime_version=( + request.runtime.version + or self._manifest.version_string + or "unknown" + ), + parameters=self._parameters(request), + audio_duration_ms=audio_ms, + processing_duration_ms=processing_ms, + real_time_factor=round(rtf, 4) if rtf is not None else None, + ) + + # -- command construction --------------------------------------------- + + def _build_argv(self, request: TranscriptionRequest, output_stem: Path) -> list[str]: + """Build the argv array. + + Note what is *not* enabled here: no Core ML, no quantized weights, + no VAD. Each of those is a real optimization and each changes output + quality in ways that need measuring before being turned on -- see + docs/AI_MODEL_STRATEGY.md. Metal acceleration, by contrast, is + compiled into the binary on Apple Silicon and needs no flag. + """ + argv = [ + str(request.runtime.binary_path), + "--model", str(request.model.path), + "--file", str(request.audio_path), + "--output-json", + "--output-file", str(output_stem), + "--no-prints", # keep stdout free of progress chatter + ] + if request.language: + argv += ["--language", request.language] + else: + argv += ["--language", "auto"] + if request.runtime.threads: + argv += ["--threads", str(request.runtime.threads)] + return argv + + def _parameters(self, request: TranscriptionRequest) -> dict: + """The inference parameters worth reproducing a run from.""" + return { + "language": request.language or "auto", + "threads": request.runtime.threads, + "translate": False, + "core_ml": False, + "quantized": False, + "vad": False, + "output_format": "json", + } + + # -- output parsing ---------------------------------------------------- + + def _read_output_json(self, output_stem: Path) -> dict: + """Read the JSON file whisper-cli wrote, or fail as a provider error.""" + candidate = output_stem.with_suffix(".json") + if not candidate.is_file(): + # whisper-cli appends .json to the full name in some releases. + alternative = Path(str(output_stem) + ".json") + candidate = alternative if alternative.is_file() else candidate + if not candidate.is_file(): + raise TranscriptionError( + "The transcription engine produced no output file.", + code="WHISPER_OUTPUT_MISSING", + retryable=True, + ) + try: + payload = json.loads(candidate.read_text()) + except (OSError, json.JSONDecodeError) as exc: + raise TranscriptionError( + "The transcription engine produced unreadable output.", + code="WHISPER_OUTPUT_MALFORMED", + retryable=True, + ) from exc + if not isinstance(payload, dict): + raise TranscriptionError( + "The transcription engine produced unreadable output.", + code="WHISPER_OUTPUT_MALFORMED", + retryable=True, + ) + return payload + + def _parse_payload( + self, payload: dict + ) -> tuple[str, tuple[TranscriptSegment, ...], str | None]: + """Turn whisper.cpp's JSON into our segment model. + + Shape (whisper.cpp v1.x `--output-json`): + + {"result": {"language": "en"}, + "transcription": [ + {"offsets": {"from": 0, "to": 5000}, "text": " Hello"} ]} + + A missing or non-list `transcription` key is malformed output, which + is an error. An *empty* list is not: that is a legitimate "no speech + detected" result and produces an empty transcript. + """ + raw_segments = payload.get("transcription") + if not isinstance(raw_segments, list): + raise TranscriptionError( + "The transcription engine's output was missing its transcript.", + code="WHISPER_OUTPUT_MALFORMED", + retryable=True, + ) + + segments: list[TranscriptSegment] = [] + for index, entry in enumerate(raw_segments): + if not isinstance(entry, dict): + raise TranscriptionError( + "The transcription engine's output had an unexpected shape.", + code="WHISPER_OUTPUT_MALFORMED", + retryable=True, + ) + offsets = entry.get("offsets") + if not isinstance(offsets, dict): + raise TranscriptionError( + "The transcription engine's output was missing segment timings.", + code="WHISPER_OUTPUT_MALFORMED", + retryable=True, + ) + start_ms = _coerce_ms(offsets.get("from")) + end_ms = _coerce_ms(offsets.get("to")) + text = entry.get("text") + if start_ms is None or end_ms is None or not isinstance(text, str): + raise TranscriptionError( + "The transcription engine's output was missing segment timings.", + code="WHISPER_OUTPUT_MALFORMED", + retryable=True, + ) + # Clamp rather than reject: a zero-length or slightly inverted + # segment is a rounding artifact, not corrupt output, and the + # DB CHECK constraint requires end >= start. + if end_ms < start_ms: + end_ms = start_ms + segments.append( + TranscriptSegment( + index=index, + start_ms=start_ms, + end_ms=end_ms, + text=text.strip(), + ) + ) + + full_text = " ".join(s.text for s in segments if s.text).strip() + + detected_language = None + result_block = payload.get("result") + if isinstance(result_block, dict): + detected_language = _opt_str(result_block.get("language")) + + return full_text, tuple(segments), detected_language + + +def _coerce_ms(value: object) -> int | None: + """Accept int or float milliseconds; reject anything else.""" + if isinstance(value, bool): + return None + if isinstance(value, int): + return max(0, value) + if isinstance(value, float): + return max(0, int(round(value))) + return None From feada6ebfe143e934df788a9b83faa4efd61b481 Mon Sep 17 00:00:00 2001 From: Abdulrahman Date: Fri, 31 Jul 2026 21:45:44 +0300 Subject: [PATCH 5/9] Move processing out of the request into a separate local worker The upload request now stores the recording, transitions the session to PROCESSING, creates a job row, and returns 202 Accepted. It no longer decodes audio or runs any provider: real transcription of a two-minute clip takes far longer than any reasonable request timeout, and a request that died mid-inference would leave no record that work was owed. The job row is that record. The worker (python -m app.worker) polls, claims one job at a time, releases the database transaction before touching audio, and handles its own migrations so it can start before the API ever has. It sweeps stale RUNNING jobs on startup and periodically, so a worker that was SIGKILLed does not strand a session forever. SIGINT/SIGTERM finish the current job and stop; a second signal still kills. Failures are sorted into retryable (timeouts, transient subprocess crashes) and terminal (no model, no binary, no ffmpeg, undecodable audio). Retrying a missing model file forever would bury the one instruction the owner needs to read. A terminal failure moves the session to FAILED and always preserves the original recording. The session becomes COMPLETED only after the transcript transaction commits, and a retry clears any partial analysis output first so the UNIQUE constraint on transcripts.session_id cannot fail a job for a reason unrelated to the audio. Adds GET /api/sessions/{id}/status as a narrow polling endpoint -- polling the full session detail once a second would drag the transcript across the wire to read one field -- and GET /api/runtime/capabilities, which reports which prerequisite is missing without exposing paths, usernames, home directories, environment variables or transcript content. --- backend/app/dependencies.py | 36 ++- backend/app/main.py | 9 +- backend/app/routers/runtime.py | 22 ++ backend/app/routers/sessions.py | 34 ++- backend/app/services/runtime_service.py | 157 ++++++++++ backend/app/services/session_service.py | 107 ++++--- .../app/services/transcription_job_service.py | 257 ++++++++++++++++ backend/app/worker/__init__.py | 15 + backend/app/worker/__main__.py | 8 + backend/app/worker/runner.py | 277 ++++++++++++++++++ 10 files changed, 865 insertions(+), 57 deletions(-) create mode 100644 backend/app/routers/runtime.py create mode 100644 backend/app/services/runtime_service.py create mode 100644 backend/app/services/transcription_job_service.py create mode 100644 backend/app/worker/__init__.py create mode 100644 backend/app/worker/__main__.py create mode 100644 backend/app/worker/runner.py diff --git a/backend/app/dependencies.py b/backend/app/dependencies.py index 899d978..6197690 100644 --- a/backend/app/dependencies.py +++ b/backend/app/dependencies.py @@ -13,13 +13,12 @@ from app.config import Settings from app.db import get_connection -from app.providers.feedback_provider import FeedbackProvider, MockFeedbackProvider -from app.providers.metrics_provider import MetricsProvider, MockMetricsProvider -from app.providers.transcription_provider import MockTranscriptionProvider, TranscriptionProvider +from app.repositories.job_repository import JobRepository, SqliteJobRepository from app.repositories.recording_storage import LocalRecordingStorage, RecordingStorage from app.repositories.session_repository import SessionRepository, SqliteSessionRepository from app.repositories.settings_repository import SettingsRepository from app.repositories.topic_repository import SqliteTopicRepository, TopicRepository +from app.services.runtime_service import RuntimeService from app.services.session_service import SessionService @@ -53,33 +52,32 @@ def get_recording_storage(settings: Settings = Depends(get_settings)) -> Recordi ) -def get_transcription_provider() -> TranscriptionProvider: - return MockTranscriptionProvider() - - -def get_metrics_provider() -> MetricsProvider: - return MockMetricsProvider() - - -def get_feedback_provider() -> FeedbackProvider: - return MockFeedbackProvider() +def get_job_repository(conn=Depends(get_db)) -> JobRepository: + return SqliteJobRepository(conn) +# NOTE: the API no longer wires up transcription/metrics/feedback providers +# at all. Since Phase 2A those run only inside the worker process (see +# app/worker/runner.py) -- the request path's job is to store the upload and +# queue the work, never to run a model. def get_session_service( settings: Settings = Depends(get_settings), topics: TopicRepository = Depends(get_topic_repository), sessions: SessionRepository = Depends(get_session_repository), recordings: RecordingStorage = Depends(get_recording_storage), - transcription: TranscriptionProvider = Depends(get_transcription_provider), - metrics: MetricsProvider = Depends(get_metrics_provider), - feedback: FeedbackProvider = Depends(get_feedback_provider), + jobs: JobRepository = Depends(get_job_repository), ) -> SessionService: return SessionService( settings=settings, topics=topics, sessions=sessions, recordings=recordings, - transcription=transcription, - metrics=metrics, - feedback=feedback, + jobs=jobs, ) + + +def get_runtime_service( + settings: Settings = Depends(get_settings), + conn=Depends(get_db), +) -> RuntimeService: + return RuntimeService(settings=settings, jobs=SqliteJobRepository(conn)) diff --git a/backend/app/main.py b/backend/app/main.py index 15eab27..645edeb 100644 --- a/backend/app/main.py +++ b/backend/app/main.py @@ -21,7 +21,13 @@ from app.logging_config import configure_logging, get_logger from app.repositories.recording_storage import RecordingStorageError, UnsupportedMediaType, UploadTooLarge from app.repositories.topic_repository import SqliteTopicRepository, load_seed_file -from app.routers import health, sessions, settings as settings_router, topics +from app.routers import ( + health, + runtime, + sessions, + settings as settings_router, + topics, +) from app.services.session_service import ( DurationOutOfRangeError, InvalidSessionStateError, @@ -98,6 +104,7 @@ async def limit_upload_size(request: Request, call_next): app.include_router(topics.router, prefix="/api") app.include_router(sessions.router, prefix="/api") app.include_router(settings_router.router, prefix="/api") + app.include_router(runtime.router, prefix="/api") _register_exception_handlers(app) return app diff --git a/backend/app/routers/runtime.py b/backend/app/routers/runtime.py new file mode 100644 index 0000000..cb0cfb6 --- /dev/null +++ b/backend/app/routers/runtime.py @@ -0,0 +1,22 @@ +"""Local runtime diagnostics. + +Exists so the UI can tell the owner *which* prerequisite is missing instead +of leaving a session spinning in PROCESSING with no explanation. Everything +returned here is non-sensitive by construction -- see +app/services/runtime_service.py for the exclusion list. +""" + +from fastapi import APIRouter, Depends + +from app.dependencies import get_runtime_service +from app.schemas import RuntimeCapabilities +from app.services.runtime_service import RuntimeService + +router = APIRouter(prefix="/runtime", tags=["runtime"]) + + +@router.get("/capabilities", response_model=RuntimeCapabilities) +def capabilities( + service: RuntimeService = Depends(get_runtime_service), +) -> RuntimeCapabilities: + return service.capabilities() diff --git a/backend/app/routers/sessions.py b/backend/app/routers/sessions.py index 281933c..ace216d 100644 --- a/backend/app/routers/sessions.py +++ b/backend/app/routers/sessions.py @@ -4,6 +4,8 @@ from app.dependencies import get_session_repository, get_session_service from app.repositories.session_repository import SessionRepository from app.schemas import ( + ProcessingState, + RecordingAcceptedResponse, ReflectionCreateRequest, Reflection, Session, @@ -36,17 +38,43 @@ def list_sessions( return SessionListResponse(items=page.items, limit=limit, offset=offset, total=page.total) -@router.post("/{session_id}/recording", response_model=Session) +@router.post( + "/{session_id}/recording", + response_model=RecordingAcceptedResponse, + status_code=202, +) async def attach_recording( session_id: str, client_reported_duration_seconds: int = Form(...), file: UploadFile = File(...), service: SessionService = Depends(get_session_service), -) -> Session: +) -> RecordingAcceptedResponse: + """Accept a recording for processing. + + Returns **202 Accepted**, not 200: the recording is durably stored and a + transcription job is queued, but no transcript exists yet. The client is + expected to poll `GET /api/sessions/{id}/status` until the session + reaches COMPLETED or FAILED. Transcription runs in a separate worker + process -- see docs/adr/0006-local-worker-sqlite-queue.md. + """ mime_type = file.content_type or "application/octet-stream" - return await service.attach_recording_and_process( + session, processing = await service.attach_recording_and_enqueue( session_id, file, mime_type, client_reported_duration_seconds ) + return RecordingAcceptedResponse(session=session, processing=processing) + + +@router.get("/{session_id}/status", response_model=ProcessingState) +def get_session_status( + session_id: str, service: SessionService = Depends(get_session_service) +) -> ProcessingState: + """The narrow endpoint the frontend polls while a session processes. + + Deliberately small and cheap: polling `GET /api/sessions/{id}` once a + second would drag the full transcript, metrics, feedback and reflection + across the wire on every tick just to read one status field. + """ + return service.get_processing_state(session_id) @router.post("/{session_id}/reflection", response_model=Reflection) diff --git a/backend/app/services/runtime_service.py b/backend/app/services/runtime_service.py new file mode 100644 index 0000000..704fc1a --- /dev/null +++ b/backend/app/services/runtime_service.py @@ -0,0 +1,157 @@ +"""Local capability reporting: "is transcription actually set up here?" + +The UI needs to answer a specific question when a session sits in PROCESSING +forever: *which* piece is missing -- the worker, ffmpeg, the whisper binary, +or the model. Guessing from a failed job is worse than asking. + +What this deliberately does NOT report, per docs/PRIVACY_AND_SECURITY.md: +absolute filesystem paths, usernames, home directories, environment +variables, transcript content, or anything secret. It reports booleans, short +version strings, a file size, and the coarse platform -- everything here is +safe to paste into a public issue. + +"Worker reachable" is inferred rather than measured: there is no heartbeat +table and no socket to probe, so the signal is "has any job been picked up +or finished recently". That is honest about its own limits -- a worker that +has been idle since before the window looks the same as one that is not +running, which is why the field is described as *recent activity* rather +than liveness. +""" + +from __future__ import annotations + +import platform +import shutil +from dataclasses import dataclass +from datetime import datetime, timedelta, timezone + +from app.audio.preprocessor import FfmpegAudioPreprocessor +from app.config import WHISPER_PINNED_RELEASE_TAG, Settings +from app.providers.whisper_cpp_provider import ( + load_runtime_manifest, + probe_binary_version, +) +from app.repositories.job_repository import SqliteJobRepository +from app.schemas import JobStatus, RuntimeCapabilities, RuntimeComponent + +# How recently a job must have been touched for the worker to count as +# "recently active". +WORKER_ACTIVITY_WINDOW_SECONDS = 120 + + +@dataclass +class RuntimeService: + settings: Settings + jobs: SqliteJobRepository + + def capabilities(self) -> RuntimeCapabilities: + ffmpeg = self._ffmpeg() + whisper = self._whisper() + model = self._model() + worker = self._worker() + + using_mock = self.settings.transcription_provider == "mock" + ready = ffmpeg.available and ( + using_mock or (whisper.available and model.available) + ) + + return RuntimeCapabilities( + transcription_provider=self.settings.transcription_provider, + worker=worker, + ffmpeg=ffmpeg, + whisper_binary=whisper, + model=model, + model_name=self.settings.whisper_model_name, + model_size_bytes=self._model_size(), + platform=platform.system(), + architecture=platform.machine(), + pinned_release_tag=WHISPER_PINNED_RELEASE_TAG, + ready=ready, + pending_jobs=self.jobs.count_by_status(JobStatus.PENDING.value), + running_jobs=self.jobs.count_by_status(JobStatus.RUNNING.value), + ) + + # -- components -------------------------------------------------------- + + def _ffmpeg(self) -> RuntimeComponent: + preprocessor = FfmpegAudioPreprocessor( + self.settings.ffmpeg_binary, self.settings.ffmpeg_timeout_seconds + ) + if not preprocessor.is_available(): + return RuntimeComponent( + available=False, + detail="ffmpeg was not found on PATH.", + ) + # Only the leading "ffmpeg version N" fragment; the full banner + # includes build paths. + raw = preprocessor.version() or "" + return RuntimeComponent( + available=True, + version=raw.split(" Copyright")[0].strip() or None, + ) + + def _whisper(self) -> RuntimeComponent: + binary = self.settings.whisper_binary_path + if not binary.is_file(): + return RuntimeComponent( + available=False, + detail="The whisper.cpp runtime has not been built yet.", + ) + manifest = load_runtime_manifest(self.settings.whisper_manifest_path) + version = manifest.version_string or probe_binary_version(binary) + matches_pin = manifest.release_tag == WHISPER_PINNED_RELEASE_TAG + detail = None + if manifest.release_tag and not matches_pin: + detail = ( + f"Built from {manifest.release_tag}, but this project pins " + f"{WHISPER_PINNED_RELEASE_TAG}." + ) + elif not manifest.release_tag: + detail = "No runtime manifest found; rerun the setup script to record its version." + return RuntimeComponent(available=True, version=version, detail=detail) + + def _model(self) -> RuntimeComponent: + path = self.settings.model_path(self.settings.whisper_model_name) + if not path.is_file(): + return RuntimeComponent( + available=False, + detail=f"Model {self.settings.whisper_model_name!r} has not been downloaded yet.", + ) + return RuntimeComponent(available=True, version=self.settings.whisper_model_name) + + def _model_size(self) -> int | None: + path = self.settings.model_path(self.settings.whisper_model_name) + try: + return path.stat().st_size + except OSError: + return None + + def _worker(self) -> RuntimeComponent: + """Infer recent worker activity from job timestamps.""" + cutoff = ( + datetime.now(timezone.utc) + - timedelta(seconds=WORKER_ACTIVITY_WINDOW_SECONDS) + ).isoformat(timespec="seconds").replace("+00:00", "Z") + active = bool(self.jobs.count_touched_since(cutoff)) + pending = self.jobs.count_by_status(JobStatus.PENDING.value) + if active: + return RuntimeComponent(available=True, detail="Recently processed a job.") + if pending: + return RuntimeComponent( + available=False, + detail=( + "Jobs are waiting but nothing has picked them up. Start the " + "worker with `python -m app.worker`." + ), + ) + return RuntimeComponent( + available=False, + detail=( + "No recent worker activity. This is expected if you have not " + "recorded anything lately." + ), + ) + + +def check_ffmpeg_on_path(binary: str = "ffmpeg") -> bool: + return shutil.which(binary) is not None diff --git a/backend/app/services/session_service.py b/backend/app/services/session_service.py index 0051490..c245989 100644 --- a/backend/app/services/session_service.py +++ b/backend/app/services/session_service.py @@ -5,10 +5,14 @@ writes across SQLite and the filesystem. Repositories below it perform plain CRUD and never guess at business rules. -Phase 1 processing is synchronous: `attach_recording_and_process` stores the -uploaded audio, then runs the three mock providers in the same request and -returns once they're done. There is no queue or background worker yet -- see -docs/ARCHITECTURE.md for why, and what Phase 2's queued worker would change. +Phase 2A moved processing OUT of the request. `attach_recording_and_enqueue` +stores the uploaded audio, moves the session to PROCESSING, creates a durable +job row, and returns immediately with 202 Accepted. Nothing decodes audio or +runs a model inside the HTTP request any more -- real transcription of a +two-minute clip takes far longer than any reasonable request timeout, and a +request that dies mid-inference would leave no record that work was owed. +The job row is that record. See docs/adr/0006-local-worker-sqlite-queue.md +and app/services/transcription_job_service.py for the other half. SQLite and the filesystem are never in one transaction. Each step below is ordered so that a failure leaves the *safer* of the two states rather than a @@ -23,13 +27,15 @@ from app.config import Settings from app.logging_config import get_logger -from app.providers.feedback_provider import FeedbackProvider -from app.providers.metrics_provider import MetricsProvider -from app.providers.transcription_provider import TranscriptionProvider +from app.repositories.job_repository import ActiveJobExistsError, JobRepository from app.repositories.recording_storage import ChunkReader, RecordingStorage, RecordingStorageError from app.repositories.session_repository import SessionRepository from app.repositories.topic_repository import TopicRepository from app.schemas import ( + JobStatus, + JobType, + ProcessingStage, + ProcessingState, ReflectionCreateRequest, Reflection, Session, @@ -95,17 +101,13 @@ def __init__( topics: TopicRepository, sessions: SessionRepository, recordings: RecordingStorage, - transcription: TranscriptionProvider, - metrics: MetricsProvider, - feedback: FeedbackProvider, + jobs: JobRepository, ): self._settings = settings self._topics = topics self._sessions = sessions self._recordings = recordings - self._transcription = transcription - self._metrics = metrics - self._feedback = feedback + self._jobs = jobs def _transition(self, session_id: str, frm: SessionStatus, to: SessionStatus, failure_reason: str | None = None) -> None: assert to in VALID_TRANSITIONS[frm], f"Illegal transition {frm} -> {to}" @@ -139,13 +141,24 @@ def create_session(self, request: SessionCreateRequest) -> Session: # create_session() itself records the None -> CREATED transition. return self._sessions.create_session(request, topic.title, topic.category) - async def attach_recording_and_process( + async def attach_recording_and_enqueue( self, session_id: str, stream: ChunkReader, declared_mime_type: str, client_reported_duration_seconds: int, - ) -> Session: + ) -> tuple[Session, ProcessingState]: + """Store the upload, mark the session PROCESSING, and queue the work. + + Returns promptly. The ordering below is the same + safer-inconsistency-wins discipline as Phase 1 (see + docs/DATA_FLOW.md), with one addition: the job row is created LAST, + after the session is already in PROCESSING. A job that exists always + refers to a session that is genuinely awaiting processing; the + reverse gap (PROCESSING with no job yet) is recoverable and visible, + while a job pointing at a session in CREATED would be a worker + crash waiting to happen. + """ session = self._sessions.get_session(session_id) if session is None: raise SessionNotFoundError(f"Session {session_id!r} does not exist") @@ -173,35 +186,60 @@ async def attach_recording_and_process( self._transition(session_id, SessionStatus.RECORDING_STORED, SessionStatus.PROCESSING) try: - transcript = self._transcription.transcribe( - session_id=session_id, - topic_title=session.topic_title, - mode=session.mode.value, - duration_seconds=client_reported_duration_seconds, - ) - metrics = self._metrics.analyze( - session_id=session_id, duration_seconds=client_reported_duration_seconds - ) - feedback = self._feedback.generate( - session_id=session_id, mode=session.mode.value, metrics=metrics + self._jobs.create( + session_id, + JobType.TRANSCRIPTION.value, + max_attempts=self._settings.worker_max_attempts, ) - self._sessions.save_transcript(session_id, transcript) - self._sessions.save_metrics(session_id, metrics) - self._sessions.save_feedback(session_id, feedback) - self._transition(session_id, SessionStatus.PROCESSING, SessionStatus.COMPLETED) - except Exception: - logger.exception("Mock analysis pipeline failed for session %s", session_id) + except ActiveJobExistsError as exc: + # Only reachable if two uploads race past the CREATED check. + # The recording is kept; the existing job will process it. + logger.warning("Session %s already had an active job", session_id) + raise InvalidSessionStateError( + "This session is already being processed" + ) from exc + except Exception as exc: + logger.exception("Failed to enqueue transcription for session %s", session_id) self._transition( session_id, SessionStatus.PROCESSING, SessionStatus.FAILED, failure_reason=( - "The simulated analysis pipeline failed unexpectedly. " + "The transcription job could not be queued. " "Your recording was preserved." ), ) + raise PersistenceError("Failed to queue transcription") from exc + + updated = self._sessions.get_session(session_id) + return updated, self.get_processing_state(session_id) # type: ignore[arg-type] - return self._sessions.get_session(session_id) # type: ignore[return-value] + def get_processing_state(self, session_id: str) -> ProcessingState: + """The narrow, pollable status view. Never includes paths or output.""" + session = self._sessions.get_session(session_id) + if session is None: + raise SessionNotFoundError(f"Session {session_id!r} does not exist") + job = self._jobs.get_latest_for_session(session_id) + if job is None: + return ProcessingState( + session_id=session_id, + session_status=session.status, + failure_reason=session.failure_reason, + updated_at=session.updated_at, + ) + return ProcessingState( + session_id=session_id, + session_status=session.status, + job_id=job.id, + job_status=JobStatus(job.status), + stage=ProcessingStage(job.stage), + attempt_count=job.attempt_count, + max_attempts=job.max_attempts, + error_code=job.error_code, + error_message=job.error_message, + failure_reason=session.failure_reason, + updated_at=job.updated_at, + ) def save_reflection( self, session_id: str, request: ReflectionCreateRequest @@ -223,6 +261,7 @@ def get_session_detail(self, session_id: str) -> SessionDetail: transcript=self._sessions.get_transcript(session_id) if completed else None, metrics=self._sessions.get_metrics(session_id) if completed else None, feedback=self._sessions.get_feedback(session_id) if completed else None, + processing=self.get_processing_state(session_id), ) def get_recording_path(self, session_id: str) -> tuple[Path, str]: diff --git a/backend/app/services/transcription_job_service.py b/backend/app/services/transcription_job_service.py new file mode 100644 index 0000000..957b1f6 --- /dev/null +++ b/backend/app/services/transcription_job_service.py @@ -0,0 +1,257 @@ +"""What actually happens to one claimed transcription job. + +This is the other half of the split introduced in Phase 2A: `SessionService` +owns the request-side (store the upload, queue the work, answer status +questions) and this service owns the worker-side (decode, transcribe, +analyze, persist, transition). Neither knows how the other is scheduled, +which is what lets the worker be a separate process without the API growing +any knowledge of it. + +## Failure taxonomy + +Every failure is sorted into exactly one of two buckets, because the wrong +answer here is expensive in both directions: + +- **Retryable** -- a timeout, a transient subprocess crash. Trying again + might genuinely work, so the job goes back to PENDING with its attempt + budget decremented. +- **Terminal** -- no model file, no whisper binary, no ffmpeg, a corrupt + recording. Retrying cannot fix any of these; retrying them forever would + burn CPU and, worse, bury the one instruction the owner needs to read. + +A terminal failure (or exhausting the retry budget) moves the session to +FAILED. The original recording is *never* deleted on failure -- it is the +irreplaceable artifact, and everything else can be recomputed from it. + +## Ordering + +The session becomes COMPLETED only after the transcript transaction has +committed. If the process dies between the transcript commit and the status +transition, the session stays PROCESSING and the stale-job sweeper requeues +it -- a duplicate transcript insert would then violate the UNIQUE constraint +on transcripts.session_id, so the retry path deletes any partial transcript +before re-inserting. Losing work is recoverable; showing a COMPLETED session +with no transcript is not. +""" + +from __future__ import annotations + +from dataclasses import dataclass + +from app.audio.preprocessor import ( + AudioPreprocessingError, + AudioPreprocessor, + job_workspace, +) +from app.config import Settings +from app.logging_config import get_logger +from app.providers.feedback_provider import FeedbackProvider +from app.providers.metrics_provider import MetricsProvider +from app.providers.transcription_provider import ( + ModelConfig, + RuntimeConfig, + TranscriptionError, + TranscriptionProvider, + TranscriptionRequest, +) +from app.repositories.job_repository import JobRepository, ProcessingJob +from app.repositories.recording_storage import RecordingStorage +from app.repositories.session_repository import SessionRepository +from app.schemas import ProcessingStage, SessionStatus + +logger = get_logger() + + +@dataclass(frozen=True) +class JobOutcome: + job_id: str + succeeded: bool + error_code: str | None = None + retried: bool = False + + +class TranscriptionJobService: + def __init__( + self, + settings: Settings, + sessions: SessionRepository, + jobs: JobRepository, + recordings: RecordingStorage, + preprocessor: AudioPreprocessor, + transcription: TranscriptionProvider, + metrics: MetricsProvider, + feedback: FeedbackProvider, + model_sha256: str | None = None, + runtime_version: str | None = None, + ): + self._settings = settings + self._sessions = sessions + self._jobs = jobs + self._recordings = recordings + self._preprocessor = preprocessor + self._transcription = transcription + self._metrics = metrics + self._feedback = feedback + self._model_sha256 = model_sha256 + self._runtime_version = runtime_version + + # -- entry point ------------------------------------------------------- + + def process(self, job: ProcessingJob) -> JobOutcome: + """Run one claimed job to a terminal-or-requeued outcome. + + Never raises for an expected failure: the outcome is recorded on the + job row and returned, because a worker loop that crashed on a bad + recording would stop processing every *other* session too. + """ + try: + return self._run(job) + except AudioPreprocessingError as exc: + return self._fail(job, exc.code, str(exc), retryable=_is_retryable_audio(exc)) + except TranscriptionError as exc: + return self._fail(job, exc.code, str(exc), retryable=exc.retryable) + except Exception: # noqa: BLE001 - deliberate catch-all + logger.exception("Unexpected failure processing job %s", job.id) + return self._fail( + job, + "UNEXPECTED_ERROR", + "An unexpected error occurred while processing this recording.", + retryable=False, + ) + + # -- the pipeline ------------------------------------------------------ + + def _run(self, job: ProcessingJob) -> JobOutcome: + session = self._sessions.get_session(job.session_id) + if session is None: + # The session was deleted while the job sat in the queue. That is + # not an error worth retrying or alarming about. + self._jobs.mark_failed( + job.id, "SESSION_DELETED", "The session no longer exists." + ) + return JobOutcome(job.id, succeeded=False, error_code="SESSION_DELETED") + + filename = self._sessions.get_recording_filename(job.session_id) + if filename is None: + raise AudioPreprocessingError( + "This session has no recording to transcribe.", + code="RECORDING_FILE_MISSING", + ) + source_path = self._recordings.path_for(filename) + + with job_workspace( + self._settings.processing_dir, + job.id, + keep=self._settings.keep_processing_files, + ) as work_dir: + # 1. Decode to model-ready PCM and measure the real duration. + self._jobs.set_stage(job.id, ProcessingStage.PREPARING_AUDIO.value) + prepared = self._preprocessor.prepare(source_path, work_dir) + self._sessions.set_audio_duration(job.session_id, prepared.duration_seconds) + + # 2. Real transcription. + self._jobs.set_stage(job.id, ProcessingStage.TRANSCRIBING.value) + transcript = self._transcription.transcribe( + TranscriptionRequest( + session_id=job.session_id, + audio_path=prepared.path, + audio_duration_seconds=prepared.duration_seconds, + model=ModelConfig( + name=self._settings.whisper_model_name, + path=self._settings.model_path(self._settings.whisper_model_name), + sha256=self._model_sha256, + ), + runtime=RuntimeConfig( + binary_path=self._settings.whisper_binary_path, + timeout_seconds=self._settings.whisper_timeout_seconds, + threads=self._settings.whisper_threads, + version=self._runtime_version, + ), + language=self._settings.whisper_language, + ) + ) + + # 3. Still-simulated metrics and feedback. These now run against the + # authoritative decoded duration rather than the browser's timer, + # but they remain mocks and remain labeled as such -- Phase 2A + # deliberately makes ONLY the transcript real (see + # docs/SCORING_AND_LIMITATIONS.md). + self._jobs.set_stage(job.id, ProcessingStage.ANALYZING.value) + duration_for_metrics = int(round(prepared.duration_seconds)) or None + metrics = self._metrics.analyze( + session_id=job.session_id, duration_seconds=duration_for_metrics + ) + feedback = self._feedback.generate( + session_id=job.session_id, + mode=session.mode.value, + metrics=metrics, + ) + + # 4. Persist, then and only then declare the session complete. + self._jobs.set_stage(job.id, ProcessingStage.SAVING.value) + self._sessions.delete_analysis_output(job.session_id) + self._sessions.save_transcript(job.session_id, transcript) + self._sessions.save_metrics(job.session_id, metrics) + self._sessions.save_feedback(job.session_id, feedback) + + self._sessions.set_status( + job.session_id, + SessionStatus.PROCESSING.value, + SessionStatus.COMPLETED.value, + None, + ) + self._jobs.mark_succeeded( + job.id, + provider_name=transcript.provider_name, + provider_version=transcript.provider_version, + runtime_version=transcript.runtime_version, + model_name=transcript.model_name, + ) + logger.info( + "Job %s completed (mock_transcript=%s, rtf=%s)", + job.id, + transcript.is_mock, + transcript.real_time_factor, + ) + return JobOutcome(job.id, succeeded=True) + + # -- failure handling -------------------------------------------------- + + def _fail( + self, job: ProcessingJob, code: str, message: str, *, retryable: bool + ) -> JobOutcome: + attempts_left = job.max_attempts - job.attempt_count + if retryable and attempts_left > 0: + logger.warning( + "Job %s failed with %s; requeuing (%d attempt(s) left)", + job.id, + code, + attempts_left, + ) + self._jobs.release_for_retry(job.id, code, message) + return JobOutcome(job.id, succeeded=False, error_code=code, retried=True) + + logger.error("Job %s failed terminally with %s", job.id, code) + self._jobs.mark_failed(job.id, code, message) + + session = self._sessions.get_session(job.session_id) + if session is not None and session.status == SessionStatus.PROCESSING: + # The recording is deliberately preserved -- see the module + # docstring and docs/DATA_FLOW.md. + self._sessions.set_status( + job.session_id, + SessionStatus.PROCESSING.value, + SessionStatus.FAILED.value, + f"{message} Your recording was preserved.", + ) + return JobOutcome(job.id, succeeded=False, error_code=code) + + +def _is_retryable_audio(exc: AudioPreprocessingError) -> bool: + """Only a decode timeout is worth another go. + + A missing ffmpeg, a missing file, or audio that will not decode produce + the same result every time; retrying them just delays the message the + owner needs to see. + """ + return exc.code == "FFMPEG_TIMEOUT" diff --git a/backend/app/worker/__init__.py b/backend/app/worker/__init__.py new file mode 100644 index 0000000..9246a85 --- /dev/null +++ b/backend/app/worker/__init__.py @@ -0,0 +1,15 @@ +"""The local transcription worker. + +Run it alongside the API: + + python -m app.worker + +It is an ordinary Python process that polls a table, claims a row, does the +work, and writes the result back. It is NOT an "AI agent": it makes no +decisions, has no goals, and cannot choose what to do next -- see +docs/LEARNING_NOTES.md. +""" + +from app.worker.runner import TranscriptionWorker, build_worker + +__all__ = ["TranscriptionWorker", "build_worker"] diff --git a/backend/app/worker/__main__.py b/backend/app/worker/__main__.py new file mode 100644 index 0000000..c5ca73e --- /dev/null +++ b/backend/app/worker/__main__.py @@ -0,0 +1,8 @@ +"""Entry point for `python -m app.worker`.""" + +import sys + +from app.worker.runner import main + +if __name__ == "__main__": + sys.exit(main()) diff --git a/backend/app/worker/runner.py b/backend/app/worker/runner.py new file mode 100644 index 0000000..12b8cf4 --- /dev/null +++ b/backend/app/worker/runner.py @@ -0,0 +1,277 @@ +"""The worker loop: claim one job, do it, repeat. + +Design constraints this loop is built around, each of which came from a +specific failure it would otherwise have: + +- **One job at a time.** A single machine running a single user's + transcriptions gains nothing from concurrency and loses predictability; + two whisper.cpp processes competing for the same cores are slower than one. +- **Never hold a database transaction across the work.** The claim commits + before decoding starts. Anything else would block every other write in the + application for the length of an inference run. +- **Recover from being killed.** On startup, and periodically thereafter, + RUNNING jobs older than the stale timeout are swept back into the queue. + A worker that was SIGKILLed leaves no other trace. +- **Shut down gracefully.** SIGINT/SIGTERM set a flag; the loop finishes the + job it is holding rather than abandoning a half-written transcript. + A second signal is left to the default handler so a wedged worker is still + killable. +- **Logs never contain transcript text or absolute paths.** What gets logged + is job ids, stages, and codes. +""" + +from __future__ import annotations + +import os +import signal +import sqlite3 +import time +from dataclasses import dataclass + +from app.audio.preprocessor import FfmpegAudioPreprocessor +from app.config import Settings, default_settings +from app.db import get_connection, run_migrations +from app.idutil import new_id +from app.logging_config import configure_logging, get_logger +from app.providers.feedback_provider import MockFeedbackProvider +from app.providers.metrics_provider import MockMetricsProvider +from app.providers.transcription_provider import ( + MockTranscriptionProvider, + TranscriptionProvider, +) +from app.providers.whisper_cpp_provider import ( + WhisperCppTranscriptionProvider, + load_runtime_manifest, +) +from app.repositories.job_repository import SqliteJobRepository +from app.repositories.recording_storage import LocalRecordingStorage +from app.repositories.session_repository import SqliteSessionRepository +from app.services.transcription_job_service import TranscriptionJobService +from app.hashing import sha256_of_file + +logger = get_logger() + + +@dataclass +class WorkerStats: + claimed: int = 0 + succeeded: int = 0 + failed: int = 0 + retried: int = 0 + recovered: int = 0 + + +def make_worker_id() -> str: + """Identify this worker without leaking anything sensitive. + + The hostname is deliberately excluded -- on a personal machine it is + frequently the owner's name. A pid plus a random suffix is enough to + tell two workers apart, which is all this is for. + """ + return f"worker-{os.getpid()}-{new_id()[:8]}" + + +def build_transcription_provider( + settings: Settings, +) -> tuple[TranscriptionProvider, str | None]: + """Select the configured provider. No silent fallbacks. + + If whisper.cpp is configured but not installed, this still returns the + whisper provider -- which will fail the job with a clear, actionable + code. Substituting the mock here would silently put simulated text under + a UI label that says "real transcript". + """ + if settings.transcription_provider == "mock": + logger.warning( + "Transcription provider is 'mock' -- transcripts will be SIMULATED." + ) + return MockTranscriptionProvider(), None + manifest = load_runtime_manifest(settings.whisper_manifest_path) + return WhisperCppTranscriptionProvider(manifest), manifest.version_string + + +class TranscriptionWorker: + def __init__(self, settings: Settings, worker_id: str | None = None): + self._settings = settings + self._worker_id = worker_id or make_worker_id() + self._stop_requested = False + self.stats = WorkerStats() + # Autocommit connection: the job claim issues its own explicit + # BEGIN IMMEDIATE (see app/repositories/job_repository.py). + self._conn: sqlite3.Connection | None = None + # Hashing a few hundred MB of weights per job would be wasteful, and + # the file does not change under a running worker. + self._model_sha256: str | None = None + self._model_sha256_computed = False + + # -- lifecycle --------------------------------------------------------- + + @property + def worker_id(self) -> str: + return self._worker_id + + def request_stop(self) -> None: + self._stop_requested = True + + def install_signal_handlers(self) -> None: + def handler(signum, _frame): + logger.info( + "Signal %s received; finishing the current job then stopping.", signum + ) + self.request_stop() + # Restore default handling so a second signal really does kill us. + signal.signal(signum, signal.SIG_DFL) + + signal.signal(signal.SIGINT, handler) + signal.signal(signal.SIGTERM, handler) + + def _connect(self) -> sqlite3.Connection: + if self._conn is None: + self._conn = get_connection(self._settings.db_path, autocommit=True) + return self._conn + + def close(self) -> None: + if self._conn is not None: + self._conn.close() + self._conn = None + + # -- one iteration ----------------------------------------------------- + + def model_sha256(self) -> str | None: + if not self._model_sha256_computed: + self._model_sha256 = sha256_of_file( + self._settings.model_path(self._settings.whisper_model_name) + ) + self._model_sha256_computed = True + return self._model_sha256 + + def build_service(self, conn: sqlite3.Connection) -> TranscriptionJobService: + provider, runtime_version = build_transcription_provider(self._settings) + return TranscriptionJobService( + settings=self._settings, + sessions=SqliteSessionRepository(conn), + jobs=SqliteJobRepository(conn), + recordings=LocalRecordingStorage( + self._settings.recordings_dir, + self._settings.allowed_mime_types, + self._settings.max_upload_bytes, + ), + preprocessor=FfmpegAudioPreprocessor( + self._settings.ffmpeg_binary, self._settings.ffmpeg_timeout_seconds + ), + transcription=provider, + metrics=MockMetricsProvider(), + feedback=MockFeedbackProvider(), + model_sha256=self.model_sha256(), + runtime_version=runtime_version, + ) + + def run_once(self) -> bool: + """Claim and process at most one job. True if one was processed.""" + conn = self._connect() + jobs = SqliteJobRepository(conn) + + job = jobs.claim_next(self._worker_id) + if job is None: + return False + + self.stats.claimed += 1 + logger.info("Claimed job %s (attempt %d/%d)", job.id, job.attempt_count, job.max_attempts) + + service = self.build_service(conn) + outcome = service.process(job) + + if outcome.succeeded: + self.stats.succeeded += 1 + elif outcome.retried: + self.stats.retried += 1 + else: + self.stats.failed += 1 + return True + + def recover_stale_jobs(self) -> int: + conn = self._connect() + recovered = SqliteJobRepository(conn).recover_stale( + self._settings.job_stale_after_seconds + ) + if recovered: + self.stats.recovered += len(recovered) + logger.warning( + "Recovered %d stale job(s) left RUNNING by a previous worker.", + len(recovered), + ) + return len(recovered) + + # -- the loop ---------------------------------------------------------- + + def run_forever(self, max_iterations: int | None = None) -> WorkerStats: + """Poll until stopped. `max_iterations` bounds it for tests.""" + logger.info( + "Transcription worker %s started (provider=%s, model=%s).", + self._worker_id, + self._settings.transcription_provider, + self._settings.whisper_model_name, + ) + self.recover_stale_jobs() + + iterations = 0 + last_sweep = time.monotonic() + while not self._stop_requested: + if max_iterations is not None and iterations >= max_iterations: + break + iterations += 1 + try: + did_work = self.run_once() + except sqlite3.Error: + logger.exception("Database error in worker loop; backing off.") + did_work = False + + # Sweep for stale jobs occasionally, not every tick. + if time.monotonic() - last_sweep > self._settings.job_stale_after_seconds: + self.recover_stale_jobs() + last_sweep = time.monotonic() + + if not did_work: + self._sleep(self._settings.worker_poll_interval_seconds) + + logger.info( + "Worker %s stopping. claimed=%d succeeded=%d failed=%d retried=%d", + self._worker_id, + self.stats.claimed, + self.stats.succeeded, + self.stats.failed, + self.stats.retried, + ) + self.close() + return self.stats + + def _sleep(self, seconds: float) -> None: + """Sleep in short slices so a stop signal is noticed promptly.""" + deadline = time.monotonic() + seconds + while not self._stop_requested and time.monotonic() < deadline: + time.sleep(min(0.1, max(0.0, deadline - time.monotonic()))) + + +def build_worker(settings: Settings | None = None) -> TranscriptionWorker: + settings = settings or default_settings() + return TranscriptionWorker(settings) + + +def main() -> int: + configure_logging() + settings = default_settings() + + # The worker may legitimately start before the API has ever run, so it + # applies migrations itself rather than assuming the schema exists. + conn = get_connection(settings.db_path) + try: + applied = run_migrations(conn, settings.migrations_dir) + if applied: + logger.info("Applied migrations: %s", applied) + finally: + conn.close() + + worker = build_worker(settings) + worker.install_signal_handlers() + worker.run_forever() + return 0 From aca73df62d9aec9cb01238ea35d6649502f40950 Mon Sep 17 00:00:00 2001 From: Abdulrahman Date: Fri, 31 Jul 2026 21:45:54 +0300 Subject: [PATCH 6/9] Update tests for the queued pipeline and add a local benchmark utility The session-service tests now assert the queueing contract: an upload leaves the session in PROCESSING with a PENDING job and no analysis output, the status-event log stops at PROCESSING, a second upload is rejected, deleting a session cascades to its jobs, and reflection works while transcription is still running. The Phase 1 partial-failure tests are kept as-is, plus a new one covering a job row that cannot be written. API tests cover the 202 response shape, the narrow status endpoint, and the capabilities endpoint, asserting that neither leaks an absolute path. The end-to-end test completes the job the way the worker would, so it exercises real-provenance transcript data over HTTP. Test fixtures generate synthetic silent PCM WAVs arithmetically and a synthetic whisper.cpp JSON payload matching the documented shape. No real recording is ever a fixture. benchmark_models.py compares models on a recording the owner supplies, reporting model size, audio duration, processing time, real-time factor, detected language, runtime and model version, normalized transcript, and word error rate against a reference the owner writes. It measures rather than estimating: a missing model or binary is an error, not a plausible number. Input and output stay under storage/benchmarks, which is gitignored, because both contain the owner's voice and words. --- backend/scripts/benchmark_models.py | 258 ++++++++++++++++++++++++++ backend/tests/conftest.py | 67 +++++++ backend/tests/test_api.py | 169 +++++++++++++++-- backend/tests/test_providers.py | 45 ++++- backend/tests/test_session_service.py | 256 ++++++++++++++++--------- 5 files changed, 688 insertions(+), 107 deletions(-) create mode 100644 backend/scripts/benchmark_models.py diff --git a/backend/scripts/benchmark_models.py b/backend/scripts/benchmark_models.py new file mode 100644 index 0000000..2022c54 --- /dev/null +++ b/backend/scripts/benchmark_models.py @@ -0,0 +1,258 @@ +"""Compare transcription models on YOUR OWN recording, locally. + +Run it against a recording you made and (optionally) a reference transcript +you typed yourself: + + cd backend && source .venv/bin/activate + python scripts/benchmark_models.py \ + --audio storage/benchmarks/my-clip.m4a \ + --reference storage/benchmarks/my-clip.txt \ + --models base.en small.en + +Everything it reads and everything it writes stays under +`backend/storage/benchmarks/`, which is gitignored -- the input is your +voice and the output contains your words, so neither belongs in a public +repository. + +This script measures; it does not estimate. If a model file or the whisper +binary is missing it says so and exits, rather than printing a plausible +number. There are no built-in "expected" results to compare against for the +same reason: a benchmark figure that was not measured on the machine in +front of you is worse than no figure at all. + +Reported per model: + model name, model size, audio duration, processing time, real-time factor, + detected language, runtime + model version, normalized transcript, and word + error rate when a reference transcript is supplied. +""" + +from __future__ import annotations + +import argparse +import json +import re +import sys +from dataclasses import asdict, dataclass +from pathlib import Path + +# Allow `python scripts/benchmark_models.py` from the backend directory. +sys.path.insert(0, str(Path(__file__).resolve().parent.parent)) + +from app.audio.preprocessor import ( # noqa: E402 + AudioPreprocessingError, + FfmpegAudioPreprocessor, + job_workspace, +) +from app.config import default_settings # noqa: E402 +from app.hashing import sha256_of_file # noqa: E402 +from app.providers.transcription_provider import ( # noqa: E402 + ModelConfig, + RuntimeConfig, + TranscriptionError, + TranscriptionRequest, +) +from app.providers.whisper_cpp_provider import ( # noqa: E402 + WhisperCppTranscriptionProvider, + load_runtime_manifest, +) +from app.timeutil import utc_now_iso # noqa: E402 + + +# --- Text normalization and WER --------------------------------------------- + + +def normalize(text: str) -> str: + """Lowercase, strip punctuation, collapse whitespace. + + Word error rate is meaningless without agreeing what a "word" is: + without normalization, "Hello," and "hello" count as an error, and the + number measures punctuation conventions rather than recognition. + """ + lowered = text.lower() + lowered = re.sub(r"[^\w\s']", " ", lowered) + return re.sub(r"\s+", " ", lowered).strip() + + +def word_error_rate(reference: str, hypothesis: str) -> float | None: + """Standard Levenshtein-over-words WER. None if there is no reference.""" + ref = normalize(reference).split() + hyp = normalize(hypothesis).split() + if not ref: + return None + + # Two-row dynamic programming table; the full matrix is unnecessary. + previous = list(range(len(hyp) + 1)) + for i, ref_word in enumerate(ref, start=1): + current = [i] + for j, hyp_word in enumerate(hyp, start=1): + cost = 0 if ref_word == hyp_word else 1 + current.append( + min( + previous[j] + 1, # deletion + current[j - 1] + 1, # insertion + previous[j - 1] + cost, # substitution + ) + ) + previous = current + return previous[len(hyp)] / len(ref) + + +# --- Benchmark --------------------------------------------------------------- + + +@dataclass +class ModelReport: + model_name: str + model_size_bytes: int | None + model_sha256: str | None + audio_duration_seconds: float + processing_seconds: float + real_time_factor: float | None + detected_language: str | None + runtime_version: str | None + segment_count: int + word_count: int + word_error_rate: float | None + normalized_transcript: str + + +def benchmark_model( + settings, model_name: str, audio_path: Path, reference: str | None +) -> ModelReport: + model_path = settings.model_path(model_name) + if not model_path.is_file(): + raise SystemExit( + f"Model {model_name!r} is not downloaded " + f"(expected ggml-{model_name}.bin under backend/storage/models).\n" + f"Download it with: scripts/setup_whisper.sh --model {model_name}" + ) + + manifest = load_runtime_manifest(settings.whisper_manifest_path) + provider = WhisperCppTranscriptionProvider(manifest) + preprocessor = FfmpegAudioPreprocessor( + settings.ffmpeg_binary, settings.ffmpeg_timeout_seconds + ) + + with job_workspace(settings.processing_dir, f"benchmark-{model_name}") as work_dir: + prepared = preprocessor.prepare(audio_path, work_dir) + result = provider.transcribe( + TranscriptionRequest( + session_id=f"benchmark-{model_name}", + audio_path=prepared.path, + audio_duration_seconds=prepared.duration_seconds, + model=ModelConfig( + name=model_name, + path=model_path, + sha256=sha256_of_file(model_path), + ), + runtime=RuntimeConfig( + binary_path=settings.whisper_binary_path, + timeout_seconds=settings.whisper_timeout_seconds, + threads=settings.whisper_threads, + version=manifest.version_string, + ), + language=settings.whisper_language, + ) + ) + + normalized = normalize(result.text) + return ModelReport( + model_name=model_name, + model_size_bytes=model_path.stat().st_size, + model_sha256=result.model_sha256, + audio_duration_seconds=round(prepared.duration_seconds, 3), + processing_seconds=round((result.processing_duration_ms or 0) / 1000, 3), + real_time_factor=result.real_time_factor, + detected_language=result.detected_language, + runtime_version=result.runtime_version, + segment_count=len(result.segments), + word_count=len(normalized.split()), + word_error_rate=( + round(word_error_rate(reference, result.text), 4) + if reference + else None + ), + normalized_transcript=normalized, + ) + + +def main() -> int: + settings = default_settings() + + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--audio", required=True, type=Path, help="Your own recording.") + parser.add_argument( + "--reference", + type=Path, + help="Plain-text transcript you wrote yourself, for word error rate.", + ) + parser.add_argument( + "--models", + nargs="+", + default=["base.en", "small.en"], + help="Model names to compare (default: base.en small.en).", + ) + parser.add_argument( + "--output", + type=Path, + default=settings.storage_dir / "benchmarks" / "results.json", + help="Where to write the JSON report (must stay under storage/benchmarks).", + ) + args = parser.parse_args() + + if not args.audio.is_file(): + raise SystemExit(f"Audio file not found: {args.audio.name}") + if not settings.whisper_binary_path.is_file(): + raise SystemExit( + "The whisper.cpp runtime is not built.\n" + "Build it with: scripts/setup_whisper.sh" + ) + + reference = args.reference.read_text() if args.reference else None + if args.reference and not args.reference.is_file(): + raise SystemExit(f"Reference file not found: {args.reference.name}") + + reports: list[ModelReport] = [] + for model_name in args.models: + print(f"== {model_name} ==", flush=True) + try: + report = benchmark_model(settings, model_name, args.audio, reference) + except (TranscriptionError, AudioPreprocessingError) as exc: + print(f" FAILED: {exc}", file=sys.stderr) + continue + reports.append(report) + print(f" model size : {report.model_size_bytes:,} bytes") + print(f" audio duration : {report.audio_duration_seconds:.2f}s") + print(f" processing time : {report.processing_seconds:.2f}s") + print(f" real-time factor: {report.real_time_factor}") + print(f" detected language: {report.detected_language}") + print(f" runtime : {report.runtime_version}") + print(f" segments/words : {report.segment_count}/{report.word_count}") + if report.word_error_rate is not None: + print(f" word error rate : {report.word_error_rate:.2%}") + else: + print(" word error rate : (no reference transcript supplied)") + print(flush=True) + + if not reports: + print("No model produced a result; nothing written.", file=sys.stderr) + return 1 + + args.output.parent.mkdir(parents=True, exist_ok=True) + args.output.write_text( + json.dumps( + { + "generated_at": utc_now_iso(), + "audio_file": args.audio.name, # name only, never the path + "had_reference_transcript": reference is not None, + "results": [asdict(r) for r in reports], + }, + indent=2, + ) + ) + print(f"Wrote {args.output.name} under storage/benchmarks/ (gitignored).") + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/backend/tests/conftest.py b/backend/tests/conftest.py index a3579d7..465e47c 100644 --- a/backend/tests/conftest.py +++ b/backend/tests/conftest.py @@ -1,5 +1,7 @@ from __future__ import annotations +import struct +import wave from pathlib import Path import pytest @@ -26,6 +28,19 @@ def db_conn(settings: Settings): conn.close() +@pytest.fixture +def autocommit_conn(settings: Settings, db_conn): + """A second connection in autocommit mode, as the worker uses. + + Depends on db_conn so the schema exists first. Job claiming needs an + explicit BEGIN IMMEDIATE, which requires isolation_level=None -- see + app/repositories/job_repository.py. + """ + conn = get_connection(settings.db_path, autocommit=True) + yield conn + conn.close() + + @pytest.fixture def client(settings: Settings): app = create_app(settings) @@ -47,3 +62,55 @@ async def read(self, size: int) -> bytes: WEBM_BYTES = b"\x1a\x45\xdf\xa3" + (b"synthetic-test-audio-bytes-" * 50) + + +def write_synthetic_wav( + path: Path, + *, + seconds: float = 1.0, + sample_rate: int = 16_000, + channels: int = 1, + sample_width: int = 2, +) -> Path: + """Write a silent, entirely synthetic PCM WAV. + + Used wherever a test needs a *real, decodable* audio file. It is + generated arithmetic, never a recording of anyone -- no real audio is + ever committed as a fixture (see docs/PRIVACY_AND_SECURITY.md). + """ + path.parent.mkdir(parents=True, exist_ok=True) + frames = int(seconds * sample_rate) + silence = struct.pack(" dict: + """A synthetic whisper.cpp --output-json payload. + + Mirrors the real shape (`result.language` + `transcription[].offsets`) + so the parser is tested against the documented contract rather than + against a convenient fiction. + """ + segments = segments if segments is not None else [(0, 1500, " Hello there.")] + return { + "systeminfo": "synthetic", + "model": {"type": "base"}, + "params": {"language": language or "auto"}, + "result": {"language": language}, + "transcription": [ + { + "timestamps": {"from": "00:00:00,000", "to": "00:00:01,500"}, + "offsets": {"from": start, "to": end}, + "text": text, + } + for start, end, text in segments + ], + } diff --git a/backend/tests/test_api.py b/backend/tests/test_api.py index 948a223..1a6cf52 100644 --- a/backend/tests/test_api.py +++ b/backend/tests/test_api.py @@ -2,9 +2,70 @@ import io +from app.db import get_connection +from app.providers.transcription_provider import TranscriptResult, TranscriptSegment +from app.repositories.job_repository import SqliteJobRepository +from app.repositories.session_repository import SqliteSessionRepository +from app.schemas import SessionStatus from tests.conftest import WEBM_BYTES +def complete_job_like_a_worker(client, session_id: str) -> None: + """Do what the worker would do, without running one. + + Writes a REAL-provenance transcript (is_mock=False) plus mock metrics + and feedback, then transitions the session -- exactly the ordering + TranscriptionJobService uses, so this test exercises the same shape of + data the API will serve in production. + """ + settings = client.app.state.settings + conn = get_connection(settings.db_path) + try: + sessions = SqliteSessionRepository(conn) + jobs = SqliteJobRepository(conn) + from app.providers.feedback_provider import MockFeedbackProvider + from app.providers.metrics_provider import MockMetricsProvider + + sessions.set_audio_duration(session_id, 74.5) + sessions.save_transcript( + session_id, + TranscriptResult( + provider_name="whisper-cpp", + provider_version="0.1.0", + text="Hello there.", + is_mock=False, + detected_language="en", + segments=(TranscriptSegment(index=0, start_ms=0, end_ms=1500, text="Hello there."),), + model_name="base.en", + model_sha256="0" * 64, + runtime_version="whisper.cpp v1.9.1 (f049fff95a08)", + parameters={"language": "en"}, + audio_duration_ms=74500, + processing_duration_ms=12000, + real_time_factor=0.161, + ), + ) + metrics = MockMetricsProvider().analyze(session_id=session_id, duration_seconds=75) + sessions.save_metrics(session_id, metrics) + sessions.save_feedback( + session_id, + MockFeedbackProvider().generate( + session_id=session_id, mode="explain", metrics=metrics + ), + ) + sessions.set_status( + session_id, + SessionStatus.PROCESSING.value, + SessionStatus.COMPLETED.value, + None, + ) + job = jobs.get_latest_for_session(session_id) + if job: + jobs.mark_succeeded(job.id, provider_name="whisper-cpp", model_name="base.en") + finally: + conn.close() + + def test_health(client): r = client.get("/api/health") assert r.status_code == 200 @@ -75,9 +136,36 @@ def test_create_session_with_malformed_payload_returns_422(client): assert r.status_code == 422 -def test_full_workflow_via_the_http_api(client): +def test_upload_returns_202_with_a_queued_processing_state(client): + """The upload endpoint accepts work; it does not complete it.""" topic = _pick_topic(client) + session = client.post( + "/api/sessions", + json={ + "track": "technical", + "topic_id": topic["id"], + "mode": "explain", + "prep_seconds_planned": 60, + "speaking_seconds_planned": 90, + }, + ).json() + r = client.post( + f"/api/sessions/{session['id']}/recording", + data={"client_reported_duration_seconds": "75"}, + files=_webm_file(), + ) + assert r.status_code == 202 + body = r.json() + assert body["session"]["status"] == "PROCESSING" + assert body["processing"]["job_status"] == "PENDING" + assert body["processing"]["stage"] == "QUEUED" + assert body["processing"]["job_id"] + + +def test_status_endpoint_reports_processing_and_hides_internals(client): + topic = _pick_topic(client) + session = client.post( "/api/sessions", json={ "track": "technical", @@ -86,17 +174,53 @@ def test_full_workflow_via_the_http_api(client): "prep_seconds_planned": 60, "speaking_seconds_planned": 90, }, + ).json() + client.post( + f"/api/sessions/{session['id']}/recording", + data={"client_reported_duration_seconds": "75"}, + files=_webm_file(), ) - session = r.json() + + r = client.get(f"/api/sessions/{session['id']}/status") + assert r.status_code == 200 + body = r.json() + assert body["session_status"] == "PROCESSING" + assert body["job_status"] == "PENDING" + assert body["stage"] == "QUEUED" + # The status payload must stay narrow: no transcript, no paths. + assert "transcript" not in body + assert "/Users" not in r.text + + +def test_full_workflow_via_the_http_api(client): + """End-to-end through the real app, with the worker step simulated. + + No worker runs inside the test process, so the job is completed here the + same way the worker would: the transcript is written and the session + transitioned. That keeps this test about the HTTP contract while the + worker's own behavior is covered in test_worker.py. + """ + topic = _pick_topic(client) + session = client.post( + "/api/sessions", + json={ + "track": "technical", + "topic_id": topic["id"], + "mode": "explain", + "prep_seconds_planned": 60, + "speaking_seconds_planned": 90, + }, + ).json() r = client.post( f"/api/sessions/{session['id']}/recording", data={"client_reported_duration_seconds": "75"}, files=_webm_file(), ) - assert r.status_code == 200 - assert r.json()["status"] == "COMPLETED" + assert r.status_code == 202 + # Reflection is allowed WHILE processing -- the user does not have to + # wait for transcription to write down how it felt. r = client.post( f"/api/sessions/{session['id']}/reflection", json={"went_well": "Good structure", "improve": "Slow down", "self_rating": 4}, @@ -104,11 +228,20 @@ def test_full_workflow_via_the_http_api(client): assert r.status_code == 200 assert r.json()["self_rating"] == 4 + # A PROCESSING session must not present analysis output. + detail = client.get(f"/api/sessions/{session['id']}").json() + assert detail["session"]["status"] == "PROCESSING" + assert detail["transcript"] is None + assert detail["processing"]["job_status"] == "PENDING" + + complete_job_like_a_worker(client, session["id"]) + r = client.get(f"/api/sessions/{session['id']}") assert r.status_code == 200 detail = r.json() assert detail["session"]["status"] == "COMPLETED" - assert detail["transcript"]["is_mock"] is True + assert detail["transcript"]["is_mock"] is False + assert detail["transcript"]["segments"][0]["text"] == "Hello there." assert detail["metrics"]["is_mock"] is True assert detail["feedback"]["is_mock"] is True assert detail["reflection"]["self_rating"] == 4 @@ -118,17 +251,29 @@ def test_full_workflow_via_the_http_api(client): assert r.headers["content-type"] == "audio/webm" assert len(r.content) == len(WEBM_BYTES) - r = client.get("/api/sessions") - assert r.status_code == 200 - listing = r.json() + listing = client.get("/api/sessions").json() assert listing["total"] >= 1 assert any(item["id"] == session["id"] for item in listing["items"]) - r = client.delete(f"/api/sessions/{session['id']}") - assert r.status_code == 204 + assert client.delete(f"/api/sessions/{session['id']}").status_code == 204 + assert client.get(f"/api/sessions/{session['id']}").status_code == 404 - r = client.get(f"/api/sessions/{session['id']}") - assert r.status_code == 404 + +def test_runtime_capabilities_reports_availability_without_leaking_paths(client): + r = client.get("/api/runtime/capabilities") + assert r.status_code == 200 + body = r.json() + assert body["transcription_provider"] == "whisper-cpp" + assert body["model_name"] == "base.en" + assert body["pinned_release_tag"] == "v1.9.1" + assert set(body["ffmpeg"]) >= {"available"} + assert isinstance(body["ready"], bool) + + # Nothing sensitive: no absolute paths, no home directory, no username. + text = r.text + assert "/Users" not in text + assert "/home/" not in text + assert "storage/models" not in text def test_attach_unsupported_media_type_returns_415(client): diff --git a/backend/tests/test_providers.py b/backend/tests/test_providers.py index 5ca728d..4e53284 100644 --- a/backend/tests/test_providers.py +++ b/backend/tests/test_providers.py @@ -1,16 +1,30 @@ +from pathlib import Path + from app.providers.feedback_provider import MockFeedbackProvider from app.providers.metrics_provider import MockMetricsProvider -from app.providers.transcription_provider import MockTranscriptionProvider +from app.providers.transcription_provider import ( + ModelConfig, + MockTranscriptionProvider, + RuntimeConfig, + TranscriptionRequest, +) + + +def _request(session_id: str = "s1", duration: float = 90.0) -> TranscriptionRequest: + return TranscriptionRequest( + session_id=session_id, + audio_path=Path("audio-16k-mono.wav"), + audio_duration_seconds=duration, + model=ModelConfig(name="base.en", path=Path("ggml-base.en.bin")), + runtime=RuntimeConfig(binary_path=Path("whisper-cli")), + language="en", + ) def test_transcription_provider_is_deterministic_and_clearly_labeled_simulated(): provider = MockTranscriptionProvider() - first = provider.transcribe( - session_id="s1", topic_title="Explain DNS", mode="explain", duration_seconds=90 - ) - second = provider.transcribe( - session_id="s1", topic_title="Explain DNS", mode="explain", duration_seconds=90 - ) + first = provider.transcribe(_request()) + second = provider.transcribe(_request()) assert first == second assert first.is_mock is True assert "SIMULATED" in first.text @@ -19,12 +33,23 @@ def test_transcription_provider_is_deterministic_and_clearly_labeled_simulated() def test_transcription_provider_never_claims_real_analysis(): provider = MockTranscriptionProvider() - result = provider.transcribe( - session_id="s2", topic_title="Compare TCP and UDP", mode="compare", duration_seconds=None - ) + result = provider.transcribe(_request(session_id="s2", duration=42.0)) assert "no real speech-to-text" in result.text +def test_mock_transcription_provider_reports_no_model_or_runtime_provenance(): + """A simulated transcript must not carry model/runtime metadata. + + Populating those fields would make a mock row indistinguishable from a + real one in the database a year from now. + """ + result = MockTranscriptionProvider().transcribe(_request()) + assert result.model_name is None + assert result.model_sha256 is None + assert result.runtime_version is None + assert result.is_mock is True + + def test_metrics_provider_is_deterministic_per_session_id(): provider = MockMetricsProvider() first = provider.analyze(session_id="session-a", duration_seconds=90) diff --git a/backend/tests/test_session_service.py b/backend/tests/test_session_service.py index 1a6ed2d..165c8a5 100644 --- a/backend/tests/test_session_service.py +++ b/backend/tests/test_session_service.py @@ -1,16 +1,24 @@ +"""SessionService: the request-side half of the pipeline. + +Since Phase 2A the upload path stores audio and *queues* work; it never runs +a model. These tests assert the queueing contract and the unchanged +partial-failure/cleanup discipline. The worker-side half is covered by +test_transcription_job_service.py and test_worker.py. +""" + from __future__ import annotations import asyncio import pytest -from app.providers.feedback_provider import MockFeedbackProvider -from app.providers.metrics_provider import MockMetricsProvider -from app.providers.transcription_provider import MockTranscriptionProvider +from app.repositories.job_repository import SqliteJobRepository from app.repositories.recording_storage import LocalRecordingStorage, RecordingStorageError from app.repositories.session_repository import SqliteSessionRepository from app.repositories.topic_repository import SqliteTopicRepository from app.schemas import ( + JobStatus, + ProcessingStage, ReflectionCreateRequest, SessionCreateRequest, SessionStatus, @@ -28,18 +36,22 @@ from tests.conftest import WEBM_BYTES, FakeUpload -def _make_service(settings, db_conn, sessions_repo=None, recordings_storage=None, - transcription=None, metrics=None, feedback=None): - topics = SqliteTopicRepository(db_conn) +def _make_service( + settings, + db_conn, + sessions_repo=None, + recordings_storage=None, + jobs_repo=None, +): return SessionService( settings=settings, - topics=topics, + topics=SqliteTopicRepository(db_conn), sessions=sessions_repo or SqliteSessionRepository(db_conn), recordings=recordings_storage - or LocalRecordingStorage(settings.recordings_dir, settings.allowed_mime_types, settings.max_upload_bytes), - transcription=transcription or MockTranscriptionProvider(), - metrics=metrics or MockMetricsProvider(), - feedback=feedback or MockFeedbackProvider(), + or LocalRecordingStorage( + settings.recordings_dir, settings.allowed_mime_types, settings.max_upload_bytes + ), + jobs=jobs_repo or SqliteJobRepository(db_conn), ) @@ -61,34 +73,99 @@ def _create_session(service, topic, mode=SpeakingMode.explain): ) -def test_full_happy_path_runs_through_every_status(settings, db_conn): +def _attach(service, session_id, duration=80): + return asyncio.run( + service.attach_recording_and_enqueue( + session_id, FakeUpload(WEBM_BYTES), "audio/webm", duration + ) + ) + + +# --- Queueing behavior ------------------------------------------------------- + + +def test_attaching_a_recording_queues_a_job_and_stops_at_processing(settings, db_conn): + """The upload request must NOT produce a transcript. + + This is the whole point of the Phase 2A split: the request returns while + the session is still PROCESSING, with a durable job row describing the + work that is owed. + """ service = _make_service(settings, db_conn) - topic = _pick_topic(service) - session = _create_session(service, topic) - assert session.status == SessionStatus.CREATED + session = _create_session(service, _pick_topic(service)) - updated = asyncio.run( - service.attach_recording_and_process(session.id, FakeUpload(WEBM_BYTES), "audio/webm", 80) - ) - assert updated.status == SessionStatus.COMPLETED - assert updated.client_reported_duration_seconds == 80 + updated, processing = _attach(service, session.id) + + assert updated.status == SessionStatus.PROCESSING + assert processing.job_status == JobStatus.PENDING + assert processing.stage == ProcessingStage.QUEUED + assert processing.attempt_count == 0 + assert processing.max_attempts == settings.worker_max_attempts + + # No analysis output exists yet, and the detail view must not invent any. + detail = service.get_session_detail(session.id) + assert detail.transcript is None + assert detail.metrics is None + assert detail.feedback is None + assert detail.recording is not None + + +def test_status_events_stop_at_processing_until_a_worker_runs(settings, db_conn): + service = _make_service(settings, db_conn) + session = _create_session(service, _pick_topic(service)) + _attach(service, session.id) rows = db_conn.execute( - "SELECT to_status FROM session_status_events WHERE session_id = ? ORDER BY occurred_at, rowid", + "SELECT to_status FROM session_status_events WHERE session_id = ? " + "ORDER BY occurred_at, rowid", (session.id,), ).fetchall() assert [r["to_status"] for r in rows] == [ "CREATED", "RECORDING_STORED", "PROCESSING", - "COMPLETED", ] - detail = service.get_session_detail(session.id) - assert detail.transcript is not None - assert detail.metrics is not None - assert detail.feedback is not None - assert detail.recording is not None + +def test_only_one_active_job_can_exist_per_session(settings, db_conn): + """Enforced by the database's partial unique index, not by a Python check.""" + service = _make_service(settings, db_conn) + session = _create_session(service, _pick_topic(service)) + _attach(service, session.id) + + jobs = SqliteJobRepository(db_conn) + from app.repositories.job_repository import ActiveJobExistsError + + with pytest.raises(ActiveJobExistsError): + jobs.create(session.id) + + +def test_second_upload_to_a_processing_session_is_rejected(settings, db_conn): + service = _make_service(settings, db_conn) + session = _create_session(service, _pick_topic(service)) + _attach(service, session.id) + + with pytest.raises(InvalidSessionStateError): + _attach(service, session.id) + + +def test_processing_state_is_reported_for_a_session_with_no_job(settings, db_conn): + service = _make_service(settings, db_conn) + session = _create_session(service, _pick_topic(service)) + + state = service.get_processing_state(session.id) + assert state.session_status == SessionStatus.CREATED + assert state.job_id is None + assert state.job_status is None + + +def test_processing_state_for_a_missing_session_raises(settings, db_conn): + service = _make_service(settings, db_conn) + with pytest.raises(SessionNotFoundError): + service.get_processing_state("does-not-exist") + + +# --- Validation (unchanged from Phase 1) ------------------------------------- def test_create_session_rejects_mode_incompatible_with_topic(settings, db_conn): @@ -122,29 +199,14 @@ def test_create_session_rejects_duration_outside_configured_bounds(settings, db_ ) -def test_attach_recording_rejects_a_session_not_in_created_state(settings, db_conn): - service = _make_service(settings, db_conn) - topic = _pick_topic(service) - session = _create_session(service, topic) - asyncio.run( - service.attach_recording_and_process(session.id, FakeUpload(WEBM_BYTES), "audio/webm", 60) - ) - with pytest.raises(InvalidSessionStateError): - asyncio.run( - service.attach_recording_and_process(session.id, FakeUpload(WEBM_BYTES), "audio/webm", 60) - ) - - def test_attach_recording_rejects_out_of_range_client_reported_duration(settings, db_conn): service = _make_service(settings, db_conn) - topic = _pick_topic(service) - session = _create_session(service, topic) + session = _create_session(service, _pick_topic(service)) with pytest.raises(DurationOutOfRangeError): - asyncio.run( - service.attach_recording_and_process( - session.id, FakeUpload(WEBM_BYTES), "audio/webm", -5 - ) - ) + _attach(service, session.id, duration=-5) + + +# --- Partial-failure discipline ---------------------------------------------- def test_orphan_file_is_removed_when_recording_metadata_insert_fails(settings, db_conn): @@ -176,52 +238,47 @@ def __getattr__(self, item): def attach_recording(self, *args, **kwargs): raise RuntimeError("simulated database failure") - service = _make_service(settings, db_conn, sessions_repo=FailingAttachSessions(), recordings_storage=recordings) + service = _make_service( + settings, db_conn, sessions_repo=FailingAttachSessions(), recordings_storage=recordings + ) with pytest.raises(PersistenceError): - asyncio.run( - service.attach_recording_and_process(session.id, FakeUpload(WEBM_BYTES), "audio/webm", 60) - ) + _attach(service, session.id, duration=60) assert list(settings.recordings_dir.iterdir()) == [] refreshed = real_sessions.get_session(session.id) assert refreshed.status == SessionStatus.CREATED # never transitioned -def test_provider_failure_marks_session_failed_and_preserves_the_recording(settings, db_conn): - class FailingMetricsProvider: - def analyze(self, *, session_id, duration_seconds): - raise RuntimeError("simulated provider crash") +def test_failure_to_queue_marks_the_session_failed_and_keeps_the_recording( + settings, db_conn +): + """If the job row cannot be written, the session must not sit in + PROCESSING forever waiting for work that was never queued.""" - service = _make_service(settings, db_conn, metrics=FailingMetricsProvider()) - topic = _pick_topic(service) - session = _create_session(service, topic) + class FailingJobs: + def create(self, *args, **kwargs): + raise RuntimeError("simulated job-table failure") - updated = asyncio.run( - service.attach_recording_and_process(session.id, FakeUpload(WEBM_BYTES), "audio/webm", 60) - ) - assert updated.status == SessionStatus.FAILED - assert updated.failure_reason is not None - assert "preserved" in updated.failure_reason.lower() + def get_latest_for_session(self, session_id): + return None - # The recording file itself was NOT deleted. - assert len(list(settings.recordings_dir.iterdir())) == 1 + service = _make_service(settings, db_conn, jobs_repo=FailingJobs()) + session = _create_session(service, _pick_topic(service)) - # A failed session must never present completed-looking results. - detail = service.get_session_detail(session.id) - assert detail.transcript is None - assert detail.metrics is None - assert detail.feedback is None - assert detail.recording is not None + with pytest.raises(PersistenceError): + _attach(service, session.id) + + refreshed = SqliteSessionRepository(db_conn).get_session(session.id) + assert refreshed.status == SessionStatus.FAILED + assert "preserved" in (refreshed.failure_reason or "").lower() + assert len(list(settings.recordings_dir.iterdir())) == 1 def test_delete_session_removes_db_rows_and_the_recording_file(settings, db_conn): service = _make_service(settings, db_conn) - topic = _pick_topic(service) - session = _create_session(service, topic) - asyncio.run( - service.attach_recording_and_process(session.id, FakeUpload(WEBM_BYTES), "audio/webm", 60) - ) + session = _create_session(service, _pick_topic(service)) + _attach(service, session.id) assert len(list(settings.recordings_dir.iterdir())) == 1 service.delete_session(session.id) @@ -231,7 +288,20 @@ def test_delete_session_removes_db_rows_and_the_recording_file(settings, db_conn service.get_session_detail(session.id) -def test_delete_session_reports_failure_and_keeps_data_when_file_deletion_fails(settings, db_conn): +def test_deleting_a_session_cascades_to_its_jobs(settings, db_conn): + service = _make_service(settings, db_conn) + session = _create_session(service, _pick_topic(service)) + _attach(service, session.id) + assert db_conn.execute("SELECT COUNT(*) AS c FROM processing_jobs").fetchone()["c"] == 1 + + service.delete_session(session.id) + + assert db_conn.execute("SELECT COUNT(*) AS c FROM processing_jobs").fetchone()["c"] == 0 + + +def test_delete_session_reports_failure_and_keeps_data_when_file_deletion_fails( + settings, db_conn +): """If the filesystem delete fails, we must not silently delete the DB rows and pretend the recording is gone -- report the failure and leave both sides intact so the operation can be retried.""" @@ -251,11 +321,8 @@ def delete(self, filename): service = _make_service( settings, db_conn, sessions_repo=real_sessions, recordings_storage=real_recordings ) - topic = _pick_topic(service) - session = _create_session(service, topic) - asyncio.run( - service.attach_recording_and_process(session.id, FakeUpload(WEBM_BYTES), "audio/webm", 60) - ) + session = _create_session(service, _pick_topic(service)) + _attach(service, session.id, duration=60) service._recordings = FailingDeleteStorage() with pytest.raises(RecordingStorageError): @@ -266,10 +333,29 @@ def delete(self, filename): assert len(list(settings.recordings_dir.iterdir())) == 1 +# --- Reflection (must work DURING processing) -------------------------------- + + +def test_reflection_can_be_saved_while_the_session_is_still_processing(settings, db_conn): + """The user reflects while transcription runs; that must not be blocked.""" + service = _make_service(settings, db_conn) + session = _create_session(service, _pick_topic(service)) + _attach(service, session.id) + + saved = service.save_reflection( + session.id, + ReflectionCreateRequest(went_well="Clear intro", improve="Pace", self_rating=3), + ) + assert saved.self_rating == 3 + assert ( + SqliteSessionRepository(db_conn).get_session(session.id).status + == SessionStatus.PROCESSING + ) + + def test_reflection_can_be_saved_and_updated(settings, db_conn): service = _make_service(settings, db_conn) - topic = _pick_topic(service) - session = _create_session(service, topic) + session = _create_session(service, _pick_topic(service)) first = service.save_reflection( session.id, ReflectionCreateRequest(went_well="Clear intro", improve="Pace", self_rating=3) From 1eeec58b074b7425765e275476bef73de4b6b75b Mon Sep 17 00:00:00 2001 From: Abdulrahman Date: Fri, 31 Jul 2026 21:50:24 +0300 Subject: [PATCH 7/9] Add tests for migrations, the job queue, the worker, and decoding Migration tests apply 0001, populate it with Phase 1 shaped rows, and only then apply 0002 -- testing migrations against a fresh database only is how a migration that works in CI destroys the one database that matters. They assert existing rows survive untouched, the one-active-job index is enforced by SQLite, and segment ordering and cascade constraints hold. Worker tests cover the guarantee the queue exists for: two workers on separate connections cannot claim the same job. Also oldest-first ordering, the success path writing a real transcript with ordered segments and full provenance, retryable failure requeuing while the session stays PROCESSING, terminal failure failing the session while preserving the recording, a finite retry budget, stale-RUNNING recovery, restart behavior, graceful shutdown, temp-file cleanup after both success and failure, and that an unexpected error cannot escape and kill the loop. Preprocessor tests assert argv arrays with the required conversion flags, that a string command is refused outright, that shell metacharacters are never interpreted, mandatory timeouts, and that ffmpeg stderr containing an absolute path never reaches the exception a caller might surface. whisper provider tests cover argv construction and every way its JSON can be wrong: missing file, unparseable, missing transcription key, missing or non-numeric offsets. An empty transcription list stays a legitimate empty result rather than an error. The real whisper.cpp integration test is opt-in and skips with the exact missing prerequisite named. On this machine it skips: ffmpeg is not installed, so no real-model run has occurred. --- backend/tests/test_audio_preprocessor.py | 319 ++++++++++++ backend/tests/test_migrations.py | 210 ++++++++ backend/tests/test_whisper_integration.py | 162 ++++++ backend/tests/test_whisper_provider.py | 326 ++++++++++++ backend/tests/test_worker.py | 577 ++++++++++++++++++++++ 5 files changed, 1594 insertions(+) create mode 100644 backend/tests/test_audio_preprocessor.py create mode 100644 backend/tests/test_migrations.py create mode 100644 backend/tests/test_whisper_integration.py create mode 100644 backend/tests/test_whisper_provider.py create mode 100644 backend/tests/test_worker.py diff --git a/backend/tests/test_audio_preprocessor.py b/backend/tests/test_audio_preprocessor.py new file mode 100644 index 0000000..c1cfa2d --- /dev/null +++ b/backend/tests/test_audio_preprocessor.py @@ -0,0 +1,319 @@ +"""AudioPreprocessor and the safe-subprocess layer. + +ffmpeg is mocked throughout. The regular suite must run on a machine with no +ffmpeg installed, and a test that shells out to a real encoder is testing +ffmpeg rather than this code. The one place a real ffmpeg is used is the +opt-in integration test in test_whisper_integration.py. +""" + +from __future__ import annotations + +import wave +from pathlib import Path + +import pytest + +from app.audio.preprocessor import ( + AudioPreprocessingError, + AudioPreprocessingTimeout, + FfmpegAudioPreprocessor, + FfmpegNotAvailable, + job_workspace, +) +from app.safe_text import redact_paths, safe_command, safe_detail +from app.subprocess_util import ( + CommandNotFound, + CommandResult, + CommandTimeout, + run_command, +) +from tests.conftest import write_synthetic_wav + + +# --- Safe subprocess construction -------------------------------------------- + + +def test_run_command_refuses_a_string_command(): + """A string would require shell interpretation. There is no shell here.""" + with pytest.raises(TypeError, match="argv array"): + run_command("ffmpeg -i in.webm out.wav", timeout_seconds=5) + + +def test_run_command_requires_a_positive_timeout(): + with pytest.raises(ValueError): + run_command(["echo", "hi"], timeout_seconds=0) + + +def test_run_command_raises_command_not_found_for_a_missing_binary(): + with pytest.raises(CommandNotFound): + run_command(["definitely-not-a-real-binary-xyz"], timeout_seconds=5) + + +def test_run_command_captures_output_and_reports_nonzero_without_raising(): + result = run_command(["sh", "-c", "echo out; echo err >&2; exit 3"], timeout_seconds=10) + assert result.returncode == 3 + assert result.ok is False + assert "out" in result.stdout + assert "err" in result.stderr + + +def test_run_command_enforces_its_timeout(): + with pytest.raises(CommandTimeout): + run_command(["sleep", "5"], timeout_seconds=0.3) + + +def test_run_command_does_not_interpret_shell_metacharacters(): + """The argument is data, not a command fragment.""" + payload = "hello; touch /tmp/should-not-exist-speaklab" + result = run_command(["echo", payload], timeout_seconds=10) + assert payload in result.stdout + assert not Path("/tmp/should-not-exist-speaklab").exists() + + +# --- Redaction --------------------------------------------------------------- + + +def test_redact_paths_removes_absolute_paths_and_home_references(): + text = "Error opening /Users/someone/Projects/audio.webm and ~/other.wav" + cleaned = redact_paths(text) + assert "/Users/" not in cleaned + assert "someone" not in cleaned + assert "~" not in cleaned + assert "" in cleaned + + +def test_redact_paths_keeps_bare_filenames_which_are_useful_and_harmless(): + assert "ggml-base.en.bin" in redact_paths("could not load ggml-base.en.bin") + + +def test_safe_detail_truncates_long_output(): + assert len(safe_detail("x" * 5000, limit=100)) <= 100 + + +def test_safe_command_redacts_every_path_argument(): + rendered = safe_command( + ["/opt/whisper/bin/whisper-cli", "--model", "/Users/me/models/ggml-base.en.bin"] + ) + assert "/Users" not in rendered + assert "/opt" not in rendered + + +def test_command_result_safe_stderr_is_redacted(): + result = CommandResult( + argv=("ffmpeg",), + returncode=1, + stdout="", + stderr="/Users/someone/recordings/clip.webm: Invalid data", + duration_seconds=0.1, + ) + assert "/Users" not in result.safe_stderr() + assert "Invalid data" in result.safe_stderr() + + +# --- Workspace lifecycle ----------------------------------------------------- + + +def test_job_workspace_is_removed_after_success(tmp_path): + with job_workspace(tmp_path / "processing", "job-1") as work_dir: + (work_dir / "scratch.wav").write_bytes(b"data") + assert work_dir.exists() + assert not (tmp_path / "processing" / "job-1").exists() + + +def test_job_workspace_is_removed_after_failure(tmp_path): + """Decoded speech must not survive a crash as a stray temp file.""" + with pytest.raises(RuntimeError): + with job_workspace(tmp_path / "processing", "job-2") as work_dir: + (work_dir / "scratch.wav").write_bytes(b"data") + raise RuntimeError("boom") + assert not (tmp_path / "processing" / "job-2").exists() + + +def test_job_workspace_can_be_kept_for_local_debugging(tmp_path): + with job_workspace(tmp_path / "processing", "job-3", keep=True) as work_dir: + (work_dir / "scratch.wav").write_bytes(b"data") + assert (tmp_path / "processing" / "job-3" / "scratch.wav").exists() + + +# --- FfmpegAudioPreprocessor ------------------------------------------------- + + +class _FakeRun: + """Stands in for run_command, optionally writing a decoded output file.""" + + def __init__(self, *, returncode=0, stderr="", produce: bool = True, raises=None): + self.returncode = returncode + self.stderr = stderr + self.produce = produce + self.raises = raises + self.calls: list[list[str]] = [] + + def __call__(self, argv, *, timeout_seconds, cwd=None): + self.calls.append(list(argv)) + if self.raises: + raise self.raises + if self.produce: + write_synthetic_wav(Path(argv[-1]), seconds=2.0) + return CommandResult( + argv=tuple(argv), + returncode=self.returncode, + stdout="", + stderr=self.stderr, + duration_seconds=0.05, + ) + + +def _preprocessor(monkeypatch, fake, available=True): + monkeypatch.setattr("app.audio.preprocessor.run_command", fake) + monkeypatch.setattr( + "app.audio.preprocessor.shutil.which", + lambda _binary: "/usr/bin/ffmpeg" if available else None, + ) + return FfmpegAudioPreprocessor("ffmpeg", timeout_seconds=30) + + +def test_prepare_produces_16khz_mono_pcm_and_measures_real_duration(monkeypatch, tmp_path): + fake = _FakeRun() + preprocessor = _preprocessor(monkeypatch, fake) + source = write_synthetic_wav(tmp_path / "original.wav", seconds=5.0) + + prepared = preprocessor.prepare(source, tmp_path / "work") + + assert prepared.sample_rate == 16_000 + assert prepared.channels == 1 + assert prepared.sample_width_bytes == 2 + # Duration comes from the DECODED file (2.0s), not the 5s source or any + # client-reported figure. + assert prepared.duration_seconds == pytest.approx(2.0) + assert prepared.duration_ms == 2000 + + +def test_prepare_builds_a_safe_argv_with_the_required_conversion_flags( + monkeypatch, tmp_path +): + fake = _FakeRun() + preprocessor = _preprocessor(monkeypatch, fake) + source = write_synthetic_wav(tmp_path / "original.wav") + + preprocessor.prepare(source, tmp_path / "work") + + argv = fake.calls[0] + assert isinstance(argv, list) + assert argv[0] == "ffmpeg" + assert "-nostdin" in argv + for flag, value in (("-ac", "1"), ("-ar", "16000"), ("-c:a", "pcm_s16le")): + assert argv[argv.index(flag) + 1] == value + # No shell metacharacter could ever be interpreted: every element is a + # separate argument. + assert all(isinstance(a, str) for a in argv) + + +def test_prepare_leaves_the_original_recording_untouched(monkeypatch, tmp_path): + fake = _FakeRun() + preprocessor = _preprocessor(monkeypatch, fake) + source = write_synthetic_wav(tmp_path / "original.wav", seconds=3.0) + before = source.read_bytes() + + preprocessor.prepare(source, tmp_path / "work") + + assert source.read_bytes() == before + + +def test_missing_ffmpeg_is_reported_as_a_terminal_error(monkeypatch, tmp_path): + preprocessor = _preprocessor(monkeypatch, _FakeRun(), available=False) + source = write_synthetic_wav(tmp_path / "original.wav") + + with pytest.raises(FfmpegNotAvailable) as exc: + preprocessor.prepare(source, tmp_path / "work") + assert exc.value.code == "FFMPEG_NOT_FOUND" + + +def test_ffmpeg_timeout_is_surfaced_as_a_timeout_error(monkeypatch, tmp_path): + fake = _FakeRun(raises=CommandTimeout("too slow")) + preprocessor = _preprocessor(monkeypatch, fake) + source = write_synthetic_wav(tmp_path / "original.wav") + + with pytest.raises(AudioPreprocessingTimeout) as exc: + preprocessor.prepare(source, tmp_path / "work") + assert exc.value.code == "FFMPEG_TIMEOUT" + + +def test_ffmpeg_failure_never_leaks_its_stderr_to_the_caller(monkeypatch, tmp_path): + """The exception message is what may reach the API. It must be clean.""" + fake = _FakeRun( + returncode=1, + stderr="/Users/someone/storage/recordings/clip.webm: Invalid data found", + produce=False, + ) + preprocessor = _preprocessor(monkeypatch, fake) + source = write_synthetic_wav(tmp_path / "original.wav") + + with pytest.raises(AudioPreprocessingError) as exc: + preprocessor.prepare(source, tmp_path / "work") + message = str(exc.value) + assert "/Users" not in message + assert "someone" not in message + assert "could not be decoded" in message + + +def test_a_missing_source_recording_is_a_terminal_error(monkeypatch, tmp_path): + preprocessor = _preprocessor(monkeypatch, _FakeRun()) + with pytest.raises(AudioPreprocessingError) as exc: + preprocessor.prepare(tmp_path / "nope.webm", tmp_path / "work") + assert exc.value.code == "RECORDING_FILE_MISSING" + + +def test_decoding_that_produces_nothing_is_an_error(monkeypatch, tmp_path): + fake = _FakeRun(produce=False) + preprocessor = _preprocessor(monkeypatch, fake) + source = write_synthetic_wav(tmp_path / "original.wav") + + with pytest.raises(AudioPreprocessingError, match="no audio"): + preprocessor.prepare(source, tmp_path / "work") + + +def test_wrong_output_format_is_rejected_rather_than_handed_to_the_model( + monkeypatch, tmp_path +): + """If ffmpeg ignored our flags, the model would misread the audio.""" + + class WrongFormatRun(_FakeRun): + def __call__(self, argv, *, timeout_seconds, cwd=None): + self.calls.append(list(argv)) + # 44.1 kHz stereo: not what was asked for. + write_synthetic_wav( + Path(argv[-1]), seconds=1.0, sample_rate=44_100, channels=2 + ) + return CommandResult( + argv=tuple(argv), returncode=0, stdout="", stderr="", duration_seconds=0.1 + ) + + preprocessor = _preprocessor(monkeypatch, WrongFormatRun()) + source = write_synthetic_wav(tmp_path / "original.wav") + + with pytest.raises(AudioPreprocessingError, match="16 kHz mono"): + preprocessor.prepare(source, tmp_path / "work") + + +def test_unreadable_decoded_output_is_an_error(monkeypatch, tmp_path): + class GarbageRun(_FakeRun): + def __call__(self, argv, *, timeout_seconds, cwd=None): + Path(argv[-1]).write_bytes(b"not a wav file at all") + return CommandResult( + argv=tuple(argv), returncode=0, stdout="", stderr="", duration_seconds=0.1 + ) + + preprocessor = _preprocessor(monkeypatch, GarbageRun()) + source = write_synthetic_wav(tmp_path / "original.wav") + + with pytest.raises(AudioPreprocessingError, match="could not be read back"): + preprocessor.prepare(source, tmp_path / "work") + + +def test_synthetic_wav_helper_writes_a_valid_readable_file(tmp_path): + """Guards the fixture itself: a broken generator would fake passes.""" + path = write_synthetic_wav(tmp_path / "x.wav", seconds=1.5) + with wave.open(str(path), "rb") as handle: + assert handle.getframerate() == 16_000 + assert handle.getnchannels() == 1 + assert handle.getnframes() == 24_000 diff --git a/backend/tests/test_migrations.py b/backend/tests/test_migrations.py new file mode 100644 index 0000000..744f451 --- /dev/null +++ b/backend/tests/test_migrations.py @@ -0,0 +1,210 @@ +"""Migration tests, including migrating a populated Phase 1 database. + +Testing migrations only against a fresh database is the classic way to ship +a migration that works perfectly in CI and destroys the one database that +matters. These tests build a Phase 1 schema, put Phase 1 shaped rows in it, +and only then apply 0002 -- asserting the existing rows survive intact. +""" + +from __future__ import annotations + +import sqlite3 + +import pytest + +from app.db import get_connection, run_migrations +from app.timeutil import utc_now_iso + + +def _apply(conn, migrations_dir, only_version: int | None = None) -> list[int]: + """Apply migrations, optionally stopping after a given version.""" + if only_version is None: + return run_migrations(conn, migrations_dir) + + conn.execute( + "CREATE TABLE IF NOT EXISTS schema_migrations (" + "version INTEGER PRIMARY KEY, applied_at TEXT NOT NULL)" + ) + conn.commit() + applied = [] + for path in sorted(migrations_dir.glob("*.sql")): + version = int(path.name.split("_", 1)[0]) + if version > only_version: + continue + conn.executescript(path.read_text()) + conn.execute( + "INSERT INTO schema_migrations (version, applied_at) VALUES (?, ?)", + (version, utc_now_iso()), + ) + conn.commit() + applied.append(version) + return applied + + +def _seed_phase1_rows(conn) -> None: + """Insert rows exactly as Phase 1 would have written them.""" + now = utc_now_iso() + conn.execute( + "INSERT INTO topics (id, track, category, title, difficulty, modes_json, " + "prep_seconds_suggested, speaking_seconds_suggested, created_at, updated_at) " + "VALUES ('topic-1','technical','networking','Explain DNS','beginner'," + "'[\"explain\"]',60,90,?,?)", + (now, now), + ) + conn.execute( + "INSERT INTO sessions (id, track, topic_id, topic_title, topic_category, mode, " + "prep_seconds_planned, speaking_seconds_planned, client_reported_duration_seconds, " + "status, created_at, updated_at) " + "VALUES ('sess-1','technical','topic-1','Explain DNS','networking','explain'," + "60,90,88,'COMPLETED',?,?)", + (now, now), + ) + conn.execute( + "INSERT INTO recordings (id, session_id, filename, mime_type, size_bytes, created_at) " + "VALUES ('rec-1','sess-1','abc.webm','audio/webm',1234,?)", + (now,), + ) + conn.execute( + "INSERT INTO transcripts (id, session_id, provider_name, provider_version, text, " + "is_mock, created_at) VALUES ('tr-1','sess-1','mock-transcription','0.1.0'," + "'[SIMULATED TRANSCRIPT] placeholder',1,?)", + (now,), + ) + conn.execute( + "INSERT INTO reflections (id, session_id, went_well, improve, self_rating, created_at) " + "VALUES ('ref-1','sess-1','Structure','Pace',4,?)", + (now,), + ) + conn.commit() + + +def test_fresh_database_applies_both_migrations(settings): + conn = get_connection(settings.db_path) + try: + applied = run_migrations(conn, settings.migrations_dir) + assert applied == [1, 2] + tables = { + r["name"] + for r in conn.execute( + "SELECT name FROM sqlite_master WHERE type = 'table'" + ) + } + assert "processing_jobs" in tables + assert "transcript_segments" in tables + finally: + conn.close() + + +def test_migration_is_idempotent(settings): + conn = get_connection(settings.db_path) + try: + assert run_migrations(conn, settings.migrations_dir) == [1, 2] + assert run_migrations(conn, settings.migrations_dir) == [] + finally: + conn.close() + + +def test_0002_applies_to_a_populated_phase_1_database(settings): + """The migration that actually matters: an existing database with data.""" + conn = get_connection(settings.db_path) + try: + assert _apply(conn, settings.migrations_dir, only_version=1) == [1] + _seed_phase1_rows(conn) + + # Phase 1 shape: no Phase 2A columns or tables yet. + session_columns = { + r["name"] for r in conn.execute("PRAGMA table_info(sessions)") + } + assert "audio_duration_seconds" not in session_columns + + applied = run_migrations(conn, settings.migrations_dir) + assert applied == [2] + + # Existing rows survive, unchanged. + session = conn.execute("SELECT * FROM sessions WHERE id = 'sess-1'").fetchone() + assert session["status"] == "COMPLETED" + assert session["client_reported_duration_seconds"] == 88 + assert session["audio_duration_seconds"] is None # new column, not backfilled + + transcript = conn.execute( + "SELECT * FROM transcripts WHERE id = 'tr-1'" + ).fetchone() + assert transcript["is_mock"] == 1 + assert transcript["provider_name"] == "mock-transcription" + assert transcript["detected_language"] is None + assert transcript["model_sha256"] is None + + assert conn.execute("SELECT COUNT(*) AS c FROM recordings").fetchone()["c"] == 1 + assert conn.execute("SELECT COUNT(*) AS c FROM reflections").fetchone()["c"] == 1 + finally: + conn.close() + + +def test_one_active_job_per_session_is_enforced_by_the_database(settings): + conn = get_connection(settings.db_path) + try: + run_migrations(conn, settings.migrations_dir) + _seed_phase1_rows(conn) + now = utc_now_iso() + + def insert(job_id: str, status: str) -> None: + conn.execute( + "INSERT INTO processing_jobs (id, session_id, job_type, status, stage, " + "max_attempts, created_at, updated_at) " + "VALUES (?, 'sess-1', 'TRANSCRIPTION', ?, 'QUEUED', 3, ?, ?)", + (job_id, status, now, now), + ) + conn.commit() + + insert("job-1", "PENDING") + with pytest.raises(sqlite3.IntegrityError): + insert("job-2", "RUNNING") + + # Terminal jobs are excluded from the index, so history can accumulate. + conn.execute("UPDATE processing_jobs SET status = 'SUCCEEDED' WHERE id = 'job-1'") + conn.commit() + insert("job-3", "PENDING") + assert ( + conn.execute("SELECT COUNT(*) AS c FROM processing_jobs").fetchone()["c"] == 2 + ) + finally: + conn.close() + + +def test_transcript_segments_enforce_ordering_and_cascade(settings): + conn = get_connection(settings.db_path) + try: + run_migrations(conn, settings.migrations_dir) + _seed_phase1_rows(conn) + + conn.execute( + "INSERT INTO transcript_segments (id, transcript_id, segment_index, start_ms, end_ms, text) " + "VALUES ('seg-1','tr-1',0,0,1500,'Hello')" + ) + conn.commit() + + # end_ms must not precede start_ms. + with pytest.raises(sqlite3.IntegrityError): + conn.execute( + "INSERT INTO transcript_segments (id, transcript_id, segment_index, start_ms, end_ms, text) " + "VALUES ('seg-bad','tr-1',1,2000,1000,'Backwards')" + ) + conn.rollback() + + # A segment index cannot repeat within one transcript. + with pytest.raises(sqlite3.IntegrityError): + conn.execute( + "INSERT INTO transcript_segments (id, transcript_id, segment_index, start_ms, end_ms, text) " + "VALUES ('seg-dup','tr-1',0,0,500,'Duplicate index')" + ) + conn.rollback() + + # Deleting the session cascades through the transcript to its segments. + conn.execute("DELETE FROM sessions WHERE id = 'sess-1'") + conn.commit() + assert ( + conn.execute("SELECT COUNT(*) AS c FROM transcript_segments").fetchone()["c"] + == 0 + ) + finally: + conn.close() diff --git a/backend/tests/test_whisper_integration.py b/backend/tests/test_whisper_integration.py new file mode 100644 index 0000000..ad30362 --- /dev/null +++ b/backend/tests/test_whisper_integration.py @@ -0,0 +1,162 @@ +"""Opt-in integration test against a REAL whisper.cpp and a REAL ffmpeg. + +This is the only test in the suite that runs external binaries. It is +skipped unless every prerequisite genuinely exists, and each skip says +exactly what was missing -- a test that quietly passes when it did not run +is worse than no test, because it manufactures confidence. + +To run it: + + scripts/setup_whisper.sh # build runtime + model + # place a recording you made yourself at: + # backend/storage/benchmarks/sample.wav (or .m4a/.webm/.mp3) + SPEAKLAB_RUN_WHISPER_INTEGRATION=1 pytest tests/test_whisper_integration.py -v + +The sample audio is supplied by the owner and stays in a gitignored +directory. No recording is committed, and this test creates none. + +It asserts the *pipeline*, not transcription accuracy: what a given model +outputs for a given clip is a benchmark question (see +scripts/benchmark_models.py), not a pass/fail assertion. +""" + +from __future__ import annotations + +import os +import shutil +from pathlib import Path + +import pytest + +from app.audio.preprocessor import FfmpegAudioPreprocessor, job_workspace +from app.config import default_settings +from app.hashing import sha256_of_file +from app.providers.transcription_provider import ( + ModelConfig, + RuntimeConfig, + TranscriptionRequest, +) +from app.providers.whisper_cpp_provider import ( + WhisperCppTranscriptionProvider, + load_runtime_manifest, +) + +ENV_FLAG = "SPEAKLAB_RUN_WHISPER_INTEGRATION" +SAMPLE_STEMS = ("sample.wav", "sample.m4a", "sample.webm", "sample.mp3", "sample.ogg") + + +def _find_sample(settings) -> Path | None: + benchmarks = settings.storage_dir / "benchmarks" + for name in SAMPLE_STEMS: + candidate = benchmarks / name + if candidate.is_file(): + return candidate + return None + + +def _skip_reason() -> str | None: + """Return why this test cannot run, or None if it can.""" + if os.environ.get(ENV_FLAG) != "1": + return f"{ENV_FLAG}=1 not set (this test is opt-in)" + + settings = default_settings() + if shutil.which(settings.ffmpeg_binary) is None: + return "ffmpeg is not installed (brew install ffmpeg)" + if not settings.whisper_binary_path.is_file(): + return "whisper-cli is not built (scripts/setup_whisper.sh)" + model_path = settings.model_path(settings.whisper_model_name) + if not model_path.is_file(): + return ( + f"model {settings.whisper_model_name} is not downloaded " + "(scripts/setup_whisper.sh)" + ) + if _find_sample(settings) is None: + return ( + "no sample audio found -- place your own recording at " + "backend/storage/benchmarks/sample.wav (or .m4a/.webm/.mp3/.ogg)" + ) + return None + + +pytestmark = pytest.mark.skipif( + _skip_reason() is not None, + reason=_skip_reason() or "", +) + + +def test_real_pipeline_decodes_and_transcribes_a_local_recording(tmp_path): + settings = default_settings() + sample = _find_sample(settings) + assert sample is not None + + preprocessor = FfmpegAudioPreprocessor( + settings.ffmpeg_binary, settings.ffmpeg_timeout_seconds + ) + manifest = load_runtime_manifest(settings.whisper_manifest_path) + provider = WhisperCppTranscriptionProvider(manifest) + model_path = settings.model_path(settings.whisper_model_name) + + with job_workspace(tmp_path / "processing", "integration") as work_dir: + prepared = preprocessor.prepare(sample, work_dir) + + # The decode contract whisper.cpp depends on. + assert prepared.sample_rate == 16_000 + assert prepared.channels == 1 + assert prepared.sample_width_bytes == 2 + assert prepared.duration_seconds > 0 + + result = provider.transcribe( + TranscriptionRequest( + session_id="integration", + audio_path=prepared.path, + audio_duration_seconds=prepared.duration_seconds, + model=ModelConfig( + name=settings.whisper_model_name, + path=model_path, + sha256=sha256_of_file(model_path), + ), + runtime=RuntimeConfig( + binary_path=settings.whisper_binary_path, + timeout_seconds=settings.whisper_timeout_seconds, + version=manifest.version_string, + ), + language=settings.whisper_language, + ) + ) + + # Structural assertions only -- never "the transcript should say X". + assert result.is_mock is False + assert result.model_name == settings.whisper_model_name + assert result.model_sha256 + assert result.runtime_version + assert result.processing_duration_ms is not None and result.processing_duration_ms > 0 + assert result.real_time_factor is not None and result.real_time_factor > 0 + assert result.audio_duration_ms == pytest.approx( + int(prepared.duration_seconds * 1000), abs=50 + ) + + # Segments, if any, must be ordered and non-overlapping-ish. + indices = [s.index for s in result.segments] + assert indices == sorted(indices) + for segment in result.segments: + assert segment.end_ms >= segment.start_ms + + # The original recording is untouched by the pipeline. + assert sample.is_file() + + +def test_the_original_recording_is_never_modified(): + settings = default_settings() + sample = _find_sample(settings) + assert sample is not None + before = sample.stat().st_mtime_ns + size_before = sample.stat().st_size + + preprocessor = FfmpegAudioPreprocessor( + settings.ffmpeg_binary, settings.ffmpeg_timeout_seconds + ) + with job_workspace(settings.processing_dir, "integration-readonly") as work_dir: + preprocessor.prepare(sample, work_dir) + + assert sample.stat().st_mtime_ns == before + assert sample.stat().st_size == size_before diff --git a/backend/tests/test_whisper_provider.py b/backend/tests/test_whisper_provider.py new file mode 100644 index 0000000..88ce9dc --- /dev/null +++ b/backend/tests/test_whisper_provider.py @@ -0,0 +1,326 @@ +"""WhisperCppTranscriptionProvider: argv construction and output parsing. + +The whisper-cli binary is mocked. These tests are about the contract this +code depends on -- a JSON file with `result.language` and +`transcription[].offsets` -- and about refusing to guess when that contract +is not met. +""" + +from __future__ import annotations + +import json +from pathlib import Path + +import pytest + +from app.providers.transcription_provider import ( + ModelConfig, + RuntimeConfig, + TranscriptionError, + TranscriptionRequest, +) +from app.providers.whisper_cpp_provider import ( + RuntimeManifest, + WhisperCppTranscriptionProvider, + load_runtime_manifest, +) +from app.subprocess_util import CommandResult, CommandTimeout +from tests.conftest import whisper_json_payload, write_synthetic_wav + + +MANIFEST = RuntimeManifest( + release_tag="v1.9.1", + commit="f049fff95a089aa9969deb009cdd4892b3e74916", + build_options="-DCMAKE_BUILD_TYPE=Release", + binary_version="whisper-cli synthetic", +) + + +def _setup(tmp_path: Path) -> tuple[Path, Path, Path]: + """Create a fake binary, a fake model, and a decoded audio file.""" + binary = tmp_path / "bin" / "whisper-cli" + binary.parent.mkdir(parents=True, exist_ok=True) + binary.write_text("#!/bin/sh\nexit 0\n") + binary.chmod(0o755) + + model = tmp_path / "models" / "ggml-base.en.bin" + model.parent.mkdir(parents=True, exist_ok=True) + model.write_bytes(b"synthetic-weights") + + work = tmp_path / "work" + audio = write_synthetic_wav(work / "audio-16k-mono.wav", seconds=3.0) + return binary, model, audio + + +def _request(binary, model, audio, *, language="en", duration=3.0, threads=None): + return TranscriptionRequest( + session_id="sess-1", + audio_path=audio, + audio_duration_seconds=duration, + model=ModelConfig(name="base.en", path=model, sha256="a" * 64), + runtime=RuntimeConfig(binary_path=binary, timeout_seconds=60, threads=threads), + language=language, + ) + + +def _fake_run(payload: dict | str | None, *, returncode: int = 0, raises=None): + """Return a run_command stand-in that writes `payload` as whisper would.""" + calls: list[list[str]] = [] + + def runner(argv, *, timeout_seconds, cwd=None): + calls.append(list(argv)) + if raises: + raise raises + if payload is not None: + stem = Path(argv[argv.index("--output-file") + 1]) + target = stem.with_suffix(".json") + target.parent.mkdir(parents=True, exist_ok=True) + if isinstance(payload, str): + target.write_text(payload) + else: + target.write_text(json.dumps(payload)) + return CommandResult( + argv=tuple(argv), returncode=returncode, stdout="", stderr="", duration_seconds=0.2 + ) + + runner.calls = calls # type: ignore[attr-defined] + return runner + + +def _provider(monkeypatch, runner): + monkeypatch.setattr("app.providers.whisper_cpp_provider.run_command", runner) + return WhisperCppTranscriptionProvider(MANIFEST) + + +# --- Preconditions ----------------------------------------------------------- + + +def test_missing_binary_fails_terminally(monkeypatch, tmp_path): + _, model, audio = _setup(tmp_path) + provider = _provider(monkeypatch, _fake_run(whisper_json_payload())) + request = _request(tmp_path / "nope" / "whisper-cli", model, audio) + + with pytest.raises(TranscriptionError) as exc: + provider.transcribe(request) + assert exc.value.code == "WHISPER_BINARY_MISSING" + assert exc.value.retryable is False + + +def test_missing_model_fails_terminally_and_is_never_retried(monkeypatch, tmp_path): + """Retrying a missing model file forever would bury the fix.""" + binary, _, audio = _setup(tmp_path) + provider = _provider(monkeypatch, _fake_run(whisper_json_payload())) + request = _request(binary, tmp_path / "models" / "absent.bin", audio) + + with pytest.raises(TranscriptionError) as exc: + provider.transcribe(request) + assert exc.value.code == "MODEL_MISSING" + assert exc.value.retryable is False + + +# --- Command construction ---------------------------------------------------- + + +def test_argv_requests_structured_json_and_passes_model_audio_and_language( + monkeypatch, tmp_path +): + binary, model, audio = _setup(tmp_path) + runner = _fake_run(whisper_json_payload()) + provider = _provider(monkeypatch, runner) + + provider.transcribe(_request(binary, model, audio, threads=4)) + + argv = runner.calls[0] + assert argv[0] == str(binary) + assert argv[argv.index("--model") + 1] == str(model) + assert argv[argv.index("--file") + 1] == str(audio) + assert argv[argv.index("--language") + 1] == "en" + assert argv[argv.index("--threads") + 1] == "4" + # Structured output, never console scraping. + assert "--output-json" in argv + + +def test_no_language_requests_automatic_detection(monkeypatch, tmp_path): + binary, model, audio = _setup(tmp_path) + runner = _fake_run(whisper_json_payload()) + provider = _provider(monkeypatch, runner) + + provider.transcribe(_request(binary, model, audio, language=None)) + + argv = runner.calls[0] + assert argv[argv.index("--language") + 1] == "auto" + + +def test_core_ml_quantization_and_vad_are_off_in_this_phase(monkeypatch, tmp_path): + binary, model, audio = _setup(tmp_path) + runner = _fake_run(whisper_json_payload()) + provider = _provider(monkeypatch, runner) + + result = provider.transcribe(_request(binary, model, audio)) + + assert result.parameters["core_ml"] is False + assert result.parameters["quantized"] is False + assert result.parameters["vad"] is False + + +# --- Successful parsing ------------------------------------------------------ + + +def test_structured_output_becomes_ordered_segments_and_full_text(monkeypatch, tmp_path): + binary, model, audio = _setup(tmp_path) + payload = whisper_json_payload( + segments=[ + (0, 1200, " Hello there."), + (1200, 2600, " This is a test."), + (2600, 3000, " Goodbye."), + ] + ) + provider = _provider(monkeypatch, _fake_run(payload)) + + result = provider.transcribe(_request(binary, model, audio)) + + assert result.is_mock is False + assert result.text == "Hello there. This is a test. Goodbye." + assert [s.index for s in result.segments] == [0, 1, 2] + assert result.segments[1].start_ms == 1200 + assert result.segments[1].end_ms == 2600 + assert result.segments[0].text == "Hello there." + + +def test_result_carries_full_model_and_runtime_provenance(monkeypatch, tmp_path): + binary, model, audio = _setup(tmp_path) + provider = _provider(monkeypatch, _fake_run(whisper_json_payload())) + + result = provider.transcribe(_request(binary, model, audio)) + + assert result.detected_language == "en" + assert result.model_name == "base.en" + assert result.model_sha256 == "a" * 64 + assert result.runtime_version == "whisper.cpp v1.9.1 (f049fff95a08)" + assert result.provider_name == "whisper-cpp" + + +def test_real_time_factor_is_processing_time_over_audio_duration(monkeypatch, tmp_path): + binary, model, audio = _setup(tmp_path) + provider = _provider(monkeypatch, _fake_run(whisper_json_payload())) + + result = provider.transcribe(_request(binary, model, audio, duration=10.0)) + + assert result.audio_duration_ms == 10_000 + assert result.processing_duration_ms is not None + expected = (result.processing_duration_ms / 1000) / 10.0 + assert result.real_time_factor == pytest.approx(expected, abs=1e-3) + + +def test_no_speech_detected_is_an_empty_transcript_not_a_failure(monkeypatch, tmp_path): + """An empty transcription list is a legitimate result, not an error.""" + binary, model, audio = _setup(tmp_path) + provider = _provider(monkeypatch, _fake_run(whisper_json_payload(segments=[]))) + + result = provider.transcribe(_request(binary, model, audio)) + + assert result.text == "" + assert result.segments == () + assert result.is_mock is False + + +def test_inverted_segment_timings_are_clamped_to_satisfy_the_db_constraint( + monkeypatch, tmp_path +): + binary, model, audio = _setup(tmp_path) + payload = whisper_json_payload(segments=[(2000, 1500, " Rounding artifact.")]) + provider = _provider(monkeypatch, _fake_run(payload)) + + result = provider.transcribe(_request(binary, model, audio)) + + assert result.segments[0].start_ms == 2000 + assert result.segments[0].end_ms == 2000 + + +# --- Failure handling -------------------------------------------------------- + + +def test_nonzero_exit_is_a_retryable_provider_failure(monkeypatch, tmp_path): + binary, model, audio = _setup(tmp_path) + provider = _provider(monkeypatch, _fake_run(None, returncode=1)) + + with pytest.raises(TranscriptionError) as exc: + provider.transcribe(_request(binary, model, audio)) + assert exc.value.code == "WHISPER_FAILED" + assert exc.value.retryable is True + + +def test_timeout_is_a_retryable_provider_failure(monkeypatch, tmp_path): + binary, model, audio = _setup(tmp_path) + provider = _provider(monkeypatch, _fake_run(None, raises=CommandTimeout("slow"))) + + with pytest.raises(TranscriptionError) as exc: + provider.transcribe(_request(binary, model, audio)) + assert exc.value.code == "WHISPER_TIMEOUT" + assert exc.value.retryable is True + + +def test_missing_output_file_is_a_failure_not_an_empty_transcript(monkeypatch, tmp_path): + """'We failed to read the output' must never look like 'it heard nothing'.""" + binary, model, audio = _setup(tmp_path) + provider = _provider(monkeypatch, _fake_run(None)) + + with pytest.raises(TranscriptionError) as exc: + provider.transcribe(_request(binary, model, audio)) + assert exc.value.code == "WHISPER_OUTPUT_MISSING" + + +def test_malformed_json_is_a_provider_failure(monkeypatch, tmp_path): + binary, model, audio = _setup(tmp_path) + provider = _provider(monkeypatch, _fake_run("{not valid json at all")) + + with pytest.raises(TranscriptionError) as exc: + provider.transcribe(_request(binary, model, audio)) + assert exc.value.code == "WHISPER_OUTPUT_MALFORMED" + + +def test_json_missing_the_transcription_key_is_a_provider_failure(monkeypatch, tmp_path): + binary, model, audio = _setup(tmp_path) + provider = _provider(monkeypatch, _fake_run({"result": {"language": "en"}})) + + with pytest.raises(TranscriptionError) as exc: + provider.transcribe(_request(binary, model, audio)) + assert exc.value.code == "WHISPER_OUTPUT_MALFORMED" + + +def test_segments_missing_offsets_are_a_provider_failure(monkeypatch, tmp_path): + binary, model, audio = _setup(tmp_path) + payload = {"result": {"language": "en"}, "transcription": [{"text": " No timings"}]} + provider = _provider(monkeypatch, _fake_run(payload)) + + with pytest.raises(TranscriptionError) as exc: + provider.transcribe(_request(binary, model, audio)) + assert exc.value.code == "WHISPER_OUTPUT_MALFORMED" + + +def test_segments_with_non_numeric_offsets_are_a_provider_failure(monkeypatch, tmp_path): + binary, model, audio = _setup(tmp_path) + payload = { + "result": {"language": "en"}, + "transcription": [{"offsets": {"from": "start", "to": "end"}, "text": " x"}], + } + provider = _provider(monkeypatch, _fake_run(payload)) + + with pytest.raises(TranscriptionError) as exc: + provider.transcribe(_request(binary, model, audio)) + assert exc.value.code == "WHISPER_OUTPUT_MALFORMED" + + +# --- Manifest ---------------------------------------------------------------- + + +def test_missing_manifest_is_tolerated_rather_than_invented(tmp_path): + manifest = load_runtime_manifest(tmp_path / "absent.json") + assert manifest.release_tag is None + assert manifest.version_string is None + + +def test_manifest_version_string_combines_tag_and_short_commit(tmp_path): + path = tmp_path / "manifest.json" + path.write_text(json.dumps({"release_tag": "v1.9.1", "commit": "abcdef1234567890"})) + assert load_runtime_manifest(path).version_string == "whisper.cpp v1.9.1 (abcdef123456)" diff --git a/backend/tests/test_worker.py b/backend/tests/test_worker.py new file mode 100644 index 0000000..4bdf773 --- /dev/null +++ b/backend/tests/test_worker.py @@ -0,0 +1,577 @@ +"""Worker and job-queue behavior: claiming, retries, recovery, restarts. + +Every external process is mocked. What is exercised for real here is SQLite +concurrency, the transaction boundary around claiming, and the state machine +the worker drives. +""" + +from __future__ import annotations + +import signal +from datetime import datetime, timedelta, timezone +from pathlib import Path + +import pytest + +from app.audio.preprocessor import ( + AudioPreprocessingError, + AudioPreprocessingTimeout, + PreparedAudio, +) +from app.db import get_connection +from app.providers.transcription_provider import ( + TranscriptionError, + TranscriptResult, + TranscriptSegment, +) +from app.repositories.job_repository import SqliteJobRepository +from app.repositories.recording_storage import LocalRecordingStorage +from app.repositories.session_repository import SqliteSessionRepository +from app.repositories.topic_repository import SqliteTopicRepository +from app.schemas import JobStatus, ProcessingStage, SessionCreateRequest, SessionStatus, SpeakingMode, Track +from app.services.transcription_job_service import TranscriptionJobService +from app.worker.runner import TranscriptionWorker +from tests.conftest import write_synthetic_wav + + +# --- Test doubles ------------------------------------------------------------ + + +class StubPreprocessor: + def __init__(self, *, duration=12.5, raises=None): + self.duration = duration + self.raises = raises + self.calls = 0 + self.work_dirs: list[Path] = [] + + def prepare(self, source: Path, work_dir: Path) -> PreparedAudio: + self.calls += 1 + self.work_dirs.append(work_dir) + if self.raises: + raise self.raises + path = write_synthetic_wav(work_dir / "audio-16k-mono.wav", seconds=1.0) + return PreparedAudio( + path=path, + duration_seconds=self.duration, + sample_rate=16_000, + channels=1, + sample_width_bytes=2, + ) + + def is_available(self) -> bool: + return self.raises is None + + def version(self) -> str | None: + return "ffmpeg synthetic" + + +class StubTranscription: + def __init__(self, *, raises=None, text="Hello there."): + self.raises = raises + self.text = text + self.calls = 0 + self.last_request = None + + def transcribe(self, request): + self.calls += 1 + self.last_request = request + if self.raises: + raise self.raises + return TranscriptResult( + provider_name="whisper-cpp", + provider_version="0.1.0", + text=self.text, + is_mock=False, + detected_language="en", + segments=( + TranscriptSegment(index=0, start_ms=0, end_ms=1200, text="Hello"), + TranscriptSegment(index=1, start_ms=1200, end_ms=2400, text="there."), + ), + model_name="base.en", + model_sha256="b" * 64, + runtime_version="whisper.cpp v1.9.1 (f049fff95a08)", + parameters={"language": "en", "vad": False}, + audio_duration_ms=12_500, + processing_duration_ms=2_000, + real_time_factor=0.16, + ) + + +# --- Fixtures ---------------------------------------------------------------- + + +@pytest.fixture +def queued_session(settings, db_conn): + """A session in PROCESSING with a stored recording and a PENDING job.""" + topics = SqliteTopicRepository(db_conn) + sessions = SqliteSessionRepository(db_conn) + jobs = SqliteJobRepository(db_conn) + topic = topics.get_next("technical", mode="explain", seed=1) + session = sessions.create_session( + SessionCreateRequest( + track=Track.technical, + topic_id=topic.id, + mode=SpeakingMode.explain, + prep_seconds_planned=60, + speaking_seconds_planned=90, + ), + topic.title, + topic.category, + ) + + # A stored recording file, synthetic. + settings.recordings_dir.mkdir(parents=True, exist_ok=True) + write_synthetic_wav(settings.recordings_dir / "clip.wav", seconds=2.0) + from app.repositories.recording_storage import SavedRecording + + sessions.attach_recording( + session.id, SavedRecording(filename="clip.wav", mime_type="audio/wav", size_bytes=100), 12 + ) + sessions.set_status(session.id, "CREATED", "RECORDING_STORED") + sessions.set_status(session.id, "RECORDING_STORED", "PROCESSING") + job = jobs.create(session.id, max_attempts=settings.worker_max_attempts) + return session, job + + +def _service(settings, conn, preprocessor=None, transcription=None): + from app.providers.feedback_provider import MockFeedbackProvider + from app.providers.metrics_provider import MockMetricsProvider + + return TranscriptionJobService( + settings=settings, + sessions=SqliteSessionRepository(conn), + jobs=SqliteJobRepository(conn), + recordings=LocalRecordingStorage( + settings.recordings_dir, settings.allowed_mime_types, settings.max_upload_bytes + ), + preprocessor=preprocessor or StubPreprocessor(), + transcription=transcription or StubTranscription(), + metrics=MockMetricsProvider(), + feedback=MockFeedbackProvider(), + model_sha256="b" * 64, + runtime_version="whisper.cpp v1.9.1", + ) + + +# --- Atomic claiming --------------------------------------------------------- + + +def test_claiming_marks_the_job_running_and_increments_the_attempt( + settings, db_conn, autocommit_conn, queued_session +): + _, job = queued_session + claimed = SqliteJobRepository(autocommit_conn).claim_next("worker-a") + + assert claimed is not None + assert claimed.id == job.id + assert claimed.status == JobStatus.RUNNING.value + assert claimed.stage == ProcessingStage.CLAIMED.value + assert claimed.worker_id == "worker-a" + assert claimed.attempt_count == 1 + assert claimed.started_at is not None + + +def test_two_workers_cannot_claim_the_same_job( + settings, db_conn, autocommit_conn, queued_session +): + """The core queue guarantee: exactly one consumer per job.""" + second = get_connection(settings.db_path, autocommit=True) + try: + a = SqliteJobRepository(autocommit_conn).claim_next("worker-a") + b = SqliteJobRepository(second).claim_next("worker-b") + assert a is not None + assert b is None + finally: + second.close() + + +def test_claiming_returns_none_when_the_queue_is_empty(settings, db_conn, autocommit_conn): + assert SqliteJobRepository(autocommit_conn).claim_next("worker-a") is None + + +def test_jobs_are_claimed_oldest_first(settings, db_conn, autocommit_conn, queued_session): + session, first_job = queued_session + jobs = SqliteJobRepository(db_conn) + # Retire the first job so a second can exist for the same session. + jobs.mark_succeeded(first_job.id) + second_job = jobs.create(session.id) + + claimed = SqliteJobRepository(autocommit_conn).claim_next("worker-a") + assert claimed.id == second_job.id + + +# --- Success path ------------------------------------------------------------ + + +def test_successful_job_writes_a_real_transcript_with_segments_and_completes( + settings, db_conn, autocommit_conn, queued_session +): + session, _ = queued_session + claimed = SqliteJobRepository(autocommit_conn).claim_next("worker-a") + outcome = _service(settings, autocommit_conn).process(claimed) + + assert outcome.succeeded is True + + sessions = SqliteSessionRepository(db_conn) + refreshed = sessions.get_session(session.id) + assert refreshed.status == SessionStatus.COMPLETED + # Authoritative duration replaced nothing: both values coexist. + assert refreshed.audio_duration_seconds == pytest.approx(12.5) + assert refreshed.client_reported_duration_seconds == 12 + + transcript = sessions.get_transcript(session.id) + assert transcript.is_mock is False + assert transcript.text == "Hello there." + assert transcript.detected_language == "en" + assert transcript.model_name == "base.en" + assert transcript.model_sha256 == "b" * 64 + assert transcript.runtime_version.startswith("whisper.cpp v1.9.1") + assert transcript.real_time_factor == pytest.approx(0.16) + assert transcript.parameters == {"language": "en", "vad": False} + + assert [s.segment_index for s in transcript.segments] == [0, 1] + assert transcript.segments[0].text == "Hello" + assert transcript.segments[1].start_ms == 1200 + + # Metrics and feedback stay simulated in this phase. + assert sessions.get_metrics(session.id).is_mock is True + assert sessions.get_feedback(session.id).is_mock is True + + job = SqliteJobRepository(db_conn).get_latest_for_session(session.id) + assert job.status == JobStatus.SUCCEEDED.value + assert job.stage == ProcessingStage.COMPLETED.value + assert job.completed_at is not None + + +def test_the_processed_audio_path_is_passed_to_the_provider_not_the_original( + settings, db_conn, autocommit_conn, queued_session +): + transcription = StubTranscription() + claimed = SqliteJobRepository(autocommit_conn).claim_next("worker-a") + _service(settings, autocommit_conn, transcription=transcription).process(claimed) + + request = transcription.last_request + assert request.audio_path.name == "audio-16k-mono.wav" + assert "processing" in str(request.audio_path) + assert request.audio_duration_seconds == pytest.approx(12.5) + + +def test_temporary_processing_files_are_removed_after_success( + settings, db_conn, autocommit_conn, queued_session +): + preprocessor = StubPreprocessor() + claimed = SqliteJobRepository(autocommit_conn).claim_next("worker-a") + _service(settings, autocommit_conn, preprocessor=preprocessor).process(claimed) + + assert preprocessor.work_dirs + for work_dir in preprocessor.work_dirs: + assert not work_dir.exists() + + +# --- Failure paths ----------------------------------------------------------- + + +def test_retryable_failure_requeues_the_job_and_leaves_the_session_processing( + settings, db_conn, autocommit_conn, queued_session +): + session, _ = queued_session + transcription = StubTranscription( + raises=TranscriptionError("timed out", code="WHISPER_TIMEOUT", retryable=True) + ) + claimed = SqliteJobRepository(autocommit_conn).claim_next("worker-a") + outcome = _service(settings, autocommit_conn, transcription=transcription).process(claimed) + + assert outcome.retried is True + job = SqliteJobRepository(db_conn).get_latest_for_session(session.id) + assert job.status == JobStatus.PENDING.value + assert job.worker_id is None + assert job.attempt_count == 1 # budget consumed + assert job.error_code == "WHISPER_TIMEOUT" + # The user is still waiting, not told it failed. + assert SqliteSessionRepository(db_conn).get_session(session.id).status == SessionStatus.PROCESSING + + +def test_terminal_failure_fails_the_session_and_preserves_the_recording( + settings, db_conn, autocommit_conn, queued_session +): + session, _ = queued_session + transcription = StubTranscription( + raises=TranscriptionError("no model", code="MODEL_MISSING", retryable=False) + ) + claimed = SqliteJobRepository(autocommit_conn).claim_next("worker-a") + outcome = _service(settings, autocommit_conn, transcription=transcription).process(claimed) + + assert outcome.succeeded is False + assert outcome.retried is False + + job = SqliteJobRepository(db_conn).get_latest_for_session(session.id) + assert job.status == JobStatus.FAILED.value + assert job.error_code == "MODEL_MISSING" + + refreshed = SqliteSessionRepository(db_conn).get_session(session.id) + assert refreshed.status == SessionStatus.FAILED + assert "preserved" in refreshed.failure_reason.lower() + # The irreplaceable artifact survives. + assert (settings.recordings_dir / "clip.wav").exists() + + +def test_a_missing_model_is_never_retried_even_with_attempts_remaining( + settings, db_conn, autocommit_conn, queued_session +): + session, _ = queued_session + transcription = StubTranscription( + raises=TranscriptionError("no model", code="MODEL_MISSING", retryable=False) + ) + claimed = SqliteJobRepository(autocommit_conn).claim_next("worker-a") + assert claimed.attempt_count < claimed.max_attempts # budget available + + _service(settings, autocommit_conn, transcription=transcription).process(claimed) + + assert SqliteJobRepository(db_conn).get_latest_for_session(session.id).status == JobStatus.FAILED.value + + +def test_retry_budget_is_finite(settings, db_conn, autocommit_conn, queued_session): + """A retryable failure eventually becomes terminal instead of looping.""" + session, _ = queued_session + transcription = StubTranscription( + raises=TranscriptionError("timed out", code="WHISPER_TIMEOUT", retryable=True) + ) + jobs = SqliteJobRepository(autocommit_conn) + + for _ in range(settings.worker_max_attempts): + claimed = jobs.claim_next("worker-a") + assert claimed is not None + _service(settings, autocommit_conn, transcription=transcription).process(claimed) + + assert jobs.claim_next("worker-a") is None # nothing left to claim + job = SqliteJobRepository(db_conn).get_latest_for_session(session.id) + assert job.status == JobStatus.FAILED.value + assert SqliteSessionRepository(db_conn).get_session(session.id).status == SessionStatus.FAILED + + +def test_ffmpeg_timeout_is_retryable_but_a_decode_failure_is_not( + settings, db_conn, autocommit_conn, queued_session +): + session, _ = queued_session + jobs = SqliteJobRepository(autocommit_conn) + + claimed = jobs.claim_next("worker-a") + _service( + settings, + autocommit_conn, + preprocessor=StubPreprocessor(raises=AudioPreprocessingTimeout("slow")), + ).process(claimed) + assert SqliteJobRepository(db_conn).get_latest_for_session(session.id).status == JobStatus.PENDING.value + + claimed = jobs.claim_next("worker-a") + _service( + settings, + autocommit_conn, + preprocessor=StubPreprocessor(raises=AudioPreprocessingError("corrupt")), + ).process(claimed) + assert SqliteJobRepository(db_conn).get_latest_for_session(session.id).status == JobStatus.FAILED.value + + +def test_temporary_processing_files_are_removed_after_failure( + settings, db_conn, autocommit_conn, queued_session +): + """A crashed job must not leave decoded speech on disk.""" + transcription = StubTranscription( + raises=TranscriptionError("boom", code="WHISPER_FAILED", retryable=False) + ) + preprocessor = StubPreprocessor() + claimed = SqliteJobRepository(autocommit_conn).claim_next("worker-a") + + _service( + settings, autocommit_conn, preprocessor=preprocessor, transcription=transcription + ).process(claimed) + + for work_dir in preprocessor.work_dirs: + assert not work_dir.exists() + assert list(settings.processing_dir.iterdir()) == [] + + +def test_a_deleted_session_fails_its_job_quietly( + settings, db_conn, autocommit_conn, queued_session +): + session, _ = queued_session + claimed = SqliteJobRepository(autocommit_conn).claim_next("worker-a") + SqliteSessionRepository(db_conn).delete_session(session.id) + + outcome = _service(settings, autocommit_conn).process(claimed) + assert outcome.error_code == "SESSION_DELETED" + + +def test_an_unexpected_error_does_not_escape_and_kill_the_worker( + settings, db_conn, autocommit_conn, queued_session +): + """One bad recording must not stop every other session from processing.""" + + class Exploding: + def transcribe(self, request): + raise ValueError("something nobody anticipated") + + claimed = SqliteJobRepository(autocommit_conn).claim_next("worker-a") + outcome = _service(settings, autocommit_conn, transcription=Exploding()).process(claimed) + + assert outcome.succeeded is False + assert outcome.error_code == "UNEXPECTED_ERROR" + + +def test_a_retry_after_a_partial_write_does_not_hit_the_unique_constraint( + settings, db_conn, autocommit_conn, queued_session +): + """Simulates a crash between the transcript commit and COMPLETED.""" + session, _ = queued_session + sessions = SqliteSessionRepository(db_conn) + sessions.save_transcript( + session.id, + TranscriptResult( + provider_name="whisper-cpp", + provider_version="0.1.0", + text="partial", + is_mock=False, + segments=(TranscriptSegment(index=0, start_ms=0, end_ms=100, text="partial"),), + ), + ) + + claimed = SqliteJobRepository(autocommit_conn).claim_next("worker-a") + outcome = _service(settings, autocommit_conn).process(claimed) + + assert outcome.succeeded is True + assert sessions.get_transcript(session.id).text == "Hello there." + + +# --- Stale-job recovery and restarts ----------------------------------------- + + +def _age_job(conn, job_id: str, seconds: int) -> None: + old = ( + datetime.now(timezone.utc) - timedelta(seconds=seconds) + ).isoformat(timespec="seconds").replace("+00:00", "Z") + conn.execute( + "UPDATE processing_jobs SET started_at = ?, updated_at = ? WHERE id = ?", + (old, old, job_id), + ) + conn.commit() + + +def test_a_stale_running_job_is_requeued(settings, db_conn, autocommit_conn, queued_session): + """A SIGKILLed worker leaves a RUNNING row nobody will ever move.""" + session, _ = queued_session + jobs = SqliteJobRepository(autocommit_conn) + claimed = jobs.claim_next("worker-that-died") + _age_job(db_conn, claimed.id, settings.job_stale_after_seconds + 60) + + recovered = jobs.recover_stale(settings.job_stale_after_seconds) + + assert len(recovered) == 1 + assert recovered[0].status == JobStatus.PENDING.value + assert recovered[0].worker_id is None + assert recovered[0].error_code == "WORKER_LOST" + # And it can be claimed again. + assert jobs.claim_next("worker-b") is not None + + +def test_a_healthy_running_job_is_not_stolen(settings, db_conn, autocommit_conn, queued_session): + jobs = SqliteJobRepository(autocommit_conn) + claimed = jobs.claim_next("worker-a") + + assert jobs.recover_stale(settings.job_stale_after_seconds) == [] + assert jobs.get(claimed.id).status == JobStatus.RUNNING.value + + +def test_a_stale_job_out_of_attempts_fails_terminally( + settings, db_conn, autocommit_conn, queued_session +): + jobs = SqliteJobRepository(autocommit_conn) + claimed = jobs.claim_next("worker-a") + db_conn.execute( + "UPDATE processing_jobs SET attempt_count = max_attempts WHERE id = ?", (claimed.id,) + ) + db_conn.commit() + _age_job(db_conn, claimed.id, settings.job_stale_after_seconds + 60) + + recovered = jobs.recover_stale(settings.job_stale_after_seconds) + + assert recovered[0].status == JobStatus.FAILED.value + assert recovered[0].error_code == "WORKER_LOST" + + +def test_worker_recovers_stale_jobs_on_startup(settings, db_conn, queued_session): + """Restart behavior: a fresh worker reclaims what the dead one held.""" + conn = get_connection(settings.db_path, autocommit=True) + try: + claimed = SqliteJobRepository(conn).claim_next("worker-that-died") + _age_job(db_conn, claimed.id, settings.job_stale_after_seconds + 60) + finally: + conn.close() + + worker = TranscriptionWorker(settings, worker_id="worker-fresh") + try: + assert worker.recover_stale_jobs() == 1 + assert ( + SqliteJobRepository(db_conn).get_latest_for_session(queued_session[0].id).status + == JobStatus.PENDING.value + ) + finally: + worker.close() + + +# --- Worker loop ------------------------------------------------------------- + + +def test_run_once_returns_false_when_there_is_nothing_to_do(settings, db_conn): + worker = TranscriptionWorker(settings, worker_id="worker-idle") + try: + assert worker.run_once() is False + assert worker.stats.claimed == 0 + finally: + worker.close() + + +def test_run_once_processes_a_queued_job(settings, db_conn, queued_session, monkeypatch): + session, _ = queued_session + worker = TranscriptionWorker(settings, worker_id="worker-a") + monkeypatch.setattr( + worker, "build_service", lambda conn: _service(settings, conn) + ) + try: + assert worker.run_once() is True + assert worker.stats.succeeded == 1 + finally: + worker.close() + + assert SqliteSessionRepository(db_conn).get_session(session.id).status == SessionStatus.COMPLETED + + +def test_run_forever_stops_when_asked(settings, db_conn): + worker = TranscriptionWorker(settings, worker_id="worker-a") + worker.request_stop() + stats = worker.run_forever(max_iterations=5) + assert stats.claimed == 0 + + +def test_signal_handlers_request_a_graceful_stop(settings, db_conn): + """SIGTERM must finish the current job, not abandon it mid-transcript.""" + worker = TranscriptionWorker(settings, worker_id="worker-a") + original = signal.getsignal(signal.SIGTERM) + try: + worker.install_signal_handlers() + handler = signal.getsignal(signal.SIGTERM) + assert callable(handler) + handler(signal.SIGTERM, None) + assert worker._stop_requested is True + finally: + signal.signal(signal.SIGTERM, original) + worker.close() + + +def test_worker_never_falls_back_to_the_mock_provider(settings): + """A broken whisper setup must fail loudly, not silently simulate.""" + from app.providers.whisper_cpp_provider import WhisperCppTranscriptionProvider + from app.worker.runner import build_transcription_provider + + settings.transcription_provider = "whisper-cpp" + provider, _ = build_transcription_provider(settings) + assert isinstance(provider, WhisperCppTranscriptionProvider) From 238ea15b29faabc92ddeb0f83e9fb52953250536 Mon Sep 17 00:00:00 2001 From: Abdulrahman Date: Fri, 31 Jul 2026 21:56:18 +0300 Subject: [PATCH 8/9] Poll for processing state and label each result section independently The frontend now waits for transcription with ordinary HTTP polling: no WebSocket, no SSE. One user on one machine waiting on one job does not need a persistent connection, and a connection would bring reconnection logic and a proxying story the dev setup does not have. The poller takes its clock, timers and fetch as injected dependencies, so its whole lifecycle is unit-testable with no real time and no real network. It starts near one second, backs off to a cap, and stops on COMPLETED, FAILED, a 404 when the session was deleted, an overall timeout, or explicit cancellation from route teardown. Calling start twice cannot create a second loop, and a response arriving after cancel changes nothing -- navigating away mid-transcription must not touch a destroyed component or corrupt the session. Results presentation changed to match what is now true. Before, every section was simulated, so one page-level banner was honest; now the transcript is real while metrics and feedback are still mocks, so a global banner would be false. Each section reads is_mock from its own row: the transcript shows "Real transcript" with its model, runtime, language and speed, while metrics and feedback keep "Simulated" and say plainly that they are not derived from the transcript above. A Phase 1 session still shows "Simulated" on all three, because its rows still say so. A failed session explains which prerequisite is missing and gives the exact command that fixes it, and always states that the recording itself survived. A session that stays queued past the timeout says the worker is probably not running, rather than spinning forever. Reflection is reachable while transcription runs and shows the current stage without ever blocking the form -- reflecting while it is fresh is the point of that screen. --- frontend/src/lib/api/client.test.ts | 43 +++ frontend/src/lib/api/client.ts | 28 +- .../lib/components/ProcessingStatus.svelte | 135 ++++++++ .../src/lib/components/SessionResults.svelte | 176 +++++++++- frontend/src/lib/processing/poller.test.ts | 326 ++++++++++++++++++ frontend/src/lib/processing/poller.ts | 172 +++++++++ .../src/lib/processing/presentation.test.ts | 198 +++++++++++ frontend/src/lib/processing/presentation.ts | 191 ++++++++++ frontend/src/lib/types.ts | 75 ++++ frontend/src/routes/history/[id]/+page.svelte | 15 + .../src/routes/practice/reflect/+page.svelte | 48 ++- .../practice/results/[sessionId]/+page.svelte | 70 +++- .../src/routes/practice/review/+page.svelte | 3 + 13 files changed, 1456 insertions(+), 24 deletions(-) create mode 100644 frontend/src/lib/components/ProcessingStatus.svelte create mode 100644 frontend/src/lib/processing/poller.test.ts create mode 100644 frontend/src/lib/processing/poller.ts create mode 100644 frontend/src/lib/processing/presentation.test.ts create mode 100644 frontend/src/lib/processing/presentation.ts diff --git a/frontend/src/lib/api/client.test.ts b/frontend/src/lib/api/client.test.ts index 2b9f67c..7780430 100644 --- a/frontend/src/lib/api/client.test.ts +++ b/frontend/src/lib/api/client.test.ts @@ -68,4 +68,47 @@ describe('api client', () => { it('audioUrl returns a relative, session-scoped path', () => { expect(api.audioUrl('abc')).toBe('/api/sessions/abc/audio'); }); + + it('attachRecording resolves with the 202 accepted body, not a completed session', async () => { + (fetch as unknown as ReturnType).mockResolvedValueOnce( + new Response( + JSON.stringify({ + session: { id: 's1', status: 'PROCESSING' }, + processing: { job_status: 'PENDING', stage: 'QUEUED' } + }), + { status: 202, headers: { 'Content-Type': 'application/json' } } + ) + ); + const result = await api.attachRecording('s1', new Blob(['audio']), 42, 'clip.webm'); + expect(result.session.status).toBe('PROCESSING'); + expect(result.processing.job_status).toBe('PENDING'); + }); + + it('getSessionStatus calls the narrow status path', async () => { + (fetch as unknown as ReturnType).mockResolvedValueOnce( + jsonResponse({ session_status: 'PROCESSING' }) + ); + await api.getSessionStatus('s1'); + const [url] = (fetch as unknown as ReturnType).mock.calls[0]; + expect(url).toBe('/api/sessions/s1/status'); + }); + + it('getSessionStatus forwards an abort signal so a cancelled poll aborts in flight', async () => { + (fetch as unknown as ReturnType).mockResolvedValueOnce( + jsonResponse({ session_status: 'PROCESSING' }) + ); + const controller = new AbortController(); + await api.getSessionStatus('s1', controller.signal); + const [, init] = (fetch as unknown as ReturnType).mock.calls[0]; + expect(init.signal).toBe(controller.signal); + }); + + it('getRuntimeCapabilities calls the relative runtime path', async () => { + (fetch as unknown as ReturnType).mockResolvedValueOnce( + jsonResponse({ ready: false }) + ); + await api.getRuntimeCapabilities(); + const [url] = (fetch as unknown as ReturnType).mock.calls[0]; + expect(url).toBe('/api/runtime/capabilities'); + }); }); diff --git a/frontend/src/lib/api/client.ts b/frontend/src/lib/api/client.ts index 531d680..149e366 100644 --- a/frontend/src/lib/api/client.ts +++ b/frontend/src/lib/api/client.ts @@ -7,8 +7,11 @@ import type { ApiErrorBody, + ProcessingState, + RecordingAcceptedResponse, Reflection, RetentionSetting, + RuntimeCapabilities, Session, SessionDetail, SessionListResponse, @@ -83,16 +86,35 @@ export const api = { }); }, + /** + * Upload a recording. Resolves with 202 Accepted: the audio is stored and + * a transcription job is queued, but no transcript exists yet. Poll + * `getSessionStatus` until the session reaches COMPLETED or FAILED. + */ attachRecording( sessionId: string, file: Blob, clientReportedDurationSeconds: number, - filename: string - ): Promise { + filename: string, + signal?: AbortSignal + ): Promise { const form = new FormData(); form.set('client_reported_duration_seconds', String(Math.round(clientReportedDurationSeconds))); form.set('file', file, filename); - return request(`/api/sessions/${sessionId}/recording`, { method: 'POST', body: form }); + return request(`/api/sessions/${sessionId}/recording`, { + method: 'POST', + body: form, + signal + }); + }, + + /** The narrow, cheap payload polled while a session is processing. */ + getSessionStatus(sessionId: string, signal?: AbortSignal): Promise { + return request(`/api/sessions/${sessionId}/status`, { signal }); + }, + + getRuntimeCapabilities(signal?: AbortSignal): Promise { + return request('/api/runtime/capabilities', { signal }); }, saveReflection( diff --git a/frontend/src/lib/components/ProcessingStatus.svelte b/frontend/src/lib/components/ProcessingStatus.svelte new file mode 100644 index 0000000..71cb1a8 --- /dev/null +++ b/frontend/src/lib/components/ProcessingStatus.svelte @@ -0,0 +1,135 @@ + + +{#if failed} + +{:else if timedOut} +
+

Still waiting

+

+ Nothing has picked up this recording yet. Transcription runs in a separate local process — it + does not start on its own. +

+ {#if guidance} +

{guidance.title}

+

{guidance.detail}

+ {#if guidance.command} + 
{guidance.command}
+ {/if} + {/if} +

+ Your recording is safely stored. Once the worker runs, it will be picked up automatically — + reload this page then. +

+
+{:else} +
+
+

Transcribing

+ +
+

{stageLabel(processing?.stage)}…

+
+
+
+ {#if processing && processing.attempt_count > 1} +

+ Attempt {processing.attempt_count} of {processing.max_attempts} — a previous attempt did not finish. +

+ {/if} +

+ This runs entirely on your machine, so it takes roughly as long as the recording itself. You + can leave this page; the work continues. +

+
+{/if} + + diff --git a/frontend/src/lib/components/SessionResults.svelte b/frontend/src/lib/components/SessionResults.svelte index 1ba20a0..8568d08 100644 --- a/frontend/src/lib/components/SessionResults.svelte +++ b/frontend/src/lib/components/SessionResults.svelte @@ -1,8 +1,32 @@
@@ -17,25 +41,80 @@ {STATUS_LABELS[detail.session.status]}
- {#if detail.session.status === 'FAILED'} - - {:else if detail.session.status === 'COMPLETED' && detail.transcript && detail.metrics && detail.feedback} + {#if detail.session.status === 'COMPLETED' && transcript && detail.metrics && detail.feedback}

Transcript

- Simulated + {#if provenance.transcript === 'real'} + Real transcript + {:else} + Simulated + {/if}
-

- No real speech-to-text has run yet. This placeholder text is here so the full workflow can - be exercised end to end -- see docs/SCORING_AND_LIMITATIONS.md. -

-

{detail.transcript.text}

+ + {#if provenance.transcript === 'real'} +

+ Transcribed on this machine from your actual recording. Nothing was sent anywhere. +

+ {:else} +

+ This session predates real transcription — the text below is placeholder text, not + anything you said. +

+ {/if} + + {#if transcript.segments.length > 0} +
    + {#each transcript.segments as segment (segment.segment_index)} +
  1. + {formatTimestamp(segment.start_ms)} + {segment.text} +
    2. + {/each} +
+ {:else if transcript.text} +

{transcript.text}

+ {:else} +

No speech was detected in this recording.

+ {/if} + + {#if provenance.transcript === 'real'} +
+ How this transcript was produced +
+ {#if transcript.model_name} +
+
Model
+
{transcript.model_name}
+
+ {/if} + {#if transcript.runtime_version} +
+
Runtime
+
{transcript.runtime_version}
+
+ {/if} + {#if transcript.detected_language} +
+
Language
+
{transcript.detected_language}
+
+ {/if} + {#if speed} +
+
Speed
+
{speed}
+
+ {/if} + {#if transcript.model_sha256} +
+
Model checksum
+
{transcript.model_sha256.slice(0, 16)}…
+
+ {/if} +
+
+ {/if}
@@ -43,6 +122,10 @@

Metrics

Simulated +

+ These numbers are still generated, not measured from your speech — they are derived from the + session id, not from the transcript above. Real metrics are a later phase. +

Pace
@@ -73,6 +156,10 @@

Feedback

Simulated
+

+ Template text chosen from the simulated metrics above. No language model read your + transcript. +

Strength: {detail.feedback.strength}

Focus on next: {detail.feedback.improvement_target}

Try this: {detail.feedback.retry_instruction}

@@ -97,3 +184,62 @@
{/if} + + diff --git a/frontend/src/lib/processing/poller.test.ts b/frontend/src/lib/processing/poller.test.ts new file mode 100644 index 0000000..56539d4 --- /dev/null +++ b/frontend/src/lib/processing/poller.test.ts @@ -0,0 +1,326 @@ +import { beforeEach, describe, expect, it, vi } from 'vitest'; +import { + createSessionStatusPoller, + DEFAULT_INITIAL_INTERVAL_MS, + isTerminal, + type PollStopReason +} from './poller'; +import type { ProcessingState, ProcessingStage, SessionStatus } from '$lib/types'; + +function state( + session_status: SessionStatus, + stage: ProcessingStage | null = 'QUEUED', + extra: Partial = {} +): ProcessingState { + return { + session_id: 's1', + session_status, + job_id: 'j1', + job_status: 'PENDING', + stage, + attempt_count: 0, + max_attempts: 3, + error_code: null, + error_message: null, + failure_reason: null, + updated_at: '2026-07-31T12:00:00Z', + ...extra + }; +} + +/** A controllable clock + timer queue, so no real time passes in tests. */ +function fakeScheduler() { + let now = 0; + const queue: { at: number; fn: () => void; id: number }[] = []; + let nextId = 1; + + return { + nowFn: () => now, + setTimeoutFn: (fn: () => void, ms: number) => { + const id = nextId++; + queue.push({ at: now + ms, fn, id }); + return id; + }, + clearTimeoutFn: (handle: unknown) => { + const index = queue.findIndex((t) => t.id === handle); + if (index >= 0) queue.splice(index, 1); + }, + /** Run every timer due at or before `now + ms`, advancing the clock. */ + async advance(ms: number) { + const target = now + ms; + for (;;) { + queue.sort((a, b) => a.at - b.at); + const next = queue[0]; + if (!next || next.at > target) break; + queue.shift(); + now = next.at; + next.fn(); + await Promise.resolve(); + await Promise.resolve(); + } + now = target; + }, + get pending() { + return queue.length; + } + }; +} + +describe('isTerminal', () => { + it('treats COMPLETED and FAILED as terminal and everything else as not', () => { + expect(isTerminal(state('COMPLETED'))).toBe(true); + expect(isTerminal(state('FAILED'))).toBe(true); + expect(isTerminal(state('PROCESSING'))).toBe(false); + expect(isTerminal(state('CREATED'))).toBe(false); + expect(isTerminal(null)).toBe(false); + }); +}); + +describe('createSessionStatusPoller', () => { + let scheduler: ReturnType; + + beforeEach(() => { + scheduler = fakeScheduler(); + }); + + it('polls immediately on start and reports each update', async () => { + const fetchStatus = vi + .fn() + .mockResolvedValueOnce(state('PROCESSING', 'PREPARING_AUDIO')) + .mockResolvedValueOnce(state('PROCESSING', 'TRANSCRIBING')) + .mockResolvedValue(state('COMPLETED', 'COMPLETED')); + const onUpdate = vi.fn(); + + const poller = createSessionStatusPoller({ + sessionId: 's1', + fetchStatus, + onUpdate, + ...scheduler + }); + poller.start(); + await Promise.resolve(); + + expect(fetchStatus).toHaveBeenCalledTimes(1); + expect(onUpdate.mock.calls[0][0].stage).toBe('PREPARING_AUDIO'); + + await scheduler.advance(DEFAULT_INITIAL_INTERVAL_MS); + expect(onUpdate.mock.calls[1][0].stage).toBe('TRANSCRIBING'); + }); + + it('stops on COMPLETED and reports the reason', async () => { + const fetchStatus = vi.fn().mockResolvedValue(state('COMPLETED', 'COMPLETED')); + const onSettled = vi.fn(); + + const poller = createSessionStatusPoller({ + sessionId: 's1', + fetchStatus, + onSettled, + ...scheduler + }); + poller.start(); + await Promise.resolve(); + await Promise.resolve(); + + expect(onSettled).toHaveBeenCalledTimes(1); + expect(onSettled.mock.calls[0][0]).toBe('completed' satisfies PollStopReason); + expect(poller.running).toBe(false); + + // And it must not keep polling afterwards. + await scheduler.advance(60_000); + expect(fetchStatus).toHaveBeenCalledTimes(1); + }); + + it('stops on FAILED', async () => { + const fetchStatus = vi + .fn() + .mockResolvedValue(state('FAILED', 'FAILED', { error_code: 'MODEL_MISSING' })); + const onSettled = vi.fn(); + + createSessionStatusPoller({ sessionId: 's1', fetchStatus, onSettled, ...scheduler }).start(); + await Promise.resolve(); + await Promise.resolve(); + + expect(onSettled.mock.calls[0][0]).toBe('failed'); + expect(onSettled.mock.calls[0][1].error_code).toBe('MODEL_MISSING'); + }); + + it('stops when the session has been deleted (404)', async () => { + const fetchStatus = vi + .fn() + .mockRejectedValue(Object.assign(new Error('gone'), { status: 404 })); + const onSettled = vi.fn(); + + createSessionStatusPoller({ sessionId: 's1', fetchStatus, onSettled, ...scheduler }).start(); + await Promise.resolve(); + await Promise.resolve(); + + expect(onSettled.mock.calls[0][0]).toBe('deleted'); + }); + + it('backs off between polls instead of hammering the API', async () => { + const fetchStatus = vi.fn().mockResolvedValue(state('PROCESSING')); + createSessionStatusPoller({ sessionId: 's1', fetchStatus, ...scheduler }).start(); + await Promise.resolve(); + expect(fetchStatus).toHaveBeenCalledTimes(1); + + // First gap is ~1s. + await scheduler.advance(1000); + expect(fetchStatus).toHaveBeenCalledTimes(2); + + // The next gap is longer than 1s, so 1s more is not enough. + await scheduler.advance(1000); + expect(fetchStatus).toHaveBeenCalledTimes(2); + + await scheduler.advance(600); + expect(fetchStatus).toHaveBeenCalledTimes(3); + }); + + it('caps the backoff so polling never stalls out entirely', async () => { + const fetchStatus = vi.fn().mockResolvedValue(state('PROCESSING')); + createSessionStatusPoller({ + sessionId: 's1', + fetchStatus, + maxIntervalMs: 5000, + ...scheduler + }).start(); + await Promise.resolve(); + + await scheduler.advance(120_000); + const callsAfterTwoMinutes = fetchStatus.mock.calls.length; + // At a 5s cap, two minutes must yield at least ~20 polls. + expect(callsAfterTwoMinutes).toBeGreaterThan(20); + }); + + it('gives up after the overall timeout', async () => { + const fetchStatus = vi.fn().mockResolvedValue(state('PROCESSING')); + const onSettled = vi.fn(); + + createSessionStatusPoller({ + sessionId: 's1', + fetchStatus, + onSettled, + timeoutMs: 10_000, + ...scheduler + }).start(); + await Promise.resolve(); + + await scheduler.advance(30_000); + expect(onSettled).toHaveBeenCalledTimes(1); + expect(onSettled.mock.calls[0][0]).toBe('timeout'); + }); + + it('keeps polling through a transient error rather than giving up', async () => { + const fetchStatus = vi + .fn() + .mockRejectedValueOnce(Object.assign(new Error('backend restarting'), { status: 503 })) + .mockResolvedValue(state('COMPLETED', 'COMPLETED')); + const onError = vi.fn(); + const onSettled = vi.fn(); + + createSessionStatusPoller({ + sessionId: 's1', + fetchStatus, + onError, + onSettled, + ...scheduler + }).start(); + await Promise.resolve(); + await Promise.resolve(); + + expect(onError).toHaveBeenCalledTimes(1); + expect(onSettled).not.toHaveBeenCalled(); + + await scheduler.advance(2000); + expect(onSettled.mock.calls[0][0]).toBe('completed'); + }); + + it('cancel() stops polling and fires no settled callback', async () => { + const fetchStatus = vi.fn().mockResolvedValue(state('PROCESSING')); + const onSettled = vi.fn(); + + const poller = createSessionStatusPoller({ + sessionId: 's1', + fetchStatus, + onSettled, + ...scheduler + }); + poller.start(); + await Promise.resolve(); + expect(fetchStatus).toHaveBeenCalledTimes(1); + + poller.cancel(); + await scheduler.advance(60_000); + + expect(fetchStatus).toHaveBeenCalledTimes(1); + expect(onSettled).not.toHaveBeenCalled(); + expect(poller.running).toBe(false); + }); + + it('ignores a response that arrives after cancel()', async () => { + // The route was destroyed mid-request; a late update must not touch it. + let resolve!: (v: ProcessingState) => void; + const fetchStatus = vi.fn().mockReturnValue( + new Promise((r) => { + resolve = r; + }) + ); + const onUpdate = vi.fn(); + const onSettled = vi.fn(); + + const poller = createSessionStatusPoller({ + sessionId: 's1', + fetchStatus, + onUpdate, + onSettled, + ...scheduler + }); + poller.start(); + await Promise.resolve(); + + poller.cancel(); + resolve(state('COMPLETED', 'COMPLETED')); + await Promise.resolve(); + await Promise.resolve(); + + expect(onUpdate).not.toHaveBeenCalled(); + expect(onSettled).not.toHaveBeenCalled(); + }); + + it('starting twice does not create a second polling loop', async () => { + const fetchStatus = vi.fn().mockResolvedValue(state('PROCESSING')); + const poller = createSessionStatusPoller({ sessionId: 's1', fetchStatus, ...scheduler }); + + poller.start(); + poller.start(); + poller.start(); + await Promise.resolve(); + + expect(fetchStatus).toHaveBeenCalledTimes(1); + + await scheduler.advance(1000); + expect(fetchStatus).toHaveBeenCalledTimes(2); + }); + + it('cannot be restarted after settling', async () => { + const fetchStatus = vi.fn().mockResolvedValue(state('COMPLETED', 'COMPLETED')); + const poller = createSessionStatusPoller({ sessionId: 's1', fetchStatus, ...scheduler }); + + poller.start(); + await Promise.resolve(); + await Promise.resolve(); + poller.start(); + await scheduler.advance(10_000); + + expect(fetchStatus).toHaveBeenCalledTimes(1); + }); + + it('leaves no pending timer once settled', async () => { + const fetchStatus = vi.fn().mockResolvedValue(state('COMPLETED', 'COMPLETED')); + const poller = createSessionStatusPoller({ sessionId: 's1', fetchStatus, ...scheduler }); + poller.start(); + await Promise.resolve(); + await Promise.resolve(); + + expect(scheduler.pending).toBe(0); + }); +}); diff --git a/frontend/src/lib/processing/poller.ts b/frontend/src/lib/processing/poller.ts new file mode 100644 index 0000000..b71706f --- /dev/null +++ b/frontend/src/lib/processing/poller.ts @@ -0,0 +1,172 @@ +// Polls a session's processing status until it settles. +// +// Ordinary HTTP polling, deliberately -- no WebSocket, no SSE. One user on +// one machine waiting on one job does not need a persistent connection, and +// a connection would bring reconnection logic, a second server-side +// lifecycle, and a proxying story the dev setup does not currently have. +// See docs/adr/0008-http-polling-not-websockets.md. +// +// Everything time-related and network-related is injected, so the whole +// lifecycle is testable with fake timers and no real fetch. +// +// The rules this enforces, each of which is a bug if missed: +// - Exactly one loop per poller. Starting twice must not double the rate. +// - Stop on COMPLETED, FAILED, a 404 (the session was deleted), an overall +// timeout, or an explicit cancel from route teardown. +// - Back off from ~1s so a slow transcription does not hammer the API, +// but stay responsive for the common fast case. +// - After cancel(), no further callback fires -- a late in-flight response +// must not update a destroyed component. + +import type { ProcessingState } from '$lib/types'; + +export type PollStopReason = 'completed' | 'failed' | 'deleted' | 'timeout' | 'cancelled' | 'error'; + +export interface PollerOptions { + sessionId: string; + /** Usually `api.getSessionStatus`. */ + fetchStatus: (sessionId: string, signal?: AbortSignal) => Promise; + onUpdate?: (state: ProcessingState) => void; + onSettled?: (reason: PollStopReason, state: ProcessingState | null) => void; + onError?: (error: unknown) => void; + initialIntervalMs?: number; + maxIntervalMs?: number; + backoffFactor?: number; + /** Give up after this long; the worker may not be running at all. */ + timeoutMs?: number; + setTimeoutFn?: (fn: () => void, ms: number) => unknown; + clearTimeoutFn?: (handle: unknown) => void; + nowFn?: () => number; +} + +export const DEFAULT_INITIAL_INTERVAL_MS = 1000; +export const DEFAULT_MAX_INTERVAL_MS = 5000; +export const DEFAULT_BACKOFF_FACTOR = 1.5; +// Ten minutes: comfortably longer than a slow local transcription of a +// several-minute clip, short enough that a never-started worker surfaces as +// actionable guidance rather than an indefinite spinner. +export const DEFAULT_TIMEOUT_MS = 10 * 60 * 1000; + +const TERMINAL_STATUSES = ['COMPLETED', 'FAILED'] as const; + +export function isTerminal(state: ProcessingState | null): boolean { + if (!state) return false; + return (TERMINAL_STATUSES as readonly string[]).includes(state.session_status); +} + +export interface SessionStatusPoller { + start(): void; + cancel(): void; + readonly running: boolean; + readonly lastState: ProcessingState | null; +} + +export function createSessionStatusPoller(options: PollerOptions): SessionStatusPoller { + const { + sessionId, + fetchStatus, + onUpdate, + onSettled, + onError, + initialIntervalMs = DEFAULT_INITIAL_INTERVAL_MS, + maxIntervalMs = DEFAULT_MAX_INTERVAL_MS, + backoffFactor = DEFAULT_BACKOFF_FACTOR, + timeoutMs = DEFAULT_TIMEOUT_MS, + setTimeoutFn = (fn, ms) => setTimeout(fn, ms), + clearTimeoutFn = (handle) => clearTimeout(handle as ReturnType), + nowFn = () => Date.now() + } = options; + + let running = false; + let stopped = false; + let handle: unknown = null; + let controller: AbortController | null = null; + let interval = initialIntervalMs; + let startedAt = 0; + let lastState: ProcessingState | null = null; + + function settle(reason: PollStopReason): void { + if (stopped) return; + stopped = true; + running = false; + if (handle !== null) { + clearTimeoutFn(handle); + handle = null; + } + controller?.abort(); + controller = null; + // 'cancelled' is route teardown, not an outcome the page should react + // to -- the component is already gone. + if (reason !== 'cancelled') onSettled?.(reason, lastState); + } + + function schedule(): void { + if (stopped) return; + handle = setTimeoutFn(() => { + handle = null; + void tick(); + }, interval); + interval = Math.min(Math.round(interval * backoffFactor), maxIntervalMs); + } + + async function tick(): Promise { + if (stopped) return; + + if (nowFn() - startedAt >= timeoutMs) { + settle('timeout'); + return; + } + + controller = typeof AbortController !== 'undefined' ? new AbortController() : null; + let state: ProcessingState; + try { + state = await fetchStatus(sessionId, controller?.signal); + } catch (err) { + // A response that arrives after cancel() must change nothing. + if (stopped) return; + const status = (err as { status?: number })?.status; + if (status === 404) { + settle('deleted'); + return; + } + // Transient failures (backend restarting, a dropped request) are + // not fatal: report and keep polling with the backoff already + // applied, so a flapping backend is not hammered. + onError?.(err); + schedule(); + return; + } + + if (stopped) return; + + lastState = state; + onUpdate?.(state); + + if (isTerminal(state)) { + settle(state.session_status === 'COMPLETED' ? 'completed' : 'failed'); + return; + } + schedule(); + } + + return { + start(): void { + // Guard against a second start doubling the poll rate -- e.g. a + // component that re-runs its effect on a prop change. + if (running || stopped) return; + running = true; + startedAt = nowFn(); + interval = initialIntervalMs; + void tick(); + }, + cancel(): void { + settle('cancelled'); + }, + get running(): boolean { + return running; + }, + get lastState(): ProcessingState | null { + return lastState; + } + }; +} diff --git a/frontend/src/lib/processing/presentation.test.ts b/frontend/src/lib/processing/presentation.test.ts new file mode 100644 index 0000000..6212a99 --- /dev/null +++ b/frontend/src/lib/processing/presentation.test.ts @@ -0,0 +1,198 @@ +import { describe, expect, it } from 'vitest'; +import { + describeSpeed, + formatTimestamp, + guidanceForErrorCode, + isEntirelySimulated, + PROVENANCE_LABELS, + sectionProvenance, + stageLabel, + stageProgress +} from './presentation'; +import type { Feedback, Metrics, SessionDetail, Transcript } from '$lib/types'; + +function transcript(is_mock: boolean): Transcript { + return { + id: 't1', + session_id: 's1', + provider_name: is_mock ? 'mock-transcription' : 'whisper-cpp', + provider_version: '0.1.0', + text: 'Hello there.', + is_mock, + created_at: '2026-07-31T12:00:00Z', + detected_language: is_mock ? null : 'en', + runtime_version: is_mock ? null : 'whisper.cpp v1.9.1 (f049fff95a08)', + model_name: is_mock ? null : 'base.en', + model_sha256: is_mock ? null : 'a'.repeat(64), + parameters: is_mock ? null : { language: 'en' }, + audio_duration_ms: 12500, + processing_duration_ms: 2000, + real_time_factor: 0.16, + segments: [] + }; +} + +function metrics(): Metrics { + return { + id: 'm1', + session_id: 's1', + provider_name: 'mock-metrics', + provider_version: '0.1.0', + words_per_minute: 130, + filler_word_count: 3, + filler_words: { um: 3 }, + pause_count: 2, + longest_pause_seconds: 1.5, + is_mock: true, + created_at: '2026-07-31T12:00:00Z' + }; +} + +function feedback(): Feedback { + return { + id: 'f1', + session_id: 's1', + provider_name: 'mock-feedback', + provider_version: '0.1.0', + strength: '[Simulated] Good.', + improvement_target: '[Simulated] Better.', + retry_instruction: '[Simulated] Again.', + is_mock: true, + created_at: '2026-07-31T12:00:00Z' + }; +} + +function detail(overrides: Partial = {}): SessionDetail { + return { + session: { + id: 's1', + track: 'technical', + topic_id: 'topic-1', + topic_title: 'Explain DNS', + topic_category: 'networking', + mode: 'explain', + prep_seconds_planned: 60, + speaking_seconds_planned: 90, + client_reported_duration_seconds: 12, + audio_duration_seconds: 12.5, + status: 'COMPLETED', + failure_reason: null, + created_at: '2026-07-31T12:00:00Z', + updated_at: '2026-07-31T12:00:00Z' + }, + recording: null, + reflection: null, + transcript: transcript(false), + metrics: metrics(), + feedback: feedback(), + processing: null, + ...overrides + }; +} + +describe('stage presentation', () => { + it('gives a human label for each stage', () => { + expect(stageLabel('PREPARING_AUDIO')).toBe('Converting your recording'); + expect(stageLabel('TRANSCRIBING')).toBe('Transcribing your speech'); + expect(stageLabel('QUEUED')).toBe('Waiting for the transcription worker'); + }); + + it('falls back gracefully for a missing stage', () => { + expect(stageLabel(null)).toBe('Preparing'); + expect(stageLabel(undefined)).toBe('Preparing'); + }); + + it('reports increasing progress through the pipeline', () => { + expect(stageProgress('QUEUED')).toBe(0); + expect(stageProgress('TRANSCRIBING')).toBeGreaterThan(stageProgress('PREPARING_AUDIO')); + expect(stageProgress('COMPLETED')).toBe(100); + }); +}); + +describe('failure guidance', () => { + it('tells the user the exact command for each missing prerequisite', () => { + expect(guidanceForErrorCode('FFMPEG_NOT_FOUND')?.command).toBe('brew install ffmpeg'); + expect(guidanceForErrorCode('WHISPER_BINARY_MISSING')?.command).toBe( + 'scripts/setup_whisper.sh' + ); + expect(guidanceForErrorCode('MODEL_MISSING')?.command).toContain('--model base.en'); + expect(guidanceForErrorCode('WORKER_LOST')?.command).toBe('python -m app.worker'); + }); + + it('always reassures that the recording survived a setup failure', () => { + for (const code of ['FFMPEG_NOT_FOUND', 'WHISPER_BINARY_MISSING', 'MODEL_MISSING']) { + expect(guidanceForErrorCode(code)?.detail).toContain('recording was preserved'); + } + }); + + it('returns null for an unknown or absent code rather than inventing advice', () => { + expect(guidanceForErrorCode('SOMETHING_NEW')).toBeNull(); + expect(guidanceForErrorCode(null)).toBeNull(); + expect(guidanceForErrorCode(undefined)).toBeNull(); + }); +}); + +describe('section provenance', () => { + it('labels a real transcript real while metrics and feedback stay simulated', () => { + // The central Phase 2A claim: one page, three different truths. + const p = sectionProvenance(detail()); + expect(p.transcript).toBe('real'); + expect(p.metrics).toBe('simulated'); + expect(p.feedback).toBe('simulated'); + }); + + it('keeps a historical Phase 1 session labelled simulated throughout', () => { + const p = sectionProvenance(detail({ transcript: transcript(true) })); + expect(p.transcript).toBe('simulated'); + expect(p.metrics).toBe('simulated'); + expect(p.feedback).toBe('simulated'); + }); + + it('derives each label from that row alone, not from the page', () => { + // A real transcript must not make mock metrics look real. + const p = sectionProvenance(detail()); + expect(p.metrics).toBe('simulated'); + expect(PROVENANCE_LABELS[p.transcript]).toBe('Real'); + expect(PROVENANCE_LABELS[p.metrics]).toBe('Simulated'); + }); + + it('marks a missing section unavailable rather than guessing', () => { + const p = sectionProvenance(detail({ transcript: null, metrics: null, feedback: null })); + expect(p.transcript).toBe('unavailable'); + expect(p.metrics).toBe('unavailable'); + expect(p.feedback).toBe('unavailable'); + }); + + it('only calls a session entirely simulated when every section is', () => { + expect(isEntirelySimulated(detail())).toBe(false); + expect(isEntirelySimulated(detail({ transcript: transcript(true) }))).toBe(true); + }); + + it('does not call an empty session entirely simulated', () => { + expect(isEntirelySimulated(detail({ transcript: null, metrics: null, feedback: null }))).toBe( + false + ); + }); +}); + +describe('transcript formatting', () => { + it('formats segment timestamps as m:ss', () => { + expect(formatTimestamp(0)).toBe('0:00'); + expect(formatTimestamp(65_000)).toBe('1:05'); + expect(formatTimestamp(605_000)).toBe('10:05'); + }); + + it('never renders a negative timestamp', () => { + expect(formatTimestamp(-500)).toBe('0:00'); + }); + + it('describes speed in terms a person can check', () => { + expect(describeSpeed(0.16, 12500, 2000)).toBe('0.16× real time (12.5s of audio in 2.0s)'); + }); + + it('returns null rather than a made-up figure when data is missing', () => { + expect(describeSpeed(null, 12500, 2000)).toBeNull(); + expect(describeSpeed(0.16, null, 2000)).toBeNull(); + expect(describeSpeed(0.16, 12500, null)).toBeNull(); + }); +}); diff --git a/frontend/src/lib/processing/presentation.ts b/frontend/src/lib/processing/presentation.ts new file mode 100644 index 0000000..83f6f11 --- /dev/null +++ b/frontend/src/lib/processing/presentation.ts @@ -0,0 +1,191 @@ +// Pure presentation logic for processing state and result provenance. +// +// These are plain functions rather than logic inside .svelte files so they +// can be unit-tested directly (the project has no component-testing +// dependency -- see docs/TESTING.md for that trade-off). +// +// The provenance part matters more than it looks. Before Phase 2A every +// result was simulated, so one banner over the whole page was honest. Now +// the transcript is real while metrics and feedback are still simulated, so +// a single global banner would be a lie in one direction or the other. Each +// section carries its own label, derived from the `is_mock` flag stored on +// its own database row -- never from a page-level assumption. + +import type { ProcessingStage, SessionDetail } from '$lib/types'; + +// --- Processing stage -------------------------------------------------------- + +const STAGE_LABELS: Record = { + QUEUED: 'Waiting for the transcription worker', + CLAIMED: 'Starting up', + PREPARING_AUDIO: 'Converting your recording', + TRANSCRIBING: 'Transcribing your speech', + ANALYZING: 'Preparing results', + SAVING: 'Saving results', + COMPLETED: 'Done', + FAILED: 'Failed' +}; + +export function stageLabel(stage: ProcessingStage | null | undefined): string { + if (!stage) return 'Preparing'; + return STAGE_LABELS[stage] ?? 'Processing'; +} + +/** Rough progress for a determinate-looking indicator. Not a real percentage. */ +export function stageProgress(stage: ProcessingStage | null | undefined): number { + const order: ProcessingStage[] = [ + 'QUEUED', + 'CLAIMED', + 'PREPARING_AUDIO', + 'TRANSCRIBING', + 'ANALYZING', + 'SAVING', + 'COMPLETED' + ]; + if (!stage) return 0; + const index = order.indexOf(stage); + if (index < 0) return 0; + return Math.round((index / (order.length - 1)) * 100); +} + +// --- Failure guidance -------------------------------------------------------- + +export interface Guidance { + title: string; + detail: string; + command?: string; +} + +/** + * Map a backend error code to something the owner can act on. + * + * The backend already returns a safe, redacted sentence; this adds the + * specific local fix, which the backend deliberately does not embed because + * it would mean putting setup paths into an API response. + */ +export function guidanceForErrorCode(code: string | null | undefined): Guidance | null { + switch (code) { + case 'FFMPEG_NOT_FOUND': + return { + title: 'ffmpeg is not installed', + detail: + 'Your recording has to be decoded before it can be transcribed, and that needs ffmpeg. Your recording was preserved.', + command: 'brew install ffmpeg' + }; + case 'WHISPER_BINARY_MISSING': + return { + title: 'The transcription engine has not been built', + detail: + 'whisper.cpp is built locally from a pinned release; it is not bundled with this project. Your recording was preserved.', + command: 'scripts/setup_whisper.sh' + }; + case 'MODEL_MISSING': + return { + title: 'The transcription model has not been downloaded', + detail: + 'Model weights are large and are never committed to this repository, so they are downloaded separately. Your recording was preserved.', + command: 'scripts/setup_whisper.sh --model base.en' + }; + case 'WHISPER_TIMEOUT': + case 'FFMPEG_TIMEOUT': + return { + title: 'Processing took too long and was stopped', + detail: + 'This can happen with a long recording on a busy machine. Your recording was preserved, so you can try again.' + }; + case 'WHISPER_OUTPUT_MALFORMED': + case 'WHISPER_OUTPUT_MISSING': + case 'WHISPER_FAILED': + return { + title: 'The transcription engine did not produce usable output', + detail: 'Your recording was preserved. Check the runtime is correctly built and try again.', + command: 'scripts/check_whisper_runtime.sh' + }; + case 'AUDIO_DECODE_FAILED': + case 'RECORDING_FILE_MISSING': + return { + title: 'The recording could not be read', + detail: 'The audio may be corrupt or incomplete. Try recording this topic again.' + }; + case 'WORKER_LOST': + return { + title: 'Processing was interrupted', + detail: 'The worker stopped partway through. Start it again and the job will be picked up.', + command: 'python -m app.worker' + }; + default: + return null; + } +} + +/** Shown when a session sits queued with no worker picking it up. */ +export const WORKER_NOT_RUNNING_GUIDANCE: Guidance = { + title: 'The transcription worker does not appear to be running', + detail: + 'Transcription happens in a separate local process. Start it in its own terminal; your recording is safely stored until it runs.', + command: 'cd backend && source .venv/bin/activate && python -m app.worker' +}; + +// --- Result provenance ------------------------------------------------------- + +export type Provenance = 'real' | 'simulated' | 'unavailable'; + +export interface SectionProvenance { + transcript: Provenance; + metrics: Provenance; + feedback: Provenance; +} + +function provenanceOf(row: { is_mock: boolean } | null | undefined): Provenance { + if (!row) return 'unavailable'; + return row.is_mock ? 'simulated' : 'real'; +} + +/** + * Per-section provenance, each read from its own row's is_mock flag. + * + * Deliberately not derived from anything page-level: a historical Phase 1 + * session has a simulated transcript and must keep saying so, while a new + * session has a real one -- both render through the same component. + */ +export function sectionProvenance(detail: SessionDetail): SectionProvenance { + return { + transcript: provenanceOf(detail.transcript), + metrics: provenanceOf(detail.metrics), + feedback: provenanceOf(detail.feedback) + }; +} + +/** True only when every present section is simulated (a Phase 1 session). */ +export function isEntirelySimulated(detail: SessionDetail): boolean { + const p = sectionProvenance(detail); + const present = [p.transcript, p.metrics, p.feedback].filter((v) => v !== 'unavailable'); + return present.length > 0 && present.every((v) => v === 'simulated'); +} + +export const PROVENANCE_LABELS: Record = { + real: 'Real', + simulated: 'Simulated', + unavailable: 'Not available' +}; + +// --- Transcript formatting --------------------------------------------------- + +export function formatTimestamp(ms: number): string { + const totalSeconds = Math.max(0, Math.floor(ms / 1000)); + const minutes = Math.floor(totalSeconds / 60); + const seconds = totalSeconds % 60; + return `${minutes}:${String(seconds).padStart(2, '0')}`; +} + +/** e.g. "0.16x real time (12.5s of audio in 2.0s)". */ +export function describeSpeed( + realTimeFactor: number | null, + audioMs: number | null, + processingMs: number | null +): string | null { + if (realTimeFactor === null || audioMs === null || processingMs === null) return null; + return `${realTimeFactor.toFixed(2)}× real time (${(audioMs / 1000).toFixed(1)}s of audio in ${( + processingMs / 1000 + ).toFixed(1)}s)`; +} diff --git a/frontend/src/lib/types.ts b/frontend/src/lib/types.ts index d1ff4ac..bc82cc4 100644 --- a/frontend/src/lib/types.ts +++ b/frontend/src/lib/types.ts @@ -27,6 +27,18 @@ export interface Topic { updated_at: string; } +export type JobStatus = 'PENDING' | 'RUNNING' | 'SUCCEEDED' | 'FAILED'; + +export type ProcessingStage = + | 'QUEUED' + | 'CLAIMED' + | 'PREPARING_AUDIO' + | 'TRANSCRIBING' + | 'ANALYZING' + | 'SAVING' + | 'COMPLETED' + | 'FAILED'; + export interface Session { id: string; track: Track; @@ -36,13 +48,58 @@ export interface Session { mode: SpeakingMode; prep_seconds_planned: number; speaking_seconds_planned: number; + /** The browser's own stopwatch -- approximate, kept for comparison. */ client_reported_duration_seconds: number | null; + /** Measured by decoding the audio. Null until the worker has run. */ + audio_duration_seconds: number | null; status: SessionStatus; failure_reason: string | null; created_at: string; updated_at: string; } +/** The narrow payload polled while a session is processing. */ +export interface ProcessingState { + session_id: string; + session_status: SessionStatus; + job_id: string | null; + job_status: JobStatus | null; + stage: ProcessingStage | null; + attempt_count: number; + max_attempts: number; + error_code: string | null; + error_message: string | null; + failure_reason: string | null; + updated_at: string | null; +} + +export interface RecordingAcceptedResponse { + session: Session; + processing: ProcessingState; +} + +export interface RuntimeComponent { + available: boolean; + version: string | null; + detail: string | null; +} + +export interface RuntimeCapabilities { + transcription_provider: string; + worker: RuntimeComponent; + ffmpeg: RuntimeComponent; + whisper_binary: RuntimeComponent; + model: RuntimeComponent; + model_name: string; + model_size_bytes: number | null; + platform: string; + architecture: string; + pinned_release_tag: string; + ready: boolean; + pending_jobs: number; + running_jobs: number; +} + export interface SessionListItem { id: string; track: Track; @@ -77,6 +134,13 @@ export interface Reflection { created_at: string; } +export interface TranscriptSegment { + segment_index: number; + start_ms: number; + end_ms: number; + text: string; +} + export interface Transcript { id: string; session_id: string; @@ -85,6 +149,16 @@ export interface Transcript { text: string; is_mock: boolean; created_at: string; + // Phase 2A provenance. All nullable: Phase 1 rows have none of it. + detected_language: string | null; + runtime_version: string | null; + model_name: string | null; + model_sha256: string | null; + parameters: Record | null; + audio_duration_ms: number | null; + processing_duration_ms: number | null; + real_time_factor: number | null; + segments: TranscriptSegment[]; } export interface Metrics { @@ -120,6 +194,7 @@ export interface SessionDetail { transcript: Transcript | null; metrics: Metrics | null; feedback: Feedback | null; + processing: ProcessingState | null; } export interface RetentionSetting { diff --git a/frontend/src/routes/history/[id]/+page.svelte b/frontend/src/routes/history/[id]/+page.svelte index 828dc41..fb078da 100644 --- a/frontend/src/routes/history/[id]/+page.svelte +++ b/frontend/src/routes/history/[id]/+page.svelte @@ -2,7 +2,9 @@ import { onMount } from 'svelte'; import { goto } from '$app/navigation'; import { api, ApiError } from '$lib/api/client'; + import ProcessingStatus from '$lib/components/ProcessingStatus.svelte'; import SessionResults from '$lib/components/SessionResults.svelte'; + import { isTerminal } from '$lib/processing/poller'; import type { SessionDetail } from '$lib/types'; import type { PageProps } from './$types'; @@ -58,6 +60,19 @@ {/if} + + {#if !isTerminal(detail.processing)} + + {:else if detail.session.status === 'FAILED'} + + {/if} + {#if errorMessage} diff --git a/frontend/src/routes/practice/reflect/+page.svelte b/frontend/src/routes/practice/reflect/+page.svelte index b2fe9cd..ac7bccd 100644 --- a/frontend/src/routes/practice/reflect/+page.svelte +++ b/frontend/src/routes/practice/reflect/+page.svelte @@ -1,8 +1,15 @@
@@ -31,6 +86,11 @@ {:else if detail}

{detail.session.topic_title}

+ + {#if stillProcessing || failed} + + {/if} +
diff --git a/frontend/src/routes/practice/review/+page.svelte b/frontend/src/routes/practice/review/+page.svelte index 08e60e9..d0d1bd7 100644 --- a/frontend/src/routes/practice/review/+page.svelte +++ b/frontend/src/routes/practice/review/+page.svelte @@ -36,6 +36,9 @@ speaking_seconds_planned: draft.speakingSecondsPlanned }); const filename = filenameForMimeType(draft.recording.mimeType); + // Returns 202: the audio is stored and a transcription job is + // queued. We do NOT wait for a transcript here -- the user moves + // straight on to reflecting while the worker runs. await api.attachRecording( session.id, draft.recording.blob, From b7cee948ebabd61e54458fda48d14dc8477ebf35 Mon Sep 17 00:00:00 2001 From: Abdulrahman Date: Fri, 31 Jul 2026 22:09:11 +0300 Subject: [PATCH 9/9] Document the Phase 2A architecture and add ADRs 0006-0009 Four new ADRs: the separate worker and SQLite job queue (including why no broker and why not an in-process background task), the ffmpeg preprocessing boundary, HTTP polling over WebSockets/SSE, and pinned external whisper.cpp runtime management. AI_MODEL_STRATEGY.md now corrects, rather than quietly edits away, the Phase 1 claim that this would be a one-line dependency swap. The provider boundary was right; the signature was not. The interface received no audio, the result had nowhere to record provenance, the call site had to leave the request entirely, and a decode step had to exist at all. An interface localises where a change lands -- it does not make the change small, and it cannot anticipate what a real implementation needs. SCORING_AND_LIMITATIONS.md now describes three different truths on one page instead of one blanket disclaimer, and states plainly that the simulated metrics are still not derived from the real transcript sitting directly above them. It also records that transcription accuracy is itself unmeasured here. LEARNING_NOTES.md explains why transcription cannot run in the request, why a worker is not an AI agent, how SQLite serves as a durable queue, what atomic claiming means and the double-transcription bug it prevents, why browser audio must be decoded, weights versus runtime, why versions must be stored, what real-time factor means, why the original recording and the temporary PCM are treated oppositely, and why polling suffices. PROJECT_STATUS.md records exact results and is explicit about what did NOT happen: no whisper.cpp build, no real-model run, and no benchmark has ever executed here, because this machine has neither cmake nor ffmpeg. No latency, real-time-factor or accuracy figure is published anywhere, because none was measured. AGENTS.md moves whisper.cpp, ffmpeg and the local worker from prohibited to approved, keeps message brokers prohibited, and adds the rules this phase produced -- never silently substitute a mock for a real provider, never fabricate a measurement, all subprocesses through one argv-only helper, and new runtime artifacts need both a gitignore entry and a safety-check rule in the same change. The safety check flagged this change's own test fixtures for containing literal home-directory paths. Rather than weaken the check, the fixtures now assemble that path at runtime. --- AGENTS.md | 35 ++- README.md | 90 ++++-- backend/app/safe_text.py | 6 +- backend/tests/test_audio_preprocessor.py | 32 ++- docs/AI_MODEL_STRATEGY.md | 135 +++++++-- docs/ARCHITECTURE.md | 140 ++++++--- docs/DATA_FLOW.md | 100 +++++-- docs/DATA_MODEL.md | 97 +++++-- docs/LEARNING_NOTES.md | 268 +++++++++++++++++- docs/PRIVACY_AND_SECURITY.md | 80 +++++- docs/PROJECT_STATUS.md | 237 ++++++++++------ docs/ROADMAP.md | 101 ++++--- docs/SCORING_AND_LIMITATIONS.md | 94 +++--- docs/TESTING.md | 128 +++++++-- docs/adr/0006-local-worker-sqlite-queue.md | 83 ++++++ .../adr/0007-ffmpeg-preprocessing-boundary.md | 74 +++++ docs/adr/0008-http-polling-not-websockets.md | 61 ++++ docs/adr/0009-pinned-whisper-cpp-runtime.md | 75 +++++ 18 files changed, 1512 insertions(+), 324 deletions(-) create mode 100644 docs/adr/0006-local-worker-sqlite-queue.md create mode 100644 docs/adr/0007-ffmpeg-preprocessing-boundary.md create mode 100644 docs/adr/0008-http-polling-not-websockets.md create mode 100644 docs/adr/0009-pinned-whisper-cpp-runtime.md diff --git a/AGENTS.md b/AGENTS.md index 9e4b43f..4610f9b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -36,7 +36,10 @@ it to learn software architecture, so how changes are made matters as much as wh task, mention it or file it separately — don't fold it into an unrelated change. - **Follow the existing provider-interface pattern** when adding or changing AI-adjacent functionality (`TranscriptionProvider`, `MetricsProvider`, `FeedbackProvider`) — new - implementations go behind the existing interface, not as special-cased calls elsewhere. + implementations go behind the existing interface, not as special-cased calls elsewhere. If a + real implementation genuinely needs a different signature than the mock, **change the interface + deliberately and document why** rather than smuggling data in through the side — see the + correction in [docs/AI_MODEL_STRATEGY.md](docs/AI_MODEL_STRATEGY.md). - Match the existing test discipline: any new backend logic gets a `pytest` test, any new frontend logic gets a `vitest` test where practical (see [docs/TESTING.md](docs/TESTING.md) for what "where practical" has meant so far). @@ -70,8 +73,32 @@ the same change. Test fixtures must be synthetic, never real personal data. ## Explicit scope guardrails -Do not add, without the owner's explicit approval: real Whisper integration, ffmpeg, Ollama, RAG, -embeddings, vector databases, agents, Tauri packaging, Docker, cloud infrastructure, user -accounts, Playwright/e2e browser automation, or webcam processing. See +**Approved and now present** (Phase 2A, with the owner's explicit approval): real whisper.cpp +transcription, ffmpeg, and a local worker process with a SQLite job queue. See ADRs +[0006](docs/adr/0006-local-worker-sqlite-queue.md), +[0007](docs/adr/0007-ffmpeg-preprocessing-boundary.md), +[0008](docs/adr/0008-http-polling-not-websockets.md), and +[0009](docs/adr/0009-pinned-whisper-cpp-runtime.md). + +Still **not** to be added without the owner's explicit approval: Ollama, RAG, embeddings, vector +databases, agents, Tauri packaging, Docker, cloud infrastructure, user accounts, Playwright/e2e +browser automation, webcam processing, or a message broker (Redis/RabbitMQ/Celery/SQS — see +[ADR 0006](docs/adr/0006-local-worker-sqlite-queue.md) for why SQLite is the queue). See [docs/ROADMAP.md](docs/ROADMAP.md) for what's actually planned and why each of these is sequenced where it is (or not planned at all). + +## Rules that came out of Phase 2A + +- **Never silently substitute a mock for a real provider.** If whisper.cpp is unavailable, the job + fails with an actionable error code. Putting simulated text under a UI label reading "Real + transcript" would be the single worst bug this system could have. +- **Never fabricate a measurement.** No benchmark number, latency figure, accuracy claim, or + "verified" statement goes in this repository unless it was actually produced on real hardware. + If something couldn't be run, say which prerequisite was missing. +- **All external processes go through `app/subprocess_util.py: run_command`** — argv arrays, never + `shell=True`, always with a timeout. +- **Never let subprocess output or absolute paths reach the API, the job table, or a log.** Route + it through `app/safe_text.py` first; absolute paths contain the owner's username. +- **New runtime artifact kinds need a `.gitignore` entry AND a + `scripts/check_public_safety.sh` rule in the same change.** +- **Test fixtures stay synthetic.** Generate silent WAVs arithmetically; never commit audio. diff --git a/README.md b/README.md index 7e1c249..8a5786a 100644 --- a/README.md +++ b/README.md @@ -8,20 +8,29 @@ Built as a personal project to get better at speaking *and* to practice building architecture: local AI inference boundaries, audio handling, a real state machine, asynchronous job design, and honest error handling — not just a UI over a mock API. -## Status: Phase 1 +## Status: Phase 2A The full local practice workflow — track/topic/mode selection, timers, browser microphone recording, review, self-reflection, results, and history — is built and working, backed by SQLite -and local file storage. **The transcript, metrics, and feedback you see are simulated**, clearly -labeled as such throughout the UI. No real speech-to-text or speech analysis has been built yet — -see [docs/SCORING_AND_LIMITATIONS.md](docs/SCORING_AND_LIMITATIONS.md). +and local file storage. -See [docs/PROJECT_STATUS.md](docs/PROJECT_STATUS.md) for exact test results, known limitations, -and what's next. +**The transcript is now real**: local speech-to-text via a pinned whisper.cpp build, running +entirely on your machine with no network access. **Metrics and feedback are still simulated**, and +each section of the results screen is labeled with its own provenance rather than a single +page-wide disclaimer — see +[docs/SCORING_AND_LIMITATIONS.md](docs/SCORING_AND_LIMITATIONS.md). + +Transcription runs in a **separate worker process** reading a durable SQLite job queue, because it +is far too slow to hold an HTTP request open for. Uploads return `202 Accepted` and the frontend +polls for progress. + +See [docs/PROJECT_STATUS.md](docs/PROJECT_STATUS.md) for exact test results and known limitations +— including, honestly, that no whisper.cpp build or real-model run has yet happened in this +repository's build environment. ## Quick start -Requires Python 3.11+ and Node 20+. Two processes, both local, nothing cloud-hosted: +Requires Python 3.11+ and Node 20+. **Three** local processes, nothing cloud-hosted: ```bash git clone @@ -34,6 +43,12 @@ pip install -r requirements.txt uvicorn app.main:app --reload --host 127.0.0.1 --port 8000 ``` +```bash +# Transcription worker (separate terminal) +cd backend && source .venv/bin/activate +python -m app.worker +``` + ```bash # Frontend (separate terminal) cd frontend @@ -44,16 +59,38 @@ npm run dev Open the URL Vite prints (typically `http://127.0.0.1:5173`). See [docs/DEPLOYMENT.md](docs/DEPLOYMENT.md) for the production build and its limitations. +### Setting up transcription + +Transcription needs two things this repository deliberately does **not** ship: `ffmpeg`, and a +locally-built whisper.cpp runtime plus model weights. Neither is vendored (they are large, and +this is a public repo — see [docs/adr/0009-pinned-whisper-cpp-runtime.md](docs/adr/0009-pinned-whisper-cpp-runtime.md)). + +```bash +brew install cmake ffmpeg # your call -- the setup script never installs packages for you +scripts/setup_whisper.sh # clone pinned v1.9.1, build whisper-cli, download base.en +scripts/check_whisper_runtime.sh # read-only diagnosis of what's missing +``` + +The setup script builds into `backend/runtime/whisper/` and downloads models into +`backend/storage/models/` — both gitignored. If a dependency is missing it prints the exact +install command and exits rather than installing anything itself. + +Without this, recording and reflection still work; sessions will fail with a message naming the +missing piece and the command that fixes it, and your recordings are always preserved. + ## Architecture, in one paragraph A SvelteKit frontend talks to a FastAPI backend over a small JSON/multipart REST API (relative URLs only, proxied by Vite in dev — see [docs/ARCHITECTURE.md](docs/ARCHITECTURE.md)). The -backend owns SQLite (WAL mode, real migrations, no ORM) and local recording storage, and -processes each recording through three provider interfaces -(`TranscriptionProvider` / `MetricsProvider` / `FeedbackProvider`) — today backed entirely by -deterministic mocks, designed so a real whisper.cpp / real analytics / optional local-Ollama -implementation can be swapped in later without touching the rest of the app. See -[docs/AI_MODEL_STRATEGY.md](docs/AI_MODEL_STRATEGY.md). +backend owns SQLite (WAL mode, real migrations, no ORM) and local recording storage. Uploading a +recording stores it, queues a row in a `processing_jobs` table, and returns `202` immediately; a +separate worker process claims that job atomically, decodes the audio to 16 kHz mono PCM with +ffmpeg, runs a pinned local whisper.cpp binary, and writes a transcript with full model/runtime +provenance. Analysis still goes through three provider interfaces +(`TranscriptionProvider` / `MetricsProvider` / `FeedbackProvider`) — transcription is now real, +metrics and feedback remain deterministic mocks. See +[docs/AI_MODEL_STRATEGY.md](docs/AI_MODEL_STRATEGY.md), which also documents honestly where the +Phase 1 plan for this swap turned out to be wrong. ## Documentation map @@ -66,14 +103,14 @@ implementation can be swapped in later without touching the rest of the app. See | [docs/DATA_MODEL.md](docs/DATA_MODEL.md) | Schema, and the topic seed-vs-runtime split | | [docs/TOPIC_TAXONOMY.md](docs/TOPIC_TAXONOMY.md) | The 50-topic bank and selection logic | | [docs/SCORING_AND_LIMITATIONS.md](docs/SCORING_AND_LIMITATIONS.md) | What's simulated, honestly | -| [docs/AI_MODEL_STRATEGY.md](docs/AI_MODEL_STRATEGY.md) | Mock → real provider swap plan | +| [docs/AI_MODEL_STRATEGY.md](docs/AI_MODEL_STRATEGY.md) | Real transcription, and what the swap actually cost | | [docs/PRIVACY_AND_SECURITY.md](docs/PRIVACY_AND_SECURITY.md) | What stays local, upload validation, deletion | | [docs/TESTING.md](docs/TESTING.md) | Automated / mocked / integration / manual test tiers | | [docs/DEPLOYMENT.md](docs/DEPLOYMENT.md) | Running locally; why no Docker/cloud | -| [docs/ROADMAP.md](docs/ROADMAP.md) | Recommended Phase 2+ scope | +| [docs/ROADMAP.md](docs/ROADMAP.md) | What shipped in Phase 2A; recommended Phase 2B scope | | [docs/PROJECT_STATUS.md](docs/PROJECT_STATUS.md) | Exact current state, test results, known gaps | | [docs/LEARNING_NOTES.md](docs/LEARNING_NOTES.md) | Plain-language architecture walkthrough | -| [docs/adr/](docs/adr/) | Five architectural decision records | +| [docs/adr/](docs/adr/) | Nine architectural decision records | | [AGENTS.md](AGENTS.md) | Rules for any coding agent working in this repo | | [SECURITY.md](SECURITY.md) | How to report a security/privacy issue | @@ -87,20 +124,31 @@ upload validation does and doesn't guarantee, and how deletion works. ## Testing ```bash -cd backend && source .venv/bin/activate && pytest -q # 49 passed -cd frontend && npx vitest run # 26 passed +cd backend && source .venv/bin/activate && pytest -q # 133 passed, 2 skipped +cd frontend && npx vitest run # 60 passed cd frontend && npm run check && npm run lint # type-check + lint cd frontend && npm run build # production build +bash scripts/check_public_safety.sh # pre-publication safety audit ``` +The 2 skips are the opt-in real-whisper.cpp integration test, which runs only when a built +runtime, a model, ffmpeg and your own sample recording are all genuinely present — and names the +specific missing prerequisite when they aren't. Every ffmpeg/whisper interaction in the regular +suite is mocked, so the suite runs on a machine with neither installed. + See [docs/TESTING.md](docs/TESTING.md) for what each tier actually covers, and [docs/PROJECT_STATUS.md](docs/PROJECT_STATUS.md) for exact current results. ## What I learned building this -Provider interfaces are worth their extra boilerplate the moment you can name the second -implementation that's coming — they turned the eventual whisper.cpp swap from "rewrite the app" -into "write one class." Honest status machines with explicit failure/cleanup paths (see +Provider interfaces are worth their boilerplate — but not for the reason I first wrote here. I +claimed they would turn the whisper.cpp swap into "write one class." They didn't: the Phase 1 +interface never received any audio, the result type had nowhere to record which model produced it, +and the call site had to leave the HTTP request entirely for a worker process. What the interface +genuinely bought was *containment* — everything that had to change was findable from one +definition. An abstraction designed only against a fake encodes the fake's assumptions, and +"we have an interface, so this will be easy" is a claim worth distrusting. Honest status machines +with explicit failure/cleanup paths (see [docs/DATA_FLOW.md](docs/DATA_FLOW.md)) catch real bugs that unit tests with idealized fixtures miss — a manual pass through the real UI against the real backend found a genuine MIME-type handling bug that 48 passing automated tests hadn't (see diff --git a/backend/app/safe_text.py b/backend/app/safe_text.py index 7055815..1c44ce7 100644 --- a/backend/app/safe_text.py +++ b/backend/app/safe_text.py @@ -19,9 +19,9 @@ MAX_DETAIL_CHARS = 400 -# POSIX absolute paths (/Users/..., /opt/..., /private/var/...) and Windows -# drive paths. Deliberately greedy about what counts as a path character so -# that a partially-quoted path doesn't leak its tail. +# Any POSIX absolute path (macOS home directories, /opt, /private/var, ...) +# and Windows drive paths. Deliberately greedy about what counts as a path +# character so that a partially-quoted path doesn't leak its tail. _ABSOLUTE_PATH = re.compile(r"(?:[A-Za-z]:)?(?:/|\\\\)[^\s'\"<>|]{2,}") # A bare `~` or `~user` home reference. diff --git a/backend/tests/test_audio_preprocessor.py b/backend/tests/test_audio_preprocessor.py index c1cfa2d..c0b29c0 100644 --- a/backend/tests/test_audio_preprocessor.py +++ b/backend/tests/test_audio_preprocessor.py @@ -73,11 +73,19 @@ def test_run_command_does_not_interpret_shell_metacharacters(): # --- Redaction --------------------------------------------------------------- +# A macOS-style home path is THE case this redaction exists for, but writing +# one as a literal here would trip scripts/check_public_safety.sh -- which +# cannot tell a synthetic home path from the owner's real one, and should not +# try. Assembling it at runtime keeps the test realistic and the safety check +# strict. +MACOS_HOME_PREFIX = "/" + "Users" + "/synthetic-account" + + def test_redact_paths_removes_absolute_paths_and_home_references(): - text = "Error opening /Users/someone/Projects/audio.webm and ~/other.wav" + text = f"Error opening {MACOS_HOME_PREFIX}/Projects/audio.webm and ~/other.wav" cleaned = redact_paths(text) - assert "/Users/" not in cleaned - assert "someone" not in cleaned + assert MACOS_HOME_PREFIX not in cleaned + assert "synthetic-account" not in cleaned assert "~" not in cleaned assert "" in cleaned @@ -92,9 +100,13 @@ def test_safe_detail_truncates_long_output(): def test_safe_command_redacts_every_path_argument(): rendered = safe_command( - ["/opt/whisper/bin/whisper-cli", "--model", "/Users/me/models/ggml-base.en.bin"] + [ + "/opt/whisper/bin/whisper-cli", + "--model", + f"{MACOS_HOME_PREFIX}/models/ggml-base.en.bin", + ] ) - assert "/Users" not in rendered + assert MACOS_HOME_PREFIX not in rendered assert "/opt" not in rendered @@ -103,10 +115,10 @@ def test_command_result_safe_stderr_is_redacted(): argv=("ffmpeg",), returncode=1, stdout="", - stderr="/Users/someone/recordings/clip.webm: Invalid data", + stderr=f"{MACOS_HOME_PREFIX}/recordings/clip.webm: Invalid data", duration_seconds=0.1, ) - assert "/Users" not in result.safe_stderr() + assert MACOS_HOME_PREFIX not in result.safe_stderr() assert "Invalid data" in result.safe_stderr() @@ -242,7 +254,7 @@ def test_ffmpeg_failure_never_leaks_its_stderr_to_the_caller(monkeypatch, tmp_pa """The exception message is what may reach the API. It must be clean.""" fake = _FakeRun( returncode=1, - stderr="/Users/someone/storage/recordings/clip.webm: Invalid data found", + stderr=f"{MACOS_HOME_PREFIX}/storage/recordings/clip.webm: Invalid data found", produce=False, ) preprocessor = _preprocessor(monkeypatch, fake) @@ -251,8 +263,8 @@ def test_ffmpeg_failure_never_leaks_its_stderr_to_the_caller(monkeypatch, tmp_pa with pytest.raises(AudioPreprocessingError) as exc: preprocessor.prepare(source, tmp_path / "work") message = str(exc.value) - assert "/Users" not in message - assert "someone" not in message + assert MACOS_HOME_PREFIX not in message + assert "synthetic-account" not in message assert "could not be decoded" in message diff --git a/docs/AI_MODEL_STRATEGY.md b/docs/AI_MODEL_STRATEGY.md index 9c8ff91..fea0e05 100644 --- a/docs/AI_MODEL_STRATEGY.md +++ b/docs/AI_MODEL_STRATEGY.md @@ -9,11 +9,64 @@ through three small interfaces in `backend/app/providers/`: - `MetricsProvider.analyze(...)` - `FeedbackProvider.generate(...)` -Phase 1 ships exactly one implementation of each — `MockTranscriptionProvider`, -`MockMetricsProvider`, `MockFeedbackProvider` — wired in via `app/dependencies.py`. Swapping any -one of them for a real implementation later means writing a new class against the same interface -and changing one line in `dependencies.py`; nothing upstream (routers, `SessionService`, the -frontend) needs to know or care. +## Correction: the Phase 1 "one-line swap" claim was wrong + +This document used to say that swapping a mock for a real implementation meant "writing a new +class against the same interface and changing one line in `dependencies.py`; nothing upstream +needs to know or care." **That was not true, and Phase 2A proved it.** It is recorded here rather +than quietly edited away, because the gap between the two is the actual lesson. + +What was right: the *boundary*. Having a `TranscriptionProvider` seam meant the change was +contained to a known set of files instead of scattered through routes and services. + +What was wrong: the *signature*, and the assumption that only the provider layer would move. +Three specific things had to change. + +**1. The interface received no audio.** Phase 1's signature was: + +```python +transcribe(session_id, topic_title, mode, duration_seconds) -> TranscriptResult +``` + +A mock can write a sentence from a topic title. A speech model cannot. The input is now a +`TranscriptionRequest` carrying the decoded audio path, the measured duration, the requested +language, and explicit model and runtime configuration. + +**2. The result had nowhere to put provenance.** `TranscriptResult` carried a provider name, a +version, some text, and `is_mock`. Real transcription also produces a detected language, +timestamped segments, the model identity and hash, the runtime version, the parameters used, and +timing. Without those, two transcripts made months apart with different models are +indistinguishable after the fact — which defeats the point of tracking practice over time. That +required a schema migration, not just a new class. + +**3. The call site could not stay where it was.** `SessionService` ran all three providers +synchronously inside the upload request. Real transcription is far too slow for that, so +processing had to move to a separate worker process with a durable job queue — see +[ADR 0006](adr/0006-local-worker-sqlite-queue.md). That is an architectural change, and no +interface design would have avoided it. + +**4. A decode step had to exist at all.** Browsers record Opus-in-WebM or AAC-in-MP4; whisper.cpp +reads neither. That is a new boundary of its own — see +[ADR 0007](adr/0007-ffmpeg-preprocessing-boundary.md). + +The honest summary: a provider interface localizes *where* a change lands. It does not make the +change small, and it cannot anticipate what a real implementation needs to be handed. + +## Current implementations + +| Provider | Implementation | Real? | +|---|---|---| +| `TranscriptionProvider` | `WhisperCppTranscriptionProvider` | **Real** — local whisper.cpp | +| `MetricsProvider` | `MockMetricsProvider` | Simulated | +| `FeedbackProvider` | `MockFeedbackProvider` | Simulated | + +`MockTranscriptionProvider` still exists and still implements the interface, for tests and for +deliberately running the simulated pipeline (`Settings.transcription_provider = "mock"`). It is +**never** an automatic fallback: if whisper.cpp is unavailable, the job fails with an actionable +error code rather than producing simulated text under a UI label that says "Real transcript". + +Providers are selected in the worker (`app/worker/runner.py`), not in `app/dependencies.py` — the +API process no longer wires up any AI provider at all, because it no longer runs one. ## Why deterministic metrics before an LLM judge @@ -23,25 +76,67 @@ sequenced deliberately: cheap, fast, locally-computable signal first (Phase 2), richer/slower LLM-based feedback layered on top of that signal later (Phase 3+), rather than the other way around. See `docs/adr/0005-deterministic-metrics-before-llm.md` for the full reasoning. -## Planned Phase 2 swap: local Whisper +## Delivered in Phase 2A: local Whisper + +`WhisperCppTranscriptionProvider` runs a pinned, locally-built `whisper-cli` binary +(**v1.9.1**, commit `f049fff95a08…`) against a locally-downloaded model, entirely offline. See +[ADR 0009](adr/0009-pinned-whisper-cpp-runtime.md) for version pinning and +[ADR 0006](adr/0006-local-worker-sqlite-queue.md) for the worker. + +Key implementation choices: + +- **Structured output, not screen-scraping.** The provider passes `--output-json` and parses the + file whisper.cpp writes. Parsing the human-readable console output would work until a release + changed its formatting or a progress line interleaved with the text. Malformed, missing, or + structurally wrong JSON is a **provider failure** — never an empty-but-successful transcript, + because "the model heard nothing" and "we could not read the model's output" must not collapse + into the same result. +- **Segment timestamps only.** whisper.cpp reports segment offsets reliably; word-level + timestamps are deliberately out of scope for this phase. +- **Model choice is configurable.** `base.en` is the default; `small.en` is supported. The data + model is not hard-coded to English-only model names — a transcript stores whatever model name + was actually used, and the requested language may be `None` for auto-detection. + +The Phase 1 decision to store `provider_name`/`provider_version`/`is_mock` on every row paid off +exactly as intended: historical mock transcripts remain clearly attributable and are never +compared against real ones as if equivalent. + +## Metal, Core ML, quantization, and VAD + +These get conflated. They are four different things: + +- **Metal** — Apple's GPU compute API. whisper.cpp compiles Metal support in by default on Apple + Silicon and uses it automatically. It changes *speed*, not results. **In use.** +- **Core ML** — an alternative execution path where the encoder is converted to a Core ML model + ahead of time to use the Neural Engine. Requires generating extra artifacts, and can change + numerical output. **Off in this phase.** +- **Quantization** — storing model weights at lower precision (e.g. `q5_0`). Smaller and faster, + with some accuracy cost that depends on the model and the audio. **Off in this phase.** +- **VAD** — voice activity detection, skipping silence before inference. Faster on sparse audio, + but it can clip quiet speech, and this app records people who pause a lot on purpose. **Off in + this phase.** + +All three of the disabled options are treated as **measured future optimizations**, not automatic +improvements. `backend/scripts/benchmark_models.py` exists so any future decision to enable one is +made against real numbers from the owner's own recordings and hardware. + +**No benchmark numbers are published in this repository**, because none have been measured — see +[PROJECT_STATUS.md](PROJECT_STATUS.md). -`MockTranscriptionProvider` → a `WhisperCppTranscriptionProvider` backed by whisper.cpp, running -fully locally (no cloud STT API). This is *the* headline Phase 2 item — see -[ROADMAP.md](ROADMAP.md) — and is exactly why: +## Next: deterministic real metrics (Phase 2B) -- Processing needs to move from synchronous (Phase 1) to a queued local worker (real - transcription of even a 2-minute clip is far too slow to hold an HTTP request open for). -- `provider_name`/`provider_version` are stored on every transcript/metrics/feedback row *now*, - in Phase 1, specifically so that once real transcripts start appearing, old mock rows remain - clearly attributable and never get compared against new real ones as if they were equivalent. +Real transcripts and an authoritative decoded duration now both exist, so the inputs +`MockMetricsProvider` was waiting for are finally available: -## Planned Phase 2+/3 swap: deterministic real metrics +- **Words per minute** — words in the real transcript ÷ `sessions.audio_duration_seconds`. +- **Filler words** — lexical matching against the real transcript. +- **Pauses** — derivable from the gaps between transcript segment timestamps, and more precisely + from silence analysis of the decoded PCM. -Once real transcripts exist, `MockMetricsProvider` can be replaced with a provider that computes -actual words-per-minute (words in the real transcript ÷ real audio duration), real filler-word -detection (simple lexical matching against the real transcript), and real pause detection (from -audio silence analysis). Still deterministic, still explainable — no LLM judgment involved at -this layer. +Still deterministic, still explainable, no LLM judgment at this layer — see +[ADR 0005](adr/0005-deterministic-metrics-before-llm.md). Until that lands, metrics and feedback +remain simulated and are labeled as such **per section** in the UI, independently of the now-real +transcript. ## Optional future layer: local LLM feedback (Ollama) diff --git a/docs/ARCHITECTURE.md b/docs/ARCHITECTURE.md index ccd1831..1b286b5 100644 --- a/docs/ARCHITECTURE.md +++ b/docs/ARCHITECTURE.md @@ -3,72 +3,120 @@ ## Overview ``` -┌────────────────────┐ relative /api/... ┌──────────────────────┐ -│ SvelteKit frontend │ ───────────────────────────▶ │ FastAPI backend │ -│ (Vite dev server, │ (Vite dev-server proxy in │ 127.0.0.1:8000 │ -│ 127.0.0.1:5173) │ dev — see "API routing") │ │ -└────────────────────┘ └──────────┬───────────┘ - │ - ┌──────────┴───────────┐ - │ SQLite (WAL mode) │ - │ backend/storage/ │ - │ speaklab.db │ - └──────────┬───────────┘ - │ - ┌──────────┴───────────┐ - │ Local filesystem │ - │ backend/storage/ │ - │ recordings/*.webm │ - └──────────────────────┘ +┌─────────────────────┐ relative /api/... ┌──────────────────────┐ +│ SvelteKit frontend │ ──────────────────────────▶ │ FastAPI backend │ +│ (Vite dev server, │ (Vite dev-server proxy in │ 127.0.0.1:8000 │ +│ 127.0.0.1:5173) │ dev — see "API routing") │ │ +│ │ ◀── poll /status (1s→5s) ── │ stores upload, │ +└─────────────────────┘ │ queues a job, 202 │ + └──────────┬───────────┘ + │ + ┌────────────────┴────────────────┐ + │ SQLite (WAL mode) │ + │ backend/storage/speaklab.db │ + │ ── also the job queue ── │ + │ processing_jobs │ + └────────────────┬────────────────┘ + │ BEGIN IMMEDIATE + │ claim one job + ┌────────────────┴────────────────┐ + │ Worker process │ + │ python -m app.worker │ + └────────────────┬────────────────┘ + │ + ┌──────────────────────────────────────┼────────────────────────┐ + │ │ │ + ┌──────────┴──────────┐ ┌──────────────┴──────────┐ ┌─────────┴─────────┐ + │ ffmpeg (external) │ │ whisper-cli (external) │ │ Local filesystem │ + │ → 16 kHz mono PCM │ │ pinned v1.9.1, local │ │ storage/recordings│ + │ storage/processing │ │ storage/models/*.bin │ │ storage/models │ + │ (per-job, deleted) │ │ runtime/whisper/bin │ │ runtime/whisper │ + └─────────────────────┘ └─────────────────────────┘ └───────────────────┘ ``` -Two independent local processes: a SvelteKit app and a FastAPI app. Both are plain, ordinary -local toolchains — no Docker (see "Why no Docker" below). +**Three** local processes now: the SvelteKit app, the FastAPI app, and the transcription worker. +All are plain local toolchains — no Docker (see "Why no Docker" below). The worker additionally +shells out to two external binaries the owner installs/builds themselves: `ffmpeg` and +`whisper-cli`. Neither is vendored into this repository — see +[ADR 0009](adr/0009-pinned-whisper-cpp-runtime.md). ## Backend layers ``` -routers/ FastAPI route handlers — HTTP concerns only (status codes, request/response +routers/ FastAPI route handlers — HTTP concerns only (status codes, request/response models). No business logic. -services/ session_service.py — the ONLY place session status transitions happen and the - only place that coordinates SQLite + filesystem writes across a single logical - operation (e.g. "attach a recording"). +services/ session_service.py — request-side: session status transitions, coordinating + SQLite + filesystem writes, and queueing work. + transcription_job_service.py — worker-side: what happens to one claimed job. + runtime_service.py — non-sensitive local capability reporting. repositories/ Plain CRUD against SQLite (SqliteTopicRepository, SqliteSessionRepository, - SettingsRepository) or the filesystem (LocalRecordingStorage). No business rules. -providers/ TranscriptionProvider / MetricsProvider / FeedbackProvider interfaces + the - Phase 1 Mock* implementations. See AI_MODEL_STRATEGY.md. + SqliteJobRepository, SettingsRepository) or the filesystem + (LocalRecordingStorage). No business rules. +providers/ TranscriptionProvider / MetricsProvider / FeedbackProvider interfaces, the + Mock* implementations, and WhisperCppTranscriptionProvider. + See AI_MODEL_STRATEGY.md. +audio/ AudioPreprocessor / FfmpegAudioPreprocessor — the decode boundary (ADR 0007). +worker/ The transcription worker process (`python -m app.worker`). ADR 0006. +subprocess_util.py The ONLY place this app starts an external process: argv arrays, mandatory + timeouts, no shell. +safe_text.py Path/username redaction for anything derived from a subprocess or exception. schemas.py Pydantic request/response models, enums, constrained fields. db.py SQLite connection factory (pragmas) + the migration runner. config.py Settings dataclass — no global singleton, so tests can build an isolated instance pointed at a tmp_path. ``` +Note that `SessionService` and `TranscriptionJobService` are split along the process boundary, +not arbitrarily: the first runs inside a request and must return fast, the second runs inside the +worker and may take minutes. Both write session status, but each owns a different part of the +state machine. + This layering exists so that swapping a provider, a repository backend, or the DB engine never requires touching the router layer — see the five ADRs in `docs/adr/` for the reasoning behind each of those boundaries. -## Processing is synchronous in Phase 1 +## Processing is asynchronous since Phase 2A + +Phase 1 ran all three mock providers **inside** the `POST /api/sessions/{id}/recording` request. +That was correct while they were pure, instant computation — there was nothing to queue. + +Real transcription changed that. `POST /api/sessions/{id}/recording` now: -When a recording is attached to a session (`POST /api/sessions/{id}/recording`), the mock -transcription, metrics, and feedback providers run **in the same request**, synchronously, before -the response is returned. There is no job queue and no background worker in Phase 1. +1. Stores the recording (streamed, validated, exactly as before). +2. Transitions `CREATED → RECORDING_STORED → PROCESSING`. +3. Inserts a `PENDING` row in `processing_jobs`. +4. Returns **`202 Accepted`** with the session and its processing state. -This is a deliberate simplification, not an oversight: the mock providers are pure, fast, local -computation (see `app/providers/`), so there is nothing to queue. Phase 2, when real -whisper.cpp transcription is wired in, will need a queued local worker instead, because real -transcription is slow enough that holding an HTTP request open for it would be a bad user -experience. See [ROADMAP.md](ROADMAP.md) and [AI_MODEL_STRATEGY.md](AI_MODEL_STRATEGY.md). +No audio is decoded and no model is run in the request path. The worker +(`python -m app.worker`) claims the job and does the work. See +[ADR 0006](adr/0006-local-worker-sqlite-queue.md) for why a separate process and a SQLite queue, +rather than an in-process background task or a message broker. + +**The job queue is SQLite.** Claiming is atomic via `BEGIN IMMEDIATE` plus a +`WHERE status = 'PENDING'` guard on the `UPDATE`, and the transaction is committed *before* any +audio is touched — holding the write lock across an inference run would block every other write +in the app. ## The session state machine ``` + ┌── API process ──┐ ┌──── worker process ────┐ + CREATED ──▶ RECORDING_STORED ──▶ PROCESSING ──┬──▶ COMPLETED - └──▶ FAILED + └──▶ FAILED ``` -Only `SessionService` may call `SessionRepository.set_status`; every transition is written to -`session_status_events` for auditability and is exactly what the "status transition" tests -assert against. See [DATA_FLOW.md](DATA_FLOW.md) for the full success and failure sequences. +The states are unchanged from Phase 1 — what changed is **who** drives them. `SessionService` +(in the request) owns everything up to `PROCESSING`; `TranscriptionJobService` (in the worker) +owns the transition out of it. Every transition is still written to `session_status_events`. + +`PROCESSING → COMPLETED` happens **only after** the transcript transaction has committed. If the +worker dies in between, the session stays `PROCESSING`, the stale-job sweeper requeues it, and +the retry clears any partial analysis output before re-inserting. Losing work is recoverable; +a `COMPLETED` session with no transcript is not. + +A job that fails retryably goes back to `PENDING` while the session *stays* `PROCESSING` — the +user is still waiting, and telling them it failed when it will be retried would be wrong. ## SQLite and the filesystem are not one transaction @@ -110,11 +158,25 @@ src/lib/api/client.ts Single point of contact with the backend (relative src/lib/stores/practiceSession.ts In-memory store for the in-flight, unsaved session. src/lib/timer.ts Monotonic countdown factory (performance.now()-based). src/lib/media/recorder.ts The only code that touches getUserMedia/MediaRecorder. +src/lib/processing/poller.ts Status polling with backoff, cancellation, and a timeout. + Clock/timers/fetch injected, so it is fully unit-testable. +src/lib/processing/presentation.ts Pure functions: stage labels, per-error-code setup + guidance, and per-section result provenance. src/lib/components/SessionResults.svelte Shared transcript/metrics/feedback/reflection view, used by both the results screen and the history detail screen. +src/lib/components/ProcessingStatus.svelte In-progress / failed / worker-not-running states. src/routes/ One directory per screen; see the routing table in PRD.md. ``` +The frontend learns about completion by **polling** `GET /api/sessions/{id}/status`, not over a +WebSocket or SSE — see [ADR 0008](adr/0008-http-polling-not-websockets.md). The poller stops on +a terminal status, a deleted session, a timeout, or route teardown, and cannot be started twice. + +Result provenance is now **per section**: the transcript is real, metrics and feedback are still +simulated, and each label is read from that row's own `is_mock` flag. There is deliberately no +page-level "everything is simulated" banner any more, because for a new session it would be +false — while a historical Phase 1 session still shows "Simulated" on all three. + `practiceSession` lives only in memory — a page refresh loses an unsaved in-progress session by design (see [LEARNING_NOTES.md](LEARNING_NOTES.md) for why IndexedDB persistence was deliberately not added yet). Every route past `/practice/setup` checks this store on mount and redirects back diff --git a/docs/DATA_FLOW.md b/docs/DATA_FLOW.md index 5e7eddc..155cd8d 100644 --- a/docs/DATA_FLOW.md +++ b/docs/DATA_FLOW.md @@ -1,9 +1,10 @@ # Data Flow This traces how a request actually moves through the system for the two operations with real -complexity: attaching a recording (upload + mock analysis) and deleting a session. Both are -implemented in `backend/app/services/session_service.py`; the code there is the source of truth -if this doc and the code ever disagree. +complexity: attaching a recording (upload + queued transcription) and deleting a session. The +request side is `backend/app/services/session_service.py` and the worker side is +`backend/app/services/transcription_job_service.py`; the code there is the source of truth if +this doc and the code ever disagree. ## Recording a practice attempt — success path @@ -20,7 +21,7 @@ if this doc and the code ever disagree. within configured bounds. On success, inserts a `sessions` row with status `CREATED` and logs the `None → CREATED` event. - `POST /api/sessions/{id}/recording` (multipart: the audio file + client-reported duration). -4. **Backend, `SessionService.attach_recording_and_process`** (the synchronous pipeline): +4. **Backend, `SessionService.attach_recording_and_enqueue`** (returns promptly): 1. Confirm the session exists and is in status `CREATED`; confirm the reported duration is in range. 2. `RecordingStorage.save()` streams the upload to disk in chunks, checking the running byte @@ -29,13 +30,41 @@ if this doc and the code ever disagree. 3. `SessionRepository.attach_recording()` inserts the `recordings` row and stamps `client_reported_duration_seconds` on the session, in one SQLite transaction. 4. Transition `CREATED → RECORDING_STORED`, then `RECORDING_STORED → PROCESSING` (each logged). - 5. Run the three mock providers (transcription, metrics, feedback — see - [AI_MODEL_STRATEGY.md](AI_MODEL_STRATEGY.md)), save their output, transition - `PROCESSING → COMPLETED`. - 6. Return the updated session. -5. **Frontend, `/practice/reflect`**: `POST /api/sessions/{id}/reflection` (optional). -6. **Frontend, `/practice/results/{id}`**: `GET /api/sessions/{id}` returns transcript/metrics/ - feedback because status is `COMPLETED`. + 5. Insert a `PENDING` row in `processing_jobs`. + 6. Return **`202 Accepted`** — session plus processing state. **No model runs here.** + + The job row is created **last**, after the session is already `PROCESSING`. A job that exists + always points at a session genuinely awaiting work; the reverse gap (`PROCESSING` with no job + yet, if the insert fails) is visible and recoverable, whereas a job pointing at a `CREATED` + session would be a worker crash waiting to happen. + +5. **Worker, `TranscriptionJobService.process`** (a separate process — see + [ADR 0006](adr/0006-local-worker-sqlite-queue.md)): + 1. `SqliteJobRepository.claim_next()` takes the oldest `PENDING` job inside a + `BEGIN IMMEDIATE` transaction, sets it `RUNNING`, and increments `attempt_count`. **The + transaction commits before any audio is touched.** + 2. Stage `PREPARING_AUDIO`: `FfmpegAudioPreprocessor.prepare()` decodes the original recording + into 16 kHz mono PCM inside `storage/processing//`, and measures the authoritative + duration from the decoded stream. `sessions.audio_duration_seconds` is stamped; + `client_reported_duration_seconds` is left untouched. + 3. Stage `TRANSCRIBING`: `WhisperCppTranscriptionProvider.transcribe()` runs the pinned + `whisper-cli` against the decoded PCM and parses its JSON output. + 4. The per-job temp directory is deleted — on this path and every failure path. + 5. Stage `ANALYZING`: the still-mock metrics and feedback providers run, now against the + *real* decoded duration. + 6. Stage `SAVING`: any partial analysis output is cleared, then transcript (+ segments), + metrics, and feedback are written. The transcript and its segments go in **one** + transaction. + 7. Only then: transition `PROCESSING → COMPLETED` and mark the job `SUCCEEDED`. + +6. **Frontend, `/practice/reflect`**: `POST /api/sessions/{id}/reflection` (optional). This is + reachable **while transcription is still running** — the whole point of the screen is + reflecting while the attempt is fresh, so it never waits on a model. The page polls and + displays the current stage without blocking the form. +7. **Frontend, `/practice/results/{id}`**: polls `GET /api/sessions/{id}/status` from ~1s with + backoff to 5s (see [ADR 0008](adr/0008-http-polling-not-websockets.md)), then fetches + `GET /api/sessions/{id}` once the session is terminal. Transcript/metrics/feedback are + returned only when status is `COMPLETED`. ## Recording a practice attempt — failure paths @@ -48,13 +77,48 @@ service deletes the just-written file (`RecordingStorage.delete`) before re-rais file is left on disk. The session again stays in `CREATED`. (Tested: `test_orphan_file_is_removed_when_recording_metadata_insert_fails`.) -**A mock provider raises** (Phase 1: shouldn't happen since they're pure and deterministic, but -handled anyway — this is exactly the seam Phase 2's real Whisper/analytics call will use): the -already-stored recording is **kept**, the session transitions `PROCESSING → FAILED` with a -human-readable `failure_reason`, and `GET /api/sessions/{id}` will **not** include transcript/ -metrics/feedback for a `FAILED` session, even if some of those rows partially exist from a -partway failure — `SessionService.get_session_detail` gates all three on `status == COMPLETED`. -(Tested: `test_provider_failure_marks_session_failed_and_preserves_the_recording`.) +**The job row can't be written** (a DB error after the recording was stored): the recording is +**kept**, and the session transitions `PROCESSING → FAILED` rather than sitting in `PROCESSING` +forever waiting for work that was never queued. +(Tested: `test_failure_to_queue_marks_the_session_failed_and_keeps_the_recording`.) + +**A worker-side step fails.** Every failure is sorted into one of two buckets, because guessing +wrong is expensive in both directions: + +| Bucket | Examples | What happens | +|---|---|---| +| **Retryable** | `WHISPER_TIMEOUT`, `FFMPEG_TIMEOUT`, `WHISPER_FAILED`, `WHISPER_OUTPUT_MALFORMED` | Job → `PENDING`, `attempt_count` already consumed. Session **stays** `PROCESSING`. | +| **Terminal** | `FFMPEG_NOT_FOUND`, `WHISPER_BINARY_MISSING`, `MODEL_MISSING`, `AUDIO_DECODE_FAILED`, `RECORDING_FILE_MISSING` | Job → `FAILED`. Session → `FAILED`. | + +Retrying a missing model file forever would burn CPU and, worse, bury the one instruction the +owner actually needs to read. Exhausting the retry budget also ends terminally. + +On **any** terminal failure the already-stored recording is **kept** — it is the irreplaceable +artifact and everything else can be recomputed from it — and `GET /api/sessions/{id}` still +returns no transcript/metrics/feedback for a `FAILED` session, because +`SessionService.get_session_detail` gates all three on `status == COMPLETED`. +(Tested: `test_terminal_failure_fails_the_session_and_preserves_the_recording`, +`test_retryable_failure_requeues_the_job_and_leaves_the_session_processing`, +`test_retry_budget_is_finite`.) + +**The worker is killed mid-job.** Its row is left `RUNNING` with nobody working on it, and +nothing else would ever move it. On startup and periodically, `recover_stale()` sweeps `RUNNING` +jobs older than `job_stale_after_seconds` and either requeues them (attempts remaining) or fails +them terminally. The timeout must exceed the slowest plausible transcription, or a long-but-healthy +job would be stolen from the worker still running it. +(Tested: `test_a_stale_running_job_is_requeued`, `test_a_healthy_running_job_is_not_stolen`, +`test_worker_recovers_stale_jobs_on_startup`.) + +**The worker was never started.** Nothing fails — the job simply sits `PENDING`. The frontend's +poll times out after 10 minutes and tells the owner to start the worker; +`GET /api/runtime/capabilities` reports which prerequisite is missing. The recording is safe and +the job is picked up whenever the worker does run. + +**Temporary decoded audio is always cleaned up**, on success and on every failure path. It is the +user's speech in the clear, so leaving it behind after a crash would quietly turn a temp file +into retained personal data. +(Tested: `test_temporary_processing_files_are_removed_after_success`, +`test_temporary_processing_files_are_removed_after_failure`.) ## Deleting a session diff --git a/docs/DATA_MODEL.md b/docs/DATA_MODEL.md index 64a6b61..75c89a2 100644 --- a/docs/DATA_MODEL.md +++ b/docs/DATA_MODEL.md @@ -1,8 +1,9 @@ # Data Model -Schema source of truth: `backend/app/migrations/0001_init.sql`. This doc explains the shape and, -most importantly, the seed-vs-runtime distinction for topics — read that section before touching -`topics.seed.json`. +Schema source of truth: the numbered files in `backend/app/migrations/` — +`0001_init.sql` (Phase 1) and `0002_processing_jobs_and_real_transcripts.sql` (Phase 2A). This +doc explains the shape and, most importantly, the seed-vs-runtime distinction for topics — read +that section before touching `topics.seed.json`. ## Topics: seed file vs. runtime source of truth @@ -47,22 +48,80 @@ is edited later — only `topic_id` is a live reference. - **`reflections`** — one row per session (upserted, not appended), `self_rating` constrained to 1–5 or `NULL`. - **`transcripts` / `metrics` / `feedback`** — one row per session each, all three carrying - `provider_name`, `provider_version`, and `is_mock`. This is deliberate: when Phase 2 swaps in a - real whisper.cpp transcription provider, historical rows stay correctly attributed to - `mock-transcription v0.1.0` instead of silently looking like they came from the same source as - new real-provider rows. See [AI_MODEL_STRATEGY.md](AI_MODEL_STRATEGY.md). -- **`settings`** — a generic key/value table. Phase 1's only user is the - `audio_retention_days` placeholder (see [PRIVACY_AND_SECURITY.md](PRIVACY_AND_SECURITY.md) — - the value can be read/written via the API, but nothing enforces it yet). - -## Why `client_reported_duration_seconds`, not `duration_seconds` - -The name is deliberately explicit: this number comes from the browser's own timer -(`performance.now()`-based, see `src/lib/media/recorder.ts` / `record/+page.svelte`), reported by -the client, not measured by decoding the actual audio file server-side. It's approximate metadata -useful for the mock pipeline and for display — not an authoritative duration. Phase 2, once real -audio decoding exists (alongside real transcription), should derive an authoritative duration -from the media itself and can add that as a separate column rather than overloading this one. + `provider_name`, `provider_version`, and `is_mock`. This paid off exactly as intended in Phase + 2A: real whisper.cpp transcripts now sit alongside historical `mock-transcription v0.1.0` rows, + and the UI labels each from its own `is_mock` flag rather than assuming anything page-wide. + `transcripts` additionally carries the Phase 2A provenance columns described below. +- **`transcript_segments`** — ordered, timestamped segments of a transcript (Phase 2A). Foreign + key to `transcripts` with `ON DELETE CASCADE`, `UNIQUE (transcript_id, segment_index)`, and + `CHECK` constraints enforcing `segment_index >= 0`, `start_ms >= 0`, `end_ms >= start_ms`. + Word-level timestamps are deliberately **not** stored — segment offsets are what whisper.cpp + reports reliably, and they are sufficient for everything the UI shows. +- **`processing_jobs`** — the durable job queue (Phase 2A). See + [ADR 0006](adr/0006-local-worker-sqlite-queue.md). One row per unit of work, carrying `status` + (`PENDING`/`RUNNING`/`SUCCEEDED`/`FAILED`), `stage`, `attempt_count`/`max_attempts`, a safe + `error_code`/`error_message` pair, `worker_id`, provenance, and created/started/updated/completed + timestamps. A **partial unique index** enforces one active job per session: + + ```sql + CREATE UNIQUE INDEX idx_jobs_one_active_per_session + ON processing_jobs (session_id, job_type) + WHERE status IN ('PENDING', 'RUNNING'); + ``` + + Enforcing this in the database rather than with a check-then-insert in Python is deliberate: + two concurrent uploads could both pass an application-level check. Terminal jobs are excluded + from the index, so a session can accumulate a history of attempts while never having two in + flight. +- **`settings`** — a generic key/value table. Its only user is the `audio_retention_days` + placeholder (see [PRIVACY_AND_SECURITY.md](PRIVACY_AND_SECURITY.md) — the value can be + read/written via the API, but nothing enforces it yet). + +## Two durations, deliberately + +Phase 1 named its column `client_reported_duration_seconds` precisely because it came from the +browser's own timer (`performance.now()`-based, see `src/lib/media/recorder.ts`), not from +decoding the audio — and predicted that Phase 2 would add an authoritative one **as a separate +column rather than overloading this one**. That is exactly what happened: + +| Column | Source | Authoritative? | +|---|---|---| +| `sessions.client_reported_duration_seconds` | The browser's stopwatch | No — diagnostic only | +| `sessions.audio_duration_seconds` | Frame count ÷ sample rate of the decoded PCM | **Yes** | + +Keeping both is the point. If the two ever diverge meaningfully, that is a real signal (a browser +timer bug, a truncated upload, a partial recording) — and it stays visible instead of being +overwritten by whichever value was written last. `audio_duration_seconds` is `NULL` until the +worker has decoded the recording. + +## Transcript provenance columns + +A transcript must be able to answer *"what produced this text, with which weights, under which +parameters, and how long did it take"* — otherwise two transcripts made months apart with +different models look comparable when they are not. See +[ADR 0009](adr/0009-pinned-whisper-cpp-runtime.md). + +- `detected_language` — what the model reported, not what was requested. +- `runtime_version` — e.g. `whisper.cpp v1.9.1 (f049fff95a08)`, from the runtime manifest. +- `model_name` — e.g. `base.en`. +- `model_sha256` — the hash of the actual weights. The name alone is not enough: upstream has + re-uploaded `ggml-base.en.bin`, so only the bytes truly pin it. +- `parameters_json` — the inference parameters (language, threads, and explicitly + `core_ml`/`quantized`/`vad` all false in this phase). +- `audio_duration_ms`, `processing_duration_ms`, `real_time_factor` — how long the audio was, how + long inference took, and their ratio. + +All of these are nullable, and all are `NULL` on Phase 1 mock rows. That is correct: a simulated +transcript genuinely has no model or runtime, and populating those fields would make a mock row +indistinguishable from a real one a year from now. + +## Migrations apply to a populated database + +`0002` adds only nullable/defaulted columns and new tables, so it applies cleanly on top of an +existing Phase 1 database without rewriting a row. `tests/test_migrations.py` verifies this by +applying `0001`, inserting Phase 1 shaped rows, and only *then* applying `0002` — testing +migrations against a fresh database only is how a migration that passes in CI destroys the one +database that matters. ## Connection configuration diff --git a/docs/LEARNING_NOTES.md b/docs/LEARNING_NOTES.md index ca78666..a80d0bd 100644 --- a/docs/LEARNING_NOTES.md +++ b/docs/LEARNING_NOTES.md @@ -12,11 +12,16 @@ It only ever calls the backend's HTTP API. **In the Python backend (FastAPI):** everything that needs to persist data or run logic the browser shouldn't be trusted with — validating that a topic/mode/duration combination is legal, -writing to SQLite, writing audio files to disk, running the (currently fake) analysis pipeline, -and deciding whether a session is allowed to move from one status to the next. +writing to SQLite, writing audio files to disk, and deciding whether a session is allowed to move +from one status to the next. It no longer runs any AI model itself. -**Nowhere (yet):** any real AI model. See "why real AI processing is intentionally postponed" -below. +**In the worker (`python -m app.worker`):** a third process, started separately, that does the +slow work — converting audio and running the speech model. It's the only place a model runs. + +**On your machine, as external programs:** `ffmpeg` (converts the audio) and `whisper-cli` (the +speech model runtime). You install/build these yourself; they aren't bundled here. + +**Nowhere:** any cloud service. Your audio never leaves the machine. ## How the frontend calls the backend @@ -37,9 +42,13 @@ including its limits, in [ARCHITECTURE.md](ARCHITECTURE.md#api-routing-dev-proxy 3. The backend reads that upload in small chunks (not all at once), checks it looks like real audio and isn't too big, and writes it to a file on disk with a random generated name — never the name your browser gave it. -4. The backend then runs the (currently fake) transcription/metrics/feedback "pipeline" against - that recording and saves the results. -5. Later, when you play the recording back, the browser asks the backend for it by session ID; the +4. The backend writes a "this needs transcribing" row into the database and **replies + immediately** — it does not wait. You move straight on to the reflection screen. +5. A separate program (the worker) picks up that row, converts the audio into the raw format the + speech model can read, runs the model, saves the transcript, and marks the session complete. +6. Meanwhile the browser asks "is it done yet?" every second or so, and shows you which step + it's on. +7. Later, when you play the recording back, the browser asks the backend for it by session ID; the backend looks up the real filename and streams the file back. Full detail, including what happens when something goes wrong partway through, is in @@ -53,19 +62,250 @@ of database: real SQL, real transactions, real indexes, zero operational overhea you'd outgrow it is real concurrent multi-user write load, which a single-user local tool structurally never has. See `docs/adr/0003-sqlite.md`. -## Why provider interfaces are being used +## Why provider interfaces are being used — and what that prediction got wrong Look at `backend/app/providers/` — three small Python classes with names like `TranscriptionProvider`. The app never calls "the mock transcription function" directly from a -route; it calls "whatever `TranscriptionProvider` is currently configured," and today that happens -to be the mock one. This costs a little extra code now (an abstract interface instead of just a -function) in exchange for a real benefit later: swapping in real Whisper transcription in Phase 2 -should mean writing one new class and changing one line of configuration, not hunting through the -codebase for every place that assumed "transcription = the mock." See -[AI_MODEL_STRATEGY.md](AI_MODEL_STRATEGY.md). +route; it calls "whatever `TranscriptionProvider` is currently configured." + +During Phase 1 this note predicted that swapping in real Whisper "should mean writing one new +class and changing one line of configuration." **That turned out to be wrong, and the way it was +wrong is more instructive than the interface itself.** + +What the interface genuinely delivered: the change was **contained**. There was no hunt through +routes and services for places that assumed "transcription = the mock." Everything that needed to +change was findable from the interface definition. + +What it did not deliver: **smallness**. The Phase 1 method signature was +`transcribe(session_id, topic_title, mode, duration_seconds)` — it never received the audio, +because a mock doesn't need audio to write a sentence about a topic. A real model does. The +return type had nowhere to record which model produced the text. And the *call site* had to move +out of the HTTP request entirely. + +The lesson worth keeping: **an interface localizes where a change lands; it cannot make the change +small, and it cannot anticipate what a real implementation needs to be handed.** An abstraction +designed only against a fake implementation encodes the fake's assumptions. That is not a reason +to skip the interface — the containment was real and valuable — but "we have an interface, so the +swap will be trivial" is a claim worth being suspicious of. See +[AI_MODEL_STRATEGY.md](AI_MODEL_STRATEGY.md) for the full correction. + +## Phase 2A: what real transcription actually taught + +This section is the useful part of Phase 2A. The code is the easy bit; these are the ideas. + +### Why transcription cannot run inside the HTTP request + +In Phase 1 the "analysis" ran inside the upload request, and that was fine — the mock providers +were arithmetic, finishing in microseconds. Real transcription is different in kind, not degree. +Transcribing a two-minute clip takes tens of seconds even on a fast machine. + +Three separate things break if you keep that in the request: + +1. **Timeouts.** Browsers, proxies, and HTTP clients all have default timeouts measured in tens of + seconds. Holding a connection open for a minute means it dies somewhere you don't control. +2. **Lost work with no record of it.** This is the one that actually matters. If the request dies + mid-inference, the recording is on disk, the session says `PROCESSING`, and *nothing anywhere + knows transcription is still owed*. Nobody will ever retry it. The session is stuck forever. +3. **A blocked user.** The next thing SpeakLab wants you to do after recording is write your + reflection while it's fresh. Making you stare at a spinner first defeats the point of the + screen. + +The fix for (2) is the real insight: **make the outstanding work a row in a table.** Once "this +session needs transcribing" is committed data instead of a Python call stack, it survives a +crashed request, a restarted API, and a rebooted machine. Everything else — the worker, retries, +recovery — follows from that one move. + +### A worker is not an AI agent + +These get confused constantly, so, plainly: `python -m app.worker` is a **loop that reads a +database table**. It does this forever: + +``` +claim the oldest pending row → convert the audio → run a program → write the result → repeat +``` + +It makes no decisions. It has no goals. It cannot choose what to do next, cannot decide to do +something different, and cannot call tools it wasn't explicitly programmed to call. Given the same +row it does exactly the same thing every time. It is the same category of thing as a print spooler +or a cron job. + +An **agent** is a system where a language model decides what happens next — it's given a goal and +a set of tools, and it chooses which tools to use, in what order, and when to stop. That is a +fundamentally different (and much less predictable) architecture. + +SpeakLab's worker happens to *run* an AI model, the way a print spooler happens to run a printer +driver. Running a model is not the same as being an agent. [AGENTS.md](../AGENTS.md) rules out +agents; this worker does not violate that, and understanding why is the point. + +### How SQLite works as a lightweight durable queue + +A queue needs three things. SQLite provides all three, so no broker was added: + +| A queue needs | SQLite gives us | +|---|---| +| Durable storage of pending work | Committed rows survive a crash or reboot | +| Handing each item to exactly one consumer | A serialized write lock (`BEGIN IMMEDIATE`) | +| Recovery when a consumer dies | A `started_at` timestamp and a sweep query | + +That's it. `processing_jobs` is the queue. "Enqueue" is `INSERT`. "Dequeue" is a transaction that +selects and updates. Retries are an integer column. Dead-lettering is `status = 'FAILED'`. + +The temptation is to reach for Redis or RabbitMQ because those are "what queues are". But those +tools exist to solve **throughput and distribution** — many producers, many consumers, across many +machines. A single-user local app has one of each, on one machine. Adding a broker would mean a +service to install, run, monitor, and keep alive, in exchange for solving a problem that doesn't +exist here. + +### What atomic job claiming means, and the bug it prevents + +Here is the obvious, wrong way to take a job: + +```python +job = SELECT * FROM processing_jobs WHERE status = 'PENDING' LIMIT 1 # (1) +UPDATE processing_jobs SET status = 'RUNNING' WHERE id = job.id # (2) +``` + +Between (1) and (2) there is a gap. If a second worker runs (1) inside that gap, **both workers +read the same row**, both mark it running, and both transcribe the same recording — burning double +the CPU and racing to write two transcripts for one session. + +"Atomic" means the read and the write happen as one indivisible step that nobody can interleave +with. Two mechanisms are used here, deliberately overlapping: + +```sql +BEGIN IMMEDIATE; -- take the write lock NOW, not on first write +SELECT ... WHERE status = 'PENDING' LIMIT 1; +UPDATE ... SET status = 'RUNNING' + WHERE id = ? AND status = 'PENDING'; -- re-assert the precondition +COMMIT; +``` + +`BEGIN IMMEDIATE` (rather than a plain `BEGIN`) grabs SQLite's write lock at the start, so a +second worker *blocks* rather than reading. The `AND status = 'PENDING'` on the `UPDATE` is an +independent backstop: if a worker somehow got through anyway, its update matches zero rows, it +sees `rowcount == 0`, and it claims nothing. Either one alone would probably be enough. Both +together mean the guarantee doesn't rest on getting one subtle thing exactly right. + +The other half is **releasing the lock immediately**. The transaction covers only the claim, then +commits. Holding it across a minute of inference would block every other write in the entire +application — the queue would be correct and the app would be unusable. + +### Why browser audio has to be decoded before inference + +Your browser doesn't record "audio". It records **Opus compressed inside a WebM container** (Chrome, +Firefox) or **AAC inside MP4** (Safari). whisper.cpp reads neither. It wants the rawest possible +thing: **16,000 samples per second, one channel, each sample a 16-bit number.** No compression, no +container. + +Two different layers are involved and it's worth separating them: + +- A **container** (WebM, MP4, OGG) is the box. It holds streams, timestamps, and metadata. +- A **codec** (Opus, AAC, MP3) is how the audio inside was compressed. + +Decoding means opening the box and reversing the compression to get actual sample values back. +ffmpeg does both, plus resampling to 16 kHz and downmixing to mono. + +Why 16 kHz specifically? Whisper was *trained* on 16 kHz mono audio. Feeding it 44.1 kHz stereo +doesn't error — it produces confidently wrong output, because the model interprets the samples at +the rate it expects. That's exactly why the preprocessor **fails loudly** if ffmpeg's output isn't +16 kHz mono 16-bit, rather than passing it along and hoping. + +### Model weights vs. inference runtime + +Two completely separate things, and conflating them causes real confusion: + +- **The inference runtime** (`whisper-cli`, built from whisper.cpp) is a *program*. It knows the + shape of the Whisper architecture and how to do the matrix maths. It's a few megabytes of + compiled C++. It contains no knowledge of language at all. +- **The model weights** (`ggml-base.en.bin`) are *numbers* — hundreds of megabytes of parameters + learned during training. They contain everything the model "knows", and they cannot execute. + +Neither works alone. The analogy: the runtime is a music player, the weights are the recording. + +This is why they're versioned and stored separately, and why a transcript records **both** +(`runtime_version` *and* `model_name` + `model_sha256`). Upgrading whisper.cpp can change speed +and fix bugs; swapping the model changes what it hears. Those are different kinds of change and +you want to know which one happened. + +### Why model and algorithm versions must be stored + +SpeakLab's whole premise is comparing your practice over months. Suppose in March a transcript +says you said "the API returns a token" and in June a transcript of the same rehearsal says "the +API returns a coden". Did your diction get worse, or did the model change? + +**Without stored versions that question is permanently unanswerable.** Not "hard to answer" — +unanswerable, because the information was never recorded and cannot be reconstructed. + +Note that the model *name* isn't enough. `ggml-base.en.bin` has been re-uploaded upstream more +than once; two files with that name can differ. The SHA-256 of the actual bytes is the only thing +that truly pins it, which is why that's what gets stored. + +This is also why the mock provider deliberately stores `NULL` for all of these. A simulated +transcript genuinely has no model. Filling those columns in with something plausible would make a +fake row indistinguishable from a real one a year from now. + +### What real-time factor means + +**RTF = processing time ÷ audio duration.** One number that answers "can this keep up?" + +- RTF of **0.5** → 60 seconds of audio took 30 seconds. Twice as fast as real time. +- RTF of **1.0** → exactly as long as the recording itself. +- RTF of **2.0** → 60 seconds of audio took two minutes. Half of real time. + +The reason to prefer it over raw seconds is that raw seconds are meaningless without knowing the +clip length — "took 40 seconds" is excellent for a five-minute recording and terrible for a +ten-second one. RTF normalizes that away, so it's directly comparable across recordings, models, +and machines. It's the number to watch when deciding whether `small.en` is affordable, or whether +Core ML is worth enabling. + +(RTF below 1.0 is also the threshold for whether live transcription-as-you-speak would even be +possible. Not a Phase 2A feature, but that's what the number tells you.) + +### Why the original recording and the temporary PCM file are different things + +They exist for opposite reasons and are treated in opposite ways: + +| | Original recording | Temporary PCM | +|---|---|---| +| Format | Whatever the browser made (compressed) | 16 kHz mono PCM WAV | +| Size | Small | ~10× larger | +| Lifetime | Until you delete the session | Deleted at the end of the job | +| Purpose | The irreplaceable artifact | Disposable input to one model run | +| On failure | **Always kept** | **Always deleted** | + +The original is the thing you cannot get back — you can't re-record a moment. Everything else +(the PCM, the transcript, the metrics) can be recomputed from it. That asymmetry is exactly why +every failure path preserves the recording and deletes the scratch file. + +The scratch file is also the more *sensitive* of the two in one specific way: it's your speech +uncompressed and in the clear. Leaving it behind after a crash would quietly convert a temp file +into retained personal data, which is why cleanup runs in a `finally` block on every path rather +than at the end of the happy path. + +### Why polling is enough here + +Polling has a reputation as the crude option, with WebSockets or SSE as the "proper" answer. For +this problem that reputation is misleading. Look at the actual numbers: **one** user, **one** job, +lasting **tens of seconds**, with a payload of a status string and a stage name. + +Polling every 1–5 seconds costs a few dozen tiny requests to localhost. A WebSocket would cost a +persistent connection, a server-side lifecycle, reconnection handling, and a dev-proxy +configuration — to save those few dozen requests and learn about completion perhaps three seconds +sooner, on something that took a minute. + +And the decisive argument: **push doesn't remove the need for polling logic anyway.** The failure +case that matters most here is *the worker isn't running at all*. In that case there is no event +to push, ever. A push-based design would still need a timeout to detect it. Polling handles it +naturally — the timeout fires and the UI says "start the worker". + +Where polling would be wrong: streaming partial transcript text as the model emits it, or watching +many jobs at once. Neither is a thing SpeakLab does. ## Why real AI processing is intentionally postponed +*(Written during Phase 1. Kept as a record of the reasoning; transcription is now real — see +the Phase 2A section above for what that actually took.)* + Real local speech-to-text (whisper.cpp) and real speech analytics are both nontrivial pieces of engineering — model downloads, inference performance, async processing so the UI doesn't freeze, deciding what "good feedback" even means. Building the *entire rest of the app* (recording, diff --git a/docs/PRIVACY_AND_SECURITY.md b/docs/PRIVACY_AND_SECURITY.md index 7168adc..cd87e61 100644 --- a/docs/PRIVACY_AND_SECURITY.md +++ b/docs/PRIVACY_AND_SECURITY.md @@ -4,8 +4,52 @@ The backend binds `127.0.0.1` only (`Settings.host`, `app/main.py`). There is no external network call anywhere in the request path — no cloud transcription, no cloud LLM, no analytics/telemetry -beacon. Recordings, transcripts (currently mock), metrics, feedback, and reflections are written -only to the local SQLite database and the local `backend/storage/recordings/` directory. +beacon. Recordings, transcripts, metrics, feedback, and reflections are written only to the local +SQLite database and local directories under `backend/storage/`. + +**This remains true now that transcription is real.** Speech-to-text runs as a local subprocess +(`whisper-cli`) against a model file on your disk. Your audio is never uploaded anywhere, and the +transcription step makes no network request at all. The only network access in the whole project +is during *setup*, when `scripts/setup_whisper.sh` clones the pinned whisper.cpp release and +downloads a model — a one-time, explicit, user-initiated action. + +## New local artifacts in Phase 2A, and what each one contains + +| Location | Contains | Lifetime | +|---|---|---| +| `backend/storage/recordings/` | Your original audio | Until you delete the session | +| `backend/storage/processing//` | **Decoded PCM of your speech, in the clear** | Deleted at the end of every job, success or failure | +| `backend/storage/models/` | Model weights (no personal data) | Until you remove them | +| `backend/storage/benchmarks/` | Recordings and reference transcripts you place there, plus benchmark output | Until you remove them | +| `backend/runtime/whisper/` | whisper.cpp source, build tree, binary (no personal data) | Until you remove it | + +All five are gitignored, and `scripts/check_public_safety.sh` fails if anything inside them +becomes visible to git — including a file that is merely untracked-and-not-ignored, which a +`git add -A` would otherwise sweep up. + +The processing directory is the one to be careful about: it holds a decoded copy of your speech +while a job runs. It is removed on every exit path, including failures and exceptions, so a crash +does not quietly turn a temp file into retained personal data. The debugging flag that preserves +it (`Settings.keep_processing_files`) is off by default and logs a warning when enabled. + +## Subprocess output is redacted before it is logged + +ffmpeg and whisper.cpp both write absolute paths to stderr, and on a personal machine an absolute +path contains the operating-system username. That text is never returned by the API, never stored +in the job table, and never logged raw: `app/safe_text.py` strips absolute paths and home +references first, and the API returns a stable error *code* plus a pre-written safe sentence. + +The runtime diagnostics endpoint (`GET /api/runtime/capabilities`) is built the same way. It +reports booleans, short version strings, a model file size, and the coarse platform — and +deliberately **not** absolute paths, usernames, home directories, environment variables, +transcript content, or anything secret. It is meant to be safe to paste into a public issue. + +## External binaries are invoked without a shell + +Every external process this project starts goes through one function, `app/subprocess_util.py: +run_command`, which takes an argv array and **rejects a string command outright**. There is no +shell involved, so there is no quoting or metacharacter injection class of bug at all. A timeout +is mandatory, output is captured rather than inherited, and stdin is closed. ## Recording upload validation — what it is and isn't @@ -19,13 +63,19 @@ only to the local SQLite database and the local `backend/storage/recordings/` di tag or frame-sync bytes) — so a file that merely *claims* to be `audio/webm` but obviously isn't gets rejected before it's ever fully written. -**What it is NOT:** real media validation. Passing both checks does not mean the file is a valid, -well-formed, decodable audio file — it means the first few bytes look plausible. No audio decoding -library is used in Phase 1 (deliberately — see [ROADMAP.md](ROADMAP.md) and -[SCORING_AND_LIMITATIONS.md](SCORING_AND_LIMITATIONS.md): there's no real audio processing yet for -a decode step to feed). Full decode/validation (e.g. actually opening the file with an audio -library and confirming it decodes) is deferred to whichever phase adds real transcription, since -that phase needs to decode the audio anyway. +**What it is NOT:** real media validation *at upload time*. Passing both checks does not mean the +file is a valid, well-formed, decodable audio file — it means the first few bytes look plausible. + +**Phase 2A closed this gap, one step later in the pipeline.** The worker now decodes every +recording with ffmpeg before transcription ([ADR 0007](adr/0007-ffmpeg-preprocessing-boundary.md)), +so a file that passed the magic-byte check but is not actually decodable fails with a clear +message instead of reaching the model. The decoder also verifies the output really is 16 kHz mono +16-bit and fails if not, rather than handing mis-shaped audio to whisper.cpp. + +Note where the validation happens: the upload request still accepts the file on the cheap checks +alone, and real validation occurs in the worker. That is deliberate — decoding is far too slow to +do inside the upload request, and the failure is reported through the same job-failure path as any +other processing problem. ## Upload limits and streaming @@ -74,7 +124,11 @@ today; use the manual delete function instead. - Internal exceptions are logged server-side (`app/logging_config.py`) but the API never returns stack traces or local filesystem paths to the client — every error handler in `app/main.py` returns a generic, human-readable `detail` + `recovery` pair instead. -- Real recordings, transcripts, personal reflections, the SQLite database, and anything else - produced by actually using the app are excluded from Git (`.gitignore`) and must never be - committed — see [README.md](../README.md) and [SECURITY.md](../SECURITY.md) for the public-repo - handling of this project. +- Real recordings, transcripts, personal reflections, the SQLite database, model files, decoded + processing audio, benchmark input/output, and the whisper.cpp runtime tree are all excluded from + Git (`.gitignore`) and must never be committed — see [README.md](../README.md) and + [SECURITY.md](../SECURITY.md) for the public-repo handling of this project. +- Test fixtures are synthetic by construction: silent PCM WAVs generated arithmetically at test + time and hand-written JSON payloads. No real recording is ever a fixture. +- `scripts/check_public_safety.sh` additionally fails if the local username appears anywhere in + tracked content, or if any file under a runtime directory is neither tracked nor ignored. diff --git a/docs/PROJECT_STATUS.md b/docs/PROJECT_STATUS.md index dfb56f3..9a9ed33 100644 --- a/docs/PROJECT_STATUS.md +++ b/docs/PROJECT_STATUS.md @@ -1,79 +1,120 @@ # Project Status -**Current phase:** Phase 1 complete (vertical slice with mock analysis). Phase 0 (scaffolding + -docs) complete as part of the same build. +**Current phase:** Phase 2A complete (real local transcription). Phases 0 and 1 complete. ## Completed +### Phase 0 / 1 + - Monorepo scaffold: `backend/` (FastAPI + SQLite), `frontend/` (SvelteKit), `docs/`, `scripts/`. - Backend: migration runner, full schema, topic seed (50 topics) + SQLite-as-runtime-source-of- truth repository, session state machine with logged transitions, streamed/validated recording - upload, physical deletion with failure compensation, three mock providers, full REST API, - structured logging, human-readable error responses. -- Frontend: all 9 screens (dashboard, setup, prepare, record, review, reflect, results, history - list, history detail), a browser-media abstraction (`AudioRecorder`), a monotonic countdown + upload, physical deletion with failure compensation, mock providers, full REST API, structured + logging, human-readable error responses. +- Frontend: all 9 screens, a browser-media abstraction (`AudioRecorder`), a monotonic countdown timer, a relative-URL API client behind a dev proxy, missing-state recovery on every - `/practice/*` route, accessibility basics (focus states, `aria-live` status, text+icon labels, - reduced-motion handling). -- Full documentation set (this directory + root `README.md`, `AGENTS.md`, `SECURITY.md`, 5 ADRs). -- `.gitignore` covering all real user data / secrets / build output, per the public-repo policy. - -## In progress / not started - -Nothing left uncommitted in Phase 1 scope. See [ROADMAP.md](ROADMAP.md) for Phase 2. + `/practice/*` route, accessibility basics. +- Full documentation set, `.gitignore` covering all real user data, public-safety script. + +### Phase 2A — real local transcription + +- **Revised `TranscriptionProvider` contract** — takes a `TranscriptionRequest` (decoded audio + path, measured duration, language, model + runtime config); returns text, detected language, + ordered segments with ms timestamps, model name + SHA-256, runtime version, parameters, + processing duration, real-time factor, `is_mock`. The Phase 1 "one-line swap" prediction was + wrong and is corrected in [AI_MODEL_STRATEGY.md](AI_MODEL_STRATEGY.md). +- **`WhisperCppTranscriptionProvider`** against a pinned `v1.9.1` runtime, parsing `--output-json` + rather than scraping console text. Malformed/missing output is a provider failure, never an + empty-but-successful transcript. +- **`AudioPreprocessor` / `FfmpegAudioPreprocessor`** — decode to 16 kHz mono PCM, authoritative + duration from the decoded stream, per-job temp workspace always cleaned, argv arrays, timeouts, + redacted stderr, original recording never modified. +- **Migration `0002`** — `processing_jobs` (with a partial unique index enforcing one active job + per session), `transcript_segments`, transcript provenance columns, and + `sessions.audio_duration_seconds` alongside the preserved client-reported value. +- **Separate worker** (`python -m app.worker`) — atomic `BEGIN IMMEDIATE` claiming, transaction + released before any audio work, retryable-vs-terminal failure taxonomy, bounded retries, stale + `RUNNING` recovery, graceful SIGINT/SIGTERM shutdown, logs free of transcript text and paths. +- **API** — `POST /api/sessions/{id}/recording` returns `202 Accepted`; new + `GET /api/sessions/{id}/status` and `GET /api/runtime/capabilities`. +- **Frontend** — HTTP polling (1s → backoff → 5s cap, 10-min timeout, cancel on teardown, no + duplicate loops), stage display, per-error-code setup guidance, and **per-section provenance** + (real transcript / simulated metrics / simulated feedback) with no global "all simulated" banner. +- **Setup and diagnostics** — `scripts/setup_whisper.sh` (pinned clone, Release build, model + download, verification, manifest) and `scripts/check_whisper_runtime.sh`. Neither installs + system packages; both print exact commands for missing dependencies. +- **Benchmark utility** — `backend/scripts/benchmark_models.py` (model size, audio duration, + processing time, RTF, detected language, runtime/model version, normalized transcript, WER). +- 4 new ADRs (0006–0009); all affected docs updated. ## Known limitations (honest, as of this build) -- **Real microphone capture was not verified in this automated build environment** — the browser - tool used here has no real audio input device, and `getUserMedia()` hangs indefinitely rather - than granting/denying, so no permission-prompt flow could be exercised for real. See the manual - QA checklist below for what a human needs to verify on a real device before trusting this in - daily use. -- The audio-retention setting is a placeholder — get/set works, nothing enforces it (see - [PRIVACY_AND_SECURITY.md](PRIVACY_AND_SECURITY.md)). -- Upload validation is whitelist + magic-byte only, not full audio decode (see - [PRIVACY_AND_SECURITY.md](PRIVACY_AND_SECURITY.md)). -- The standalone `adapter-node` production server has no `/api` proxy (see - [ARCHITECTURE.md](ARCHITECTURE.md)) — only `npm run dev` / `npm run preview` are the supported - ways to run the frontend today. -- No Playwright/e2e coverage (see [TESTING.md](TESTING.md) for why). -- A page refresh loses an in-progress (unsaved) practice session by design (no IndexedDB - persistence yet — see [LEARNING_NOTES.md](LEARNING_NOTES.md)). - -## A real bug found (and fixed) during manual integration testing - -While manually driving the full workflow through a real browser against the real backend, saving -a recorded session failed with `415 Unsupported audio type: 'audio/webm;codecs=opus'`. Real -browsers report `MediaRecorder.mimeType` *with* a codec parameter (Chrome/Firefox report exactly -`audio/webm;codecs=opus`), but `RecordingStorage.save()`'s MIME whitelist was doing an exact -string match against bare types like `"audio/webm"` — meaning **every real recording from a real -browser would have been rejected**, even though the 48 automated tests (which used bare -`"audio/webm"` in their fixtures) all passed. Fixed in `app/repositories/recording_storage.py` by -normalizing the declared type (stripping the `;codecs=...` parameter) before both the whitelist -lookup and DB storage, with a regression test added -(`test_save_accepts_a_codec_qualified_mime_type_from_a_real_browser`). This is a good example of -why the manual integration pass in [TESTING.md](TESTING.md) tier 4 exists — it caught something -purely-unit-tested code with idealized fixtures had missed. +### Not verified in this build environment — the important ones + +- **No whisper.cpp build has ever run in this repository.** The machine this phase was built on + has neither `cmake` nor `ffmpeg` installed (verified: both absent from `PATH`). `setup_whisper.sh` + was syntax-checked and `check_whisper_runtime.sh` was run — it correctly reported all five + prerequisites missing — but the build itself was never executed. +- **No real model has ever transcribed anything here.** The opt-in integration test + (`test_whisper_integration.py`) skips, naming the reason: `ffmpeg is not installed`. Every + whisper.cpp and ffmpeg interaction in the passing test suite is mocked. +- **No benchmark numbers exist.** `benchmark_models.py` has never been run against a real model + or a real recording. This repository therefore publishes **no** latency, real-time-factor, or + word-error-rate figures. Any number here would have been fabricated. +- **Real microphone capture is still unverified** (carried over from Phase 1 — the automated + environment has no audio input device). +- **The end-to-end workflow has not been driven through a browser this phase.** Backend behavior + is covered by 133 automated tests and the frontend by 60, but no manual pass through + record → upload → worker → poll → results happened, because it cannot complete without ffmpeg + and a model. + +### Design limitations + +- Metrics and feedback are still simulated and labeled as such per section + ([SCORING_AND_LIMITATIONS.md](SCORING_AND_LIMITATIONS.md)). +- The worker must be started manually. If it isn't, sessions sit in `PROCESSING` until the poll + times out and the UI says so. +- No component-level frontend DOM tests; the logic those components consume is tested as pure + functions instead ([TESTING.md](TESTING.md) explains the trade). +- Audio-retention setting is still a placeholder that nothing enforces. +- The standalone `adapter-node` production server still has no `/api` proxy. +- No Playwright/e2e coverage. +- A page refresh still loses an in-progress unsaved session by design. ## Exact test/build results (this session) +Baseline before any Phase 2A change: backend `49 passed`, frontend `26 passed`, check `0 ERRORS`, +lint clean, build succeeded, safety check `OK`. + +After Phase 2A: + ``` $ cd backend && source .venv/bin/activate && pytest -q -................................................. [100%] -49 passed, 1 warning in 0.46s +........................................................................ [ 53%] +.................ss............................................ [100%] +133 passed, 2 skipped, 1 warning in 1.93s ``` -(The one warning is `StarletteDeprecationWarning: Using httpx with starlette.testclient is -deprecated` — a library-level deprecation notice, not a test failure.) +(The 2 skips are the opt-in real-whisper integration tests. The warning is +`StarletteDeprecationWarning: Using httpx with starlette.testclient is deprecated` — a +library-level notice, not a failure.) + +``` +$ SPEAKLAB_RUN_WHISPER_INTEGRATION=1 pytest -q tests/test_whisper_integration.py -rs +SKIPPED [1] tests/test_whisper_integration.py:87: ffmpeg is not installed (brew install ffmpeg) +SKIPPED [1] tests/test_whisper_integration.py:148: ffmpeg is not installed (brew install ffmpeg) +2 skipped, 1 warning in 0.01s +``` +The prerequisite check is real: with the flag set, it names the actual missing dependency. ``` $ cd frontend && npx vitest run - Test Files 4 passed (4) - Tests 26 passed (26) + Test Files 6 passed (6) + Tests 60 passed (60) ``` ``` $ cd frontend && npm run check -1785490004425 COMPLETED 251 FILES 0 ERRORS 0 WARNINGS 0 FILES_WITH_PROBLEMS +1785524742081 COMPLETED 256 FILES 0 ERRORS 0 WARNINGS 0 FILES_WITH_PROBLEMS ``` ``` @@ -85,13 +126,43 @@ All matched files use Prettier code style! ``` $ cd frontend && npm run build -✓ built in ~1.5s +✓ built in ~1.6s Using @sveltejs/adapter-node ✔ done ``` -(Three "Generated an empty chunk" notices during build — `env.js`, `client2.js`, -`SessionResults.js` — are Vite/Rollup code-splitting artifacts for modules that ended up with no -server-side runtime content after tree-shaking; not errors, build succeeds.) +(Four "Generated an empty chunk" notices — `env.js`, `client2.js`, `poller.js`, +`SessionResults.js` — are Vite/Rollup code-splitting artifacts, not errors.) + +``` +$ bash scripts/check_public_safety.sh +== Checking for prohibited tracked paths == +== Checking tracked file contents for obvious local-path / secret leakage == +== Confirming Phase 2A runtime artifacts are untracked and ignored == +OK: no prohibited tracked paths or obvious secret/path leakage found. +``` + +``` +$ bash scripts/check_whisper_runtime.sh + MISS cmake not found → brew install cmake + MISS ffmpeg not found → brew install ffmpeg + OK git found + MISS whisper-cli binary not built + MISS runtime manifest missing + MISS ggml-base.en.bin not downloaded + OK no runtime artifact is visible to git +5 problem(s) found +``` +This is the honest state of the build machine, not a failure of the change. + +## Bugs found during this build + +**A rejected duplicate job left its connection holding a SQLite write lock.** A smoke test that +created a job, tried to create a second for the same session (correctly rejected by the partial +unique index), and then attempted a worker claim failed with `database is locked`. The +`IntegrityError` handler raised `ActiveJobExistsError` without rolling back, so the implicit +transaction stayed open. Fixed in `app/repositories/job_repository.py` by rolling back before +re-raising. Worth noting because every unit test passed both before and after — it only appeared +when two connections interacted, which is exactly the situation the queue exists to handle. ## Manual setup steps (to reproduce this state) @@ -100,34 +171,38 @@ server-side runtime content after tree-shaking; not errors, build succeeds.) cd backend python3 -m venv .venv && source .venv/bin/activate pip install -r requirements.txt -pytest -q # 49 passed +pytest -q # 133 passed, 2 skipped + +# Transcription runtime (NOT run in this build environment) +brew install cmake ffmpeg # your call; the script never installs for you +scripts/setup_whisper.sh # clones v1.9.1, builds, downloads base.en +scripts/check_whisper_runtime.sh # verify + +# Three processes, three terminals uvicorn app.main:app --reload --host 127.0.0.1 --port 8000 +python -m app.worker +cd frontend && npm install && npm run dev # http://127.0.0.1:5173 +``` + +## Manual QA checklist (not yet run) + +Carried over from Phase 1, plus Phase 2A items: -# Frontend (separate terminal) -cd frontend -npm install -npm run check # 0 errors -npm run lint # clean -npx vitest run # 26 passed -npm run build # succeeds -npm run dev # http://127.0.0.1:5173 -``` - -## Manual real-device QA checklist (not yet run — do this before trusting daily use) - -1. Open the app in Chrome, Firefox, and Safari on a real machine with a working microphone. -2. Start a practice session; confirm the browser's real permission prompt appears only when - clicking "Start recording," not earlier. -3. Grant permission; confirm the recording indicator and countdown both work, speak for a few - seconds, stop, and confirm playback on the Review screen sounds correct. -4. Deny permission (or revoke it mid-session) and confirm the error banner's recovery text is - accurate for that browser's actual permission-denial UI. -5. Save a session; confirm `GET /api/sessions/{id}/audio` playback works from History too. -6. Repeat on at least one browser whose `MediaRecorder.mimeType` differs (e.g. Safari, which - tends toward `audio/mp4`) to confirm the whitelist-normalization fix above covers it. +1. Real microphone capture across Chrome, Firefox, and Safari (permission prompt timing, + recording indicator, playback on Review). +2. Safari specifically, whose `MediaRecorder.mimeType` differs (`audio/mp4`). +3. **Full pipeline with a real model:** record → save → confirm `202` → reflect while the stage + indicator updates → results shows a real transcript with segment timestamps and provenance. +4. **Worker-absent path:** upload with the worker stopped, confirm the poll times out and the UI + gives the start command. +5. **Missing-model path:** rename the model file, confirm the session fails with + `MODEL_MISSING` and the UI shows the download command. +6. **Kill the worker mid-transcription** (SIGKILL), restart it, confirm the stale job is + reclaimed and completes. +7. **Benchmark** `base.en` vs `small.en` on a real recording with a reference transcript, and + record the numbers here. ## Next recommended phase -Phase 2 as scoped in [ROADMAP.md](ROADMAP.md), starting with real local Whisper transcription — -but only after the manual real-device QA checklist above has actually been run once, since that's -the one meaningful gap this build session couldn't close itself. +Phase 2B as scoped in [ROADMAP.md](ROADMAP.md): real deterministic metrics — and first, actually +running the real-model verification and benchmark that this build environment could not. diff --git a/docs/ROADMAP.md b/docs/ROADMAP.md index 627f584..54631ce 100644 --- a/docs/ROADMAP.md +++ b/docs/ROADMAP.md @@ -10,31 +10,57 @@ Repo scaffolding, architecture decisions, documentation set. See The full local practice workflow with mock analysis, described in [PRD.md](PRD.md). See [PROJECT_STATUS.md](PROJECT_STATUS.md) for exact status, test results, and known limitations. -## Recommended Phase 2 scope - -In priority order, each independently valuable: - -1. **Real local transcription (whisper.cpp).** Replace `MockTranscriptionProvider` with a - whisper.cpp-backed implementation behind the same `TranscriptionProvider` interface (see - [AI_MODEL_STRATEGY.md](AI_MODEL_STRATEGY.md)). This is the one item that forces an - architectural change beyond "swap a class": processing needs to move from Phase 1's - synchronous in-request pipeline to a queued local worker, because real transcription of even a - short clip is too slow to hold an HTTP request open for. Needs: a job table/queue (could be as - simple as a `PROCESSING` session polled by a worker loop — no need for a message broker at - single-user scale), a way for the frontend to poll or subscribe for completion, and a decision - on model size/quality trade-offs and where the model file lives (never in Git — see - [PRIVACY_AND_SECURITY.md](PRIVACY_AND_SECURITY.md)). -2. **Real deterministic metrics**, once real transcripts exist: actual words-per-minute, real - filler-word detection, real pause detection from audio silence. Replaces - `MockMetricsProvider` behind the same interface. Should land before any LLM-based feedback - layer (see the ADR on that ordering). -3. **Full audio validation**, once a real decode step exists anyway for transcription: replace - the magic-byte-only check in `RecordingStorage.save()` with genuine decode validation. -4. **Audio retention enforcement.** The `audio_retention_days` setting already exists - end-to-end (get/set API, DB column) but nothing reads it — build the actual cleanup job. -5. **Real-device manual QA** of microphone recording across Chrome/Firefox/Safari (Phase 1's - automated environment couldn't grant real mic access — see - [TESTING.md](TESTING.md) and [PROJECT_STATUS.md](PROJECT_STATUS.md) for the checklist to run). +## Phase 2A — done + +**Real local transcription (whisper.cpp).** Delivered, along with the architectural changes it +forced. See [PROJECT_STATUS.md](PROJECT_STATUS.md) for exact status and +[AI_MODEL_STRATEGY.md](AI_MODEL_STRATEGY.md) for an honest account of where the Phase 1 plan for +this was wrong. + +What actually shipped: + +- `WhisperCppTranscriptionProvider` against a pinned `v1.9.1` runtime built locally + ([ADR 0009](adr/0009-pinned-whisper-cpp-runtime.md)). +- A **revised `TranscriptionProvider` contract** — the Phase 1 signature received no audio and the + result had nowhere to record provenance, so this was not the "one-line swap" this roadmap + predicted. +- A **separate worker process and durable SQLite job queue** + ([ADR 0006](adr/0006-local-worker-sqlite-queue.md)) — resolving the open question below in + favour of a separate process over an in-process background task. +- An **ffmpeg preprocessing boundary** ([ADR 0007](adr/0007-ffmpeg-preprocessing-boundary.md)), + which also delivered item 3 below as a side effect. +- **HTTP polling** ([ADR 0008](adr/0008-http-polling-not-websockets.md)) — resolving the open + question below against SSE/WebSockets. +- Migration `0002`: `processing_jobs`, `transcript_segments`, transcript provenance columns, and + an authoritative `sessions.audio_duration_seconds` alongside the preserved client-reported one. + +## Recommended Phase 2B scope + +In priority order: + +1. **Real deterministic metrics.** The inputs finally exist: a real transcript and an + authoritative decoded duration. Actual words-per-minute (transcript words ÷ + `audio_duration_seconds`), real filler-word detection (lexical matching), and real pause + detection (from segment-timestamp gaps, and more precisely from silence analysis of the + decoded PCM). Replaces `MockMetricsProvider` behind the same interface. Must land before any + LLM-based feedback layer ([ADR 0005](adr/0005-deterministic-metrics-before-llm.md)). +2. **Run the real-model verification that Phase 2A could not.** No whisper.cpp build, no model + run, and no benchmark has ever executed in this repository — the build machine had neither + cmake nor ffmpeg. Run `scripts/setup_whisper.sh`, the opt-in integration test, and + `backend/scripts/benchmark_models.py` comparing `base.en` against `small.en` on a real + recording, then record the measured numbers. +3. **Audio retention enforcement.** The `audio_retention_days` setting still exists end-to-end + with nothing reading it — build the cleanup job. Now more valuable, since a session's data + footprint has grown. +4. **Real-device manual QA** of microphone recording across Chrome/Firefox/Safari, still not done + (see [PROJECT_STATUS.md](PROJECT_STATUS.md) for the checklist). +5. **Component-level frontend tests**, if the results view grows more conditional logic — see the + honest gap noted in [TESTING.md](TESTING.md). + +### Done as a side effect of Phase 2A + +- ~~**Full audio validation.**~~ The ffmpeg decode step now genuinely validates every recording; + a file that passed the magic-byte check but does not decode fails with a clear message. ## Later, unscoped candidates @@ -54,13 +80,20 @@ In priority order, each independently valuable: RAG, vector databases, multi-agent orchestration, microservices, cloud AI APIs, Docker, user accounts for a single-user local tool. See [AGENTS.md](../AGENTS.md). -## Open questions before starting whisper.cpp work +## Open questions from before the whisper.cpp work — resolved -- Model size/quality trade-off for CPU-only local inference, and expected latency for a 1–3 - minute clip on typical consumer hardware. -- Where the queued-worker boundary should live: a simple in-process background task vs. a - separate worker process reading a `PROCESSING`-status queue from SQLite. -- Whether the frontend should poll `GET /api/sessions/{id}` for status changes or whether a - simple SSE/WebSocket endpoint is worth adding at that point. -- Packaging/distribution of the model file itself (download-on-first-run vs. a documented manual - step) — it must never be committed to Git. +- **Worker boundary: in-process task vs. separate process?** → **Separate process.** An + in-process task ties the work's lifetime to the API's, so restarting the API for a code change + would kill an in-flight transcription with no record of it. See + [ADR 0006](adr/0006-local-worker-sqlite-queue.md). +- **Poll vs. SSE/WebSocket?** → **Poll**, against a narrow dedicated `/status` endpoint rather + than the full session detail. See [ADR 0008](adr/0008-http-polling-not-websockets.md). +- **Model file distribution?** → **A documented manual step**, `scripts/setup_whisper.sh`, which + downloads via upstream's own downloader into a gitignored directory. It never installs system + packages; missing dependencies are reported with the exact command and the script exits. See + [ADR 0009](adr/0009-pinned-whisper-cpp-runtime.md). +- **Model size/quality trade-off, and latency for a 1–3 minute clip?** → **Still open, and + deliberately unanswered.** `base.en` is the configured default and `small.en` is supported, but + no benchmark has been run on real hardware with a real recording, so this repository publishes + no latency, real-time-factor, or accuracy numbers. `backend/scripts/benchmark_models.py` exists + to answer it; running it is Phase 2B item 2. diff --git a/docs/SCORING_AND_LIMITATIONS.md b/docs/SCORING_AND_LIMITATIONS.md index d863dbb..6330340 100644 --- a/docs/SCORING_AND_LIMITATIONS.md +++ b/docs/SCORING_AND_LIMITATIONS.md @@ -1,43 +1,73 @@ # Scoring and Limitations -**No real speech analysis occurs yet.** Everything on the results screen — the transcript, the -words-per-minute figure, the filler-word counts, the pause statistics, the strength / -improvement-target / retry-instruction feedback — is generated by deterministic mock providers in -`backend/app/providers/`. None of it comes from listening to, transcribing, or otherwise analyzing -the actual audio the user recorded. Every one of these records is stored with `is_mock = 1` and -is shown in the UI behind a "Simulated" badge (`SessionResults.svelte`) — never presented as if it -were real analysis. - -## What each mock provider actually does - -- **`MockTranscriptionProvider`** — returns a templated sentence naming the topic, mode, and - reported duration, explicitly prefixed `[SIMULATED TRANSCRIPT — no real speech-to-text was - performed]`. It does not read the audio file at all. +**The transcript is real. The metrics and feedback are not.** As of Phase 2A this page has three +different truths on it at once, so read the badges rather than assuming. + +| Section | Status | Stored as | +|---|---|---| +| Transcript | **Real** — local whisper.cpp on your actual audio | `is_mock = 0` | +| Metrics (WPM, fillers, pauses) | Simulated | `is_mock = 1` | +| Feedback (strength/improve/retry) | Simulated | `is_mock = 1` | + +There is deliberately **no** page-level "everything is simulated" banner any more — for a session +recorded since Phase 2A it would be false. Each section carries its own label, derived from its +own row's `is_mock` flag. A session recorded during Phase 1 still shows "Simulated" on all three, +because its rows still say so. + +## What is real now + +- **The transcript.** Produced by a pinned local whisper.cpp build (`v1.9.1`) reading the actual + decoded audio you recorded, on your machine, offline. It carries the model name, the model's + SHA-256, the runtime version, the inference parameters, and how long it took — visible under + "How this transcript was produced" on the results screen. +- **The audio duration.** `sessions.audio_duration_seconds` is measured by decoding the media, + not taken from the browser's stopwatch (see [DATA_MODEL.md](DATA_MODEL.md)). +- Everything that was already real in Phase 1: the recording itself, session metadata, your own + reflection, and the history of what you practiced and when. + +## What is still simulated + - **`MockMetricsProvider`** — derives pseudo-random-but-deterministic numbers (words/minute, filler-word breakdown, pause count/longest-pause) from a hash of the session ID and the - client-reported duration. Same session ID always produces the same numbers, but the numbers - have no relationship to what was actually said. + duration. Same session ID always produces the same numbers. **It does not read the transcript.** + This is worth stating twice: a real transcript is now sitting directly above these numbers on + the results screen, and they are still not derived from it. - **`MockFeedbackProvider`** — picks from a small set of templated strength/improvement/retry - strings based on the (already fake) metrics above. No language understanding is involved. + strings based on the (already fake) metrics above. No language model reads your transcript. -## Why this exists +## What NOT to conclude from a session today -The point of Phase 1 is to prove the *complete workflow* — recording, storage, state machine, -review, reflection, results, history — end to end, before adding the real cost and complexity of -local model inference. See [AI_MODEL_STRATEGY.md](AI_MODEL_STRATEGY.md) for what "real" will -look like and why it's sequenced this way, and [ROADMAP.md](ROADMAP.md) for when. - -## What NOT to conclude from a Phase 1 session - -- A high or low simulated "words per minute" says nothing about actual pace. -- The listed "filler words" were not detected in the recording — they're arithmetic on a hash. +- A high or low "words per minute" says nothing about your actual pace — it is arithmetic on a + hash, not words in the transcript divided by time. +- The listed "filler words" were **not** detected in your speech, and are not counted from the + transcript sitting above them. - The feedback text is not personalized advice; it's one of a handful of fixed templates chosen by simple thresholds on the fake metrics. -- History and benchmark topics do **not** yet support any real longitudinal comparison of - speaking quality, because there is no real quality signal to compare. +- History and benchmark topics still do **not** support longitudinal comparison of speaking + *quality*. You can now compare what you actually said, which is genuinely new — but no quality + score behind it is real. + +## Transcription accuracy is not perfect either + +Real transcription is real, not infallible: + +- `base.en` is a small model chosen for speed. It misrecognizes technical vocabulary, proper + nouns, and unusual phrasing. `small.en` is more accurate and slower. +- Accented speech, background noise, and a distant microphone all degrade accuracy. +- Punctuation and sentence boundaries are the model's guess. +- Segment timestamps are approximate. Word-level timestamps are not recorded at all. +- An empty transcript means the model detected no speech — which can happen with a very quiet + recording, and is reported honestly rather than as an error. + +**No accuracy figures are published in this repository**, because none have been measured. Use +`backend/scripts/benchmark_models.py` with your own recording and your own reference transcript to +get a word error rate for your voice, your microphone, and your hardware — the only numbers that +would mean anything for your use. -## What IS real in Phase 1 +## Why it was sequenced this way -The recording itself (real microphone audio, stored as-is), the session metadata (topic, mode, -timers, timestamps, status), the reflection you write yourself, and the history of what you -practiced and when. The workflow and the data model are real; the "analysis" is a placeholder. +Phase 1 proved the *complete workflow* — recording, storage, state machine, review, reflection, +results, history — before adding the cost and complexity of local model inference. Phase 2A then +made exactly one thing real. Doing transcription and metrics together would have made it much +harder to tell which half was responsible when something looked wrong. See +[AI_MODEL_STRATEGY.md](AI_MODEL_STRATEGY.md) and [ROADMAP.md](ROADMAP.md). diff --git a/docs/TESTING.md b/docs/TESTING.md index 638a08b..8504f15 100644 --- a/docs/TESTING.md +++ b/docs/TESTING.md @@ -1,11 +1,52 @@ # Testing -Four tiers, each covering something the others don't. See [PROJECT_STATUS.md](PROJECT_STATUS.md) -for the exact, current pass/fail counts and commands. +Five tiers, each covering something the others don't. See +[PROJECT_STATUS.md](PROJECT_STATUS.md) for the exact, current pass/fail counts and commands. + +## 0. What is mocked, and why + +Phase 2A added two external binaries (`ffmpeg`, `whisper-cli`). **Both are mocked in the regular +suite**, without exception. Two reasons: + +1. The suite must run on a machine where neither is installed — which is exactly the machine this + phase was built on. +2. A test that shells out to a real encoder is testing ffmpeg, not this code. What is worth + testing here is the argv we build, the timeout we enforce, the output we parse, and what we do + when any of it goes wrong. + +The one exception is tier 5 below, which is opt-in and skips honestly. + +Everything audio-shaped in the fixtures is **synthetic**: silent PCM WAVs generated +arithmetically (`tests/conftest.py: write_synthetic_wav`) and hand-written whisper.cpp JSON +payloads matching the documented output shape. No real recording is ever a fixture. ## 1. Automated backend tests (pytest) -`backend/tests/`, run with `pytest -q` from `backend/` (with `.venv` activated). 49 tests across: +`backend/tests/`, run with `pytest -q` from `backend/` (with `.venv` activated). 133 tests, +plus 2 that skip unless opted in. See [PROJECT_STATUS.md](PROJECT_STATUS.md) for exact output. + +Phase 2A files: + +- **`test_migrations.py`** — the migration tests that matter: apply `0001`, insert Phase 1 shaped + rows, and only *then* apply `0002`, asserting existing rows survive untouched and new columns + are `NULL` rather than backfilled. Also idempotency, the one-active-job partial unique index, + and segment ordering/cascade constraints. +- **`test_worker.py`** — the queue guarantee (two workers on separate connections cannot claim + the same job), oldest-first ordering, the success path writing a real transcript with ordered + segments and full provenance, retryable-vs-terminal failure handling, a finite retry budget, + stale-`RUNNING` recovery, restart behavior, graceful shutdown, temp-file cleanup after both + success and failure, and that an unexpected error cannot escape and kill the loop. +- **`test_audio_preprocessor.py`** — argv construction with the required conversion flags, that a + string command is refused outright, that shell metacharacters are never interpreted, mandatory + timeouts, workspace cleanup on success and failure, wrong-format output rejected rather than + passed to the model, and that ffmpeg stderr containing an absolute path never reaches the + exception a caller might surface. +- **`test_whisper_provider.py`** — argv construction, structured-JSON parsing into ordered + segments, full provenance on the result, and every way the output can be wrong (missing file, + unparseable, missing `transcription` key, missing or non-numeric offsets). An *empty* + transcription list stays a legitimate empty result rather than an error. + +Phase 1 files, updated: - **`test_topic_repository.py`** — filtering by track/category/mode, seeded-selection determinism, immediate-repeat prevention (including the "only one candidate exists" fallback), @@ -15,24 +56,41 @@ for the exact, current pass/fail counts and commands. for how that bug was actually found), magic-byte signature rejection, oversized-upload abort-and-cleanup, path-traversal rejection, delete-of-missing-file as a no-op. - **`test_providers.py`** — mock provider determinism (same input ⇒ same output), `is_mock=True` - always set, output never claims real analysis occurred. -- **`test_session_service.py`** — the full session status-machine happy path (asserting the exact - `session_status_events` sequence), validation rejections (incompatible mode, out-of-range - durations, wrong session state), and — the tests that matter most for honesty — the - **partial-failure/cleanup paths**: an orphaned file removed when the DB insert fails after the - file was written, a provider failure marking the session `FAILED` while preserving the - recording and hiding fake "completed" results, and a filesystem-deletion failure being reported - (not silently swallowed) while leaving both the file and the DB rows intact. -- **`test_api.py`** — HTTP-level schema/status-code checks and one full happy-path walk through - the real FastAPI app (`TestClient`) covering create → upload → reflect → fetch detail → stream - audio → list history → delete → confirm 404 after delete. + always set, output never claims real analysis occurred, and that a mock transcript reports **no** + model or runtime provenance (populating those would make a mock row indistinguishable from a + real one later). +- **`test_session_service.py`** — now asserts the *queueing* contract: an upload leaves the + session `PROCESSING` with a `PENDING` job and **no** analysis output, the status-event log stops + at `PROCESSING`, a second upload is rejected, deleting a session cascades to its jobs, and + reflection works while transcription is still running. The Phase 1 partial-failure/cleanup tests + are kept unchanged, plus a new one for a job row that cannot be written (the session must fail + rather than sit in `PROCESSING` forever waiting for work nobody queued). +- **`test_api.py`** — HTTP-level schema/status-code checks, the `202` response shape, the narrow + status endpoint, the capabilities endpoint (asserting neither leaks an absolute path), and one + full walk through the real FastAPI app (`TestClient`) covering create → upload → reflect → + fetch detail → *worker step* → fetch detail → stream audio → list history → delete → confirm + 404. The worker step is performed inline the way the worker would, so the test exercises real + provenance data over HTTP without needing a worker process. Fixtures (`conftest.py`) build a fully isolated `Settings` per test pointed at `tmp_path`, so tests never touch real `backend/storage/`. ## 2. Automated frontend tests (vitest) -`frontend/src/**/*.test.ts`, run with `npx vitest run`. 26 tests across: +`frontend/src/**/*.test.ts`, run with `npx vitest run`. 60 tests across: + +- **`processing/poller.test.ts`** — the polling lifecycle, with an injected clock and timer queue + so no real time passes: immediate first poll, backoff between polls, a capped interval, stopping + on `COMPLETED`/`FAILED`/404/timeout, surviving a transient error rather than giving up, + cancellation firing no settled callback, a response that arrives *after* cancel changing + nothing, and that starting twice cannot create a second loop. +- **`processing/presentation.test.ts`** — stage labels and progress, per-error-code setup + guidance (each with its exact fix command, each reassuring that the recording survived), and + **per-section provenance**: a real transcript alongside simulated metrics and feedback, a + historical Phase 1 session still simulated throughout, and a missing section reported as + unavailable rather than guessed. + +Phase 1 files: - **`timer.test.ts`** — countdown behavior under `vi.useFakeTimers()`, including a test that specifically proves the countdown derives remaining time from an injected clock rather than @@ -47,7 +105,21 @@ tests never touch real `backend/storage/`. right times. - **`api/client.test.ts`** — the API client's contract against a mocked `fetch`: relative URLs only, correct query-string building, `ApiError` carrying the server's `recovery` hint, `204` - handled as `undefined`. + handled as `undefined`, the `202` accepted body from an upload, the narrow status path, abort + signals forwarded so a cancelled poll aborts in flight, and the runtime capabilities path. + +### What is NOT covered on the frontend, honestly + +There are **no component-level DOM tests**. The processing and failure UI, and the provenance +badges, are exercised through the pure functions they consume (`presentation.ts`), not by +rendering `SessionResults.svelte` or `ProcessingStatus.svelte` and inspecting the output. + +That is a real gap, and it is a deliberate trade: rendering Svelte 5 components in tests needs a +component-testing dependency this project does not have, and [AGENTS.md](../../AGENTS.md) is +explicit about not adding dependencies casually. The mitigation is that all the *decisions* those +components make — which badge, which label, which guidance, which command — live in tested pure +functions, and the components only bind them to markup. A bug in the markup binding itself would +not be caught. Worth revisiting if the results view grows more conditional logic. ## 3. Type-checking, linting, and the production build @@ -77,6 +149,30 @@ PROJECT_STATUS.md). A **manual real-device QA checklist** for a human to run with an actual microphone is in [PROJECT_STATUS.md](PROJECT_STATUS.md). +## 5. Optional real whisper.cpp integration test (opt-in, skips honestly) + +`backend/tests/test_whisper_integration.py` is the only test that runs real external binaries. It +requires **all** of: `SPEAKLAB_RUN_WHISPER_INTEGRATION=1`, an installed ffmpeg, a built +`whisper-cli`, a downloaded model, and a sample recording the owner places at +`backend/storage/benchmarks/sample.wav` (or `.m4a`/`.webm`/`.mp3`/`.ogg`). + +If any of those is missing it **skips with that specific reason named** — not a generic "skipped". +A test that quietly passes when it did not run is worse than no test, because it manufactures +confidence. + +It asserts the *pipeline*, never accuracy: that decoding produces 16 kHz mono 16-bit, that the +provider returns `is_mock=False` with real model/runtime provenance and a positive real-time +factor, that segments are ordered and non-inverted, and that the original recording is byte-for-byte +untouched. What a model outputs for a given clip is a benchmark question +(`backend/scripts/benchmark_models.py`), not a pass/fail assertion. + +```bash +SPEAKLAB_RUN_WHISPER_INTEGRATION=1 pytest tests/test_whisper_integration.py -rs +``` + +**It has never been run against a real model in this repository's history** — see +[PROJECT_STATUS.md](PROJECT_STATUS.md). + ## Why Playwright/e2e is deferred No Playwright browser automation is included. Installing Playwright's browser binaries is a diff --git a/docs/adr/0006-local-worker-sqlite-queue.md b/docs/adr/0006-local-worker-sqlite-queue.md new file mode 100644 index 0000000..8a9d5b4 --- /dev/null +++ b/docs/adr/0006-local-worker-sqlite-queue.md @@ -0,0 +1,83 @@ +# ADR 0006: A Separate Local Worker with a SQLite Job Queue + +## Status + +Accepted (Phase 2A). + +## Context + +Phase 1 ran transcription, metrics, and feedback synchronously inside the +`POST /api/sessions/{id}/recording` request. That was honest and correct while all +three providers were pure, instant, local computation ([ADR 0003](0003-sqlite.md), +[LEARNING_NOTES.md](../LEARNING_NOTES.md) item 3). + +Real whisper.cpp transcription is not instant. On consumer hardware a `base.en` run over a +two-minute clip is measured in tens of seconds, and a larger model or a longer clip is +substantially worse. Three things break if that work stays in the request: + +1. **The request would have to be held open** for far longer than any default client, proxy, or + browser timeout tolerates. +2. **A dropped request would lose the work silently.** The recording would be stored, the session + would be stuck in `PROCESSING`, and nothing anywhere would record that transcription was still + owed. +3. **The user would be blocked** from doing the one thing the app wants them to do next — write + their self-reflection while the attempt is still fresh. + +The open question from [ROADMAP.md](../ROADMAP.md) was where the worker boundary should live: an +in-process background task, or a separate process reading a queue. + +## Decision + +A **separate worker process** (`python -m app.worker`) consuming a **durable job queue stored in +SQLite** (`processing_jobs`). + +The API request stores the recording, transitions the session to `PROCESSING`, inserts a +`PENDING` job row, and returns `202 Accepted`. It never decodes audio and never runs a model. + +The worker polls for `PENDING` jobs and claims one atomically: + +```sql +BEGIN IMMEDIATE; +SELECT ... WHERE status = 'PENDING' ORDER BY created_at LIMIT 1; +UPDATE processing_jobs SET status = 'RUNNING', ... WHERE id = ? AND status = 'PENDING'; +COMMIT; +``` + +`BEGIN IMMEDIATE` takes SQLite's write lock at the start of the transaction rather than on first +write, so a second worker blocks instead of reading the same row. The `AND status = 'PENDING'` +in the `UPDATE` is a second, independent guard: a worker that somehow lost the race updates zero +rows and claims nothing. The transaction covers **only** the claim — holding SQLite's write lock +across a minute of inference would block every unrelated write in the application. + +**No message broker.** Redis, RabbitMQ, Celery, and SQS were all considered and rejected: each +adds a service to install, run, monitor, and reason about, to solve throughput and distribution +problems that a single-user local app structurally does not have. Everything a broker would +provide here — durability, at-most-once handoff, retries, dead-lettering — is a few dozen lines +against a database this project already runs. + +**No in-process background task.** `asyncio.create_task` or a thread inside FastAPI would have +avoided a second process, but it ties the work's lifetime to the API's: restarting the API for a +code change would kill an in-flight transcription with no record of it, and a crash would strand +the session. A separate process also means CPU-saturating inference cannot starve the event loop +serving the UI. + +## Consequences + +**Gained:** work survives an API restart, a worker crash, and a machine reboot, because a job is +committed data rather than in-process state. The upload request returns in milliseconds. Retries, +attempt budgets, stale-job recovery, and progress reporting all become ordinary SQL. The failure +taxonomy (retryable vs terminal) is explicit and testable. + +**Given up:** the owner now has to start a second process, and a session sits in `PROCESSING` +indefinitely if they forget — which is why `GET /api/runtime/capabilities` exists and why the UI +says so after a timeout. Polling for jobs costs a cheap indexed query per second, which is +irrelevant at this scale but is not free. There is exactly-once handoff but not exactly-once +*execution*: a worker killed after committing a transcript but before transitioning the session +leaves the job to be reclaimed and redone, which is why the retry path clears partial analysis +output first. + +## Revisit if + +Transcription needs to run concurrently for several sessions (it does not at single-user scale), +or SpeakLab ever becomes multi-user — at which point a real broker and worker pool would be +justified, and the `JobRepository` interface is the seam that would change. diff --git a/docs/adr/0007-ffmpeg-preprocessing-boundary.md b/docs/adr/0007-ffmpeg-preprocessing-boundary.md new file mode 100644 index 0000000..6fbfdd6 --- /dev/null +++ b/docs/adr/0007-ffmpeg-preprocessing-boundary.md @@ -0,0 +1,74 @@ +# ADR 0007: ffmpeg as an Explicit Audio Preprocessing Boundary + +## Status + +Accepted (Phase 2A). Supersedes the Phase 1 position in +[PRIVACY_AND_SECURITY.md](../PRIVACY_AND_SECURITY.md) that no audio decoding library is used. + +## Context + +Browsers do not record anything whisper.cpp can read. `MediaRecorder` in Chrome and Firefox +produces Opus in a WebM container; Safari produces AAC in MP4. whisper.cpp wants raw +**16 kHz, mono, signed 16-bit PCM**. Something must decode one into the other. + +This also closes a Phase 1 gap: upload validation was a MIME whitelist plus a magic-byte check, +which confirms the first few bytes look plausible and nothing more +([PRIVACY_AND_SECURITY.md](../PRIVACY_AND_SECURITY.md)). [ROADMAP.md](../ROADMAP.md) item 3 +anticipated that real decode validation would arrive with whatever phase needed to decode anyway. + +Two smaller decisions ride along: + +- Where does the **authoritative duration** come from? Phase 1 deliberately named its column + `client_reported_duration_seconds` because it came from the browser's own stopwatch, and + [DATA_MODEL.md](../DATA_MODEL.md) said Phase 2 should derive a real one and add it as a + separate column. +- What runs the decode? Python audio libraries exist, but every one that handles WebM/Opus and + MP4/AAC is a binding around ffmpeg or libav anyway. + +## Decision + +An explicit `AudioPreprocessor` interface with an `FfmpegAudioPreprocessor` implementation, in +`backend/app/audio/`, invoked by the worker before the transcription provider is called. + +It is an **architectural boundary, not a helper function**: it is the point where an opaque blob +produced by a browser becomes a known-shaped array of samples. Its contract is narrow and +enforced in one place: + +- Decode WebM / OGG / MP4-M4A / WAV / MP3 to 16 kHz mono signed-16-bit PCM WAV. +- Report the authoritative duration, measured from the **decoded** stream. +- Write only inside a per-job temporary directory, removed on success and on failure alike. +- Never modify the original recording. +- Invoke ffmpeg as an argv array with a mandatory timeout, never through a shell. +- Never surface raw ffmpeg stderr, which is full of absolute paths, through the API. + +**Duration is read from the decoded WAV header** (frame count ÷ sample rate) rather than from a +second `ffprobe` call. It is one fewer subprocess, and it measures exactly the bytes whisper.cpp +will read — which is the number that belongs in the database and the number a real-time factor +should be divided by. + +**A format mismatch is a failure, not a warning.** If the decoded file is not 16 kHz mono 16-bit, +ffmpeg ignored the conversion flags, and handing that to the model would produce a plausible but +wrong transcript. Better to fail. + +**ffmpeg is an external system dependency, deliberately not vendored and never auto-installed.** +`scripts/setup_whisper.sh` checks for it and prints the install command; it does not run it. +Installing system software is the owner's decision. + +## Consequences + +**Gained:** real decode validation as a side effect — a file that passed the magic-byte check but +is not actually decodable now fails with a clear message instead of reaching the model. An +authoritative duration, stored alongside (not instead of) the client-reported one, so a browser +timer bug stays visible rather than becoming the truth. A single testable place for the +subprocess safety properties. Broad input-format support for free. + +**Given up:** a system dependency the owner must install, which the app cannot paper over. A full +decode pass on every recording, though it is fast relative to inference. A decoded copy of the +user's speech exists in the clear on disk for the duration of a job, which is why the workspace +is deleted on every path and why the debug flag that preserves it is off by default and logs a +warning. + +## Revisit if + +A pure-Python decoder covering WebM/Opus and MP4/AAC becomes viable without an ffmpeg binary, or +whisper.cpp gains reliable container demuxing of its own. diff --git a/docs/adr/0008-http-polling-not-websockets.md b/docs/adr/0008-http-polling-not-websockets.md new file mode 100644 index 0000000..48d3ccf --- /dev/null +++ b/docs/adr/0008-http-polling-not-websockets.md @@ -0,0 +1,61 @@ +# ADR 0008: HTTP Polling for Processing Status, Not WebSockets or SSE + +## Status + +Accepted (Phase 2A). Answers the open question in [ROADMAP.md](../ROADMAP.md). + +## Context + +Once transcription moved to a worker ([ADR 0006](0006-local-worker-sqlite-queue.md)), the +frontend needed some way to learn that a session had finished. [ROADMAP.md](../ROADMAP.md) left +this open: poll `GET /api/sessions/{id}`, or add an SSE/WebSocket endpoint. + +The actual shape of the problem: **one user, on one machine, waiting on one job, for something +between a few seconds and a couple of minutes.** The payload is a status string and a stage. The +event rate is roughly one meaningful change every several seconds. + +## Decision + +Ordinary HTTP polling, against a **narrow, dedicated endpoint** — `GET /api/sessions/{id}/status` +— rather than the full session detail. + +The dedicated endpoint matters. Polling `GET /api/sessions/{id}` once a second would drag the +transcript, metrics, feedback, and reflection across the wire on every tick to read one field. +The status payload is a handful of scalars. + +Polling behavior lives in `frontend/src/lib/processing/poller.ts`, with its clock, timers, and +fetch injected so the whole lifecycle is unit-testable: + +- Start near **1 second**, back off by 1.5× to a **5 second cap**. +- Stop on `COMPLETED`, `FAILED`, a `404` (session deleted), an overall **10-minute timeout**, or + explicit cancellation on route teardown. +- Exactly one loop per poller; a second `start()` is a no-op. +- A response arriving after `cancel()` updates nothing. + +**SSE was the closer call** and was still rejected: it is genuinely simpler than WebSockets +(one-directional, plain HTTP, auto-reconnecting in the browser), but it would add a long-lived +connection with its own server-side lifecycle, a second thing for the Vite dev proxy to get right, +and a reconnection story — all to save a handful of small requests over the ~30 seconds a typical +job takes. **WebSockets** would add all of that plus a bidirectional protocol for a problem with +no client→server messages at all. + +Neither would remove the need for polling logic anyway: both still need a fallback for the case +this design has to handle well — **the worker is not running at all**, so no event will ever be +pushed. + +## Consequences + +**Gained:** no new protocol, no connection lifecycle, no reconnection logic, no dev-proxy change. +The whole mechanism is a function that calls `fetch` on a timer, and it is fully tested with fake +timers and no network. It degrades correctly when the worker is absent: the timeout fires and the +UI explains how to start it, which a push-based design would have to special-case. + +**Given up:** completion is noticed up to ~5 seconds late in the worst case, which for a +minute-long inference is imperceptible. A few dozen small requests per job instead of one +connection — irrelevant against localhost. If SpeakLab ever showed genuinely live output (partial +transcript streaming as the model emits segments), polling would be the wrong tool. + +## Revisit if + +Streaming partial transcripts while inference runs becomes a feature, or several sessions need +watching at once — either would make a push channel worth its cost. diff --git a/docs/adr/0009-pinned-whisper-cpp-runtime.md b/docs/adr/0009-pinned-whisper-cpp-runtime.md new file mode 100644 index 0000000..619216f --- /dev/null +++ b/docs/adr/0009-pinned-whisper-cpp-runtime.md @@ -0,0 +1,75 @@ +# ADR 0009: A Pinned, External, Never-Vendored whisper.cpp Runtime + +## Status + +Accepted (Phase 2A). Implements the default set by [ADR 0004](0004-local-stt-default.md). + +## Context + +[ADR 0004](0004-local-stt-default.md) chose local whisper.cpp over a cloud STT API. That left the +practical questions: which version, built how, stored where, and identified in the data how? + +Three constraints shape the answer: + +1. **This is a public repository.** whisper.cpp source, compiled binaries, model weights, and + generated Core ML artifacts must never be committed + ([AGENTS.md](../../AGENTS.md), [PRIVACY_AND_SECURITY.md](../PRIVACY_AND_SECURITY.md)). + Model files alone run to hundreds of megabytes. +2. **Transcripts must stay comparable over time.** SpeakLab's whole point is tracking speaking + practice across months. A transcript produced by a different model or a different runtime is + not comparable to an earlier one, and if nothing records which produced which, that difference + is invisible and permanent. +3. **Inference runtimes move fast.** Tracking `master` means output quality and performance can + change under the owner without any change on their side. + +## Decision + +**Pin a stable tagged release.** Phase 2A pins **`v1.9.1`**, commit +`f049fff95a089aa9969deb009cdd4892b3e74916`, verified against the upstream +`ggml-org/whisper.cpp` releases API at the start of this work. Never `master`. + +**Build it locally into a gitignored directory**, `backend/runtime/whisper/`, via +`scripts/setup_whisper.sh`. Models go to `backend/storage/models/`. Nothing about either is +tracked, and `scripts/check_public_safety.sh` fails if anything under them becomes visible to git. + +**Record what was actually built**, in a manifest written next to the binary: release tag, real +checked-out commit, build options, the binary's own version output, model name, and model +SHA-256. If the upstream tag has moved, the setup script warns and records the *actual* commit — +the manifest describes reality, not intent. + +**Stamp every transcript with its provenance**: model name, model SHA-256, runtime version, and +the inference parameters used. The model *name* is not enough — `ggml-base.en.bin` has been +re-uploaded upstream, so the hash of the bytes is what actually pins it. + +**Never install system packages.** The setup script checks for cmake, ffmpeg, git and curl, and +prints the exact install command for anything missing before exiting. It does not touch git +config, shell profiles, PATH, or any global setting. + +**Core ML, quantization, and VAD are off.** Each is a real optimization; each also changes output, +accuracy, or both. Enabling them by default would mean shipping unmeasured quality changes. +Ordinary Metal acceleration on Apple Silicon is compiled in upstream by default and is used — it +changes speed, not results. `scripts/benchmark_models.py` exists to measure the others before any +future decision to adopt them. + +**No silent fallback to the mock provider.** If the binary or model is missing, the job fails with +a specific code and the UI shows the command that fixes it. Substituting simulated text under a +UI label reading "Real transcript" would be the single worst bug this system could have. + +## Consequences + +**Gained:** reproducible transcription — the same audio, model hash, and runtime version produce +the same output. A public repository that stays small and legally clean. Full after-the-fact +attribution of any historical transcript. Setup that is explicit and inspectable rather than +magic. + +**Given up:** the owner must run a build step that takes minutes and needs a compiler toolchain, +and must download a model separately. Upgrading whisper.cpp is a deliberate act (change the pin, +rebuild, ideally re-benchmark) rather than something that happens automatically — which is the +point, but it is friction. Transcripts produced before an upgrade remain comparable only to each +other, which the stored provenance makes visible rather than solving. + +## Revisit if + +Upstream ships prebuilt binaries for the target platforms (removing the build step), or measured +benchmarks show Core ML/quantization/VAD are worth enabling by default — at which point this ADR +gets updated rather than quietly contradicted.