From 6953d7077a74a54744124dce5c670ea179b43ffd Mon Sep 17 00:00:00 2001 From: Tinnci <23432137+Tinnci@users.noreply.github.com> Date: Sun, 12 Jul 2026 15:08:52 +0800 Subject: [PATCH] Add Colab ASR downstream evidence --- CHANGELOG.md | 2 + ROADMAP.md | 6 +- docs/ACCELERATORS.md | 15 + docs/EVALUATION.md | 6 + examples/artifacts/README.md | 2 + .../asr-evaluator-whisper-tiny-en.json | 29 ++ examples/colab_asr_downstream.py | 267 ++++++++++++++++++ tests/test_release_artifacts.py | 16 ++ 8 files changed, 342 insertions(+), 1 deletion(-) create mode 100644 examples/artifacts/asr-evaluator-whisper-tiny-en.json create mode 100644 examples/colab_asr_downstream.py diff --git a/CHANGELOG.md b/CHANGELOG.md index 069c6cb..c1881c8 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -21,6 +21,8 @@ This project follows semantic versioning once published to PyPI. baseline/candidate matrix evidence. - Added a pinned, license-recorded LibriSpeech `dev-clean` tiny baseline specification and Colab-only real-speech matrix workflow. +- Added a pinned external Whisper evaluator specification and Colab-only transcript workflow for + precomputed downstream WER/CER evidence without adding an ASR package dependency. ## 0.6.0 - 2026-07-12 diff --git a/ROADMAP.md b/ROADMAP.md index 027ce85..5fff2dc 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -58,7 +58,7 @@ Next deliverables: - [x] Define a small licensed speech BWE evaluation set outside the repository. - [x] Commit a threshold-policy example for release regression use. - [x] Record baseline matrices for `sinc-resample` and gated `lavasr-compat` runs. -- [ ] Add precomputed ASR transcript evidence before integrating any real ASR runtime. +- [x] Add precomputed ASR transcript evidence before integrating any real ASR runtime. - [ ] Export one blind listening bundle and document how results map back to objective and stability tables. @@ -67,6 +67,10 @@ failures, but `sinc-resample` substantially outperformed `lavasr-compat` on full wideband-16k fidelity metrics. This prevents stable promotion from objective metrics alone and makes downstream ASR plus blind listening evidence the next required decision inputs. +Pinned Whisper `tiny.en` downstream evidence on the same eight utterances reported identical mean +WER (`0.1594`) and CER (`0.0757`) for degraded input, sinc output, and LavaSR output. LavaSR neither +improved nor degraded this small ASR slice, so blind listening remains the final P2 evidence gap. + Do not collapse quality, downstream usefulness, speed, stability, and governance into one ranking. The evaluation policy lives in [docs/EVALUATION.md](docs/EVALUATION.md). diff --git a/docs/ACCELERATORS.md b/docs/ACCELERATORS.md index 4067c6b..e03b4e8 100644 --- a/docs/ACCELERATORS.md +++ b/docs/ACCELERATORS.md @@ -241,6 +241,21 @@ retains the source license/README/speaker metadata, selects four female and four converts one utterance per speaker to 48 kHz. It then runs matching sinc/LavaSR matrices. The source archive, converted audio, and model computations never run or persist in the local repository. +To add downstream ASR evidence without introducing an ASR package dependency, run the pinned +external evaluator immediately after the LibriSpeech workflow in the same session: + +```sh +colab exec -s asr-librispeech -f examples/colab_asr_downstream.py --timeout 3600 +colab download -s asr-librispeech \ + /content/asr-downstream-evidence.tar.gz \ + runs/asr-downstream-evidence.tar.gz +``` + +The external model metadata and revision are pinned in +`examples/artifacts/asr-evaluator-whisper-tiny-en.json`. Whisper runs only in Colab and writes +precomputed transcripts. The package's normal `eval downstream` command remains lightweight and +computes WER/CER from JSON rather than importing or downloading an ASR model. + For upstream LavaSR parity, install upstream dependencies deliberately and use the environment gates in [../tests/README.md](../tests/README.md). Validate AudioSR separately because its external package owns checkpoint and accelerator behavior. diff --git a/docs/EVALUATION.md b/docs/EVALUATION.md index 2722ea0..dac9c89 100644 --- a/docs/EVALUATION.md +++ b/docs/EVALUATION.md @@ -441,6 +441,12 @@ generated high-frequency content is penalized heavily by reference-fidelity metr promotion therefore requires downstream ASR and blind listening evidence rather than interpreting matrix completion or one spectral score as proof of improvement. +Pinned external Whisper `tiny.en` transcription was then run in Colab over the `wideband_16k` +degraded, sinc-enhanced, and LavaSR-enhanced files. The package consumed only the resulting JSON. +Mean WER was `0.1594` and mean CER was `0.0757` for all three conditions, with zero sinc/LavaSR +delta on this eight-item slice. This rules out a measured ASR benefit here, but the sample is too +small to prove general neutrality; it remains a reproducible downstream smoke baseline. + ## Golden Compatibility Validation Golden validation compares a package-owned compatible backend with an upstream output or a diff --git a/examples/artifacts/README.md b/examples/artifacts/README.md index 4b347e5..08b2ea2 100644 --- a/examples/artifacts/README.md +++ b/examples/artifacts/README.md @@ -10,5 +10,7 @@ These files are small, static examples of the JSON artifacts produced by the CLI variance is available. - `librispeech-dev-clean-tiny-v1.json`: source, checksum, license, deterministic selection, and storage policy for the remote-only real-speech evaluation baseline. No dataset audio is committed. +- `asr-evaluator-whisper-tiny-en.json`: pinned external Whisper evaluator metadata for Colab-only + transcript generation. The package continues to consume precomputed transcripts only. They are intended for release notes, documentation, and downstream CI examples. diff --git a/examples/artifacts/asr-evaluator-whisper-tiny-en.json b/examples/artifacts/asr-evaluator-whisper-tiny-en.json new file mode 100644 index 0000000..1235463 --- /dev/null +++ b/examples/artifacts/asr-evaluator-whisper-tiny-en.json @@ -0,0 +1,29 @@ +{ + "schema_version": 1, + "evaluator_id": "whisper-tiny-en-pinned-v1", + "purpose": "External Colab-only ASR transcription for precomputed downstream WER/CER evidence.", + "model": { + "provider": "huggingface", + "id": "openai/whisper-tiny.en", + "revision": "87c7102498dcde7456f24cfd30239ca606ed9063", + "license": "Apache-2.0", + "task": "automatic-speech-recognition", + "language": "en" + }, + "runtime": { + "transformers_version": "5.13.1", + "device": "cuda", + "dtype": "float16", + "input_sample_rate": 16000 + }, + "normalization": { + "casefold": true, + "keep": "ASCII letters, digits, apostrophes, and spaces", + "collapse_whitespace": true + }, + "integration_policy": { + "package_dependency": false, + "run_location": "Colab CLI remote runtime", + "package_input": "Precomputed reference, baseline, and enhanced transcript JSON only" + } +} diff --git a/examples/colab_asr_downstream.py b/examples/colab_asr_downstream.py new file mode 100644 index 0000000..2665765 --- /dev/null +++ b/examples/colab_asr_downstream.py @@ -0,0 +1,267 @@ +"""Generate precomputed ASR downstream evidence in a Colab GPU runtime. + +Run this after ``colab_librispeech_eval.py`` in the same session. The external +Whisper model is used only to create transcript JSON; the package itself keeps +its lightweight transcript-only downstream evaluator and gains no ASR dependency. +""" + +from __future__ import annotations + +import json +import os +import re +import subprocess +import sys +import tarfile +from datetime import datetime, timezone +from pathlib import Path + +WORK_ROOT = Path(os.environ.get("ASR_LIBRISPEECH_ROOT", "/content/audio-super-resolution-librispeech")) +REPO_DIR = WORK_ROOT / "repo" +EVIDENCE_DIR = WORK_ROOT / "evidence" +EVALSET_DIR = EVIDENCE_DIR / "librispeech-dev-clean-tiny-v1" +MODEL_SPEC_PATH = REPO_DIR / "examples" / "artifacts" / "asr-evaluator-whisper-tiny-en.json" +ARCHIVE_PATH = Path(os.environ.get("ASR_DOWNSTREAM_ARCHIVE", "/content/asr-downstream-evidence.tar.gz")) + + +def _run(name: str, command: list[str]) -> None: + log_path = EVIDENCE_DIR / "logs" / f"{name}.log" + log_path.parent.mkdir(parents=True, exist_ok=True) + print(f"[colab-asr] {name}: {' '.join(command)}", flush=True) + with log_path.open("w", encoding="utf-8") as log: + subprocess.run( + command, + cwd=REPO_DIR, + stdout=log, + stderr=subprocess.STDOUT, + text=True, + check=True, + ) + + +def _normalize(text: str) -> str: + normalized = re.sub(r"[^a-z0-9' ]+", " ", text.casefold()) + return " ".join(normalized.split()) + + +def _load_run(backend: str) -> dict[str, dict[str, object]]: + run_id = f"{backend}__wideband_16k" + path = EVIDENCE_DIR / f"{'sinc' if backend == 'sinc-resample' else 'lavasr'}-matrix" / "runs" / f"{run_id}.json" + manifest = json.loads(path.read_text(encoding="utf-8")) + return {str(record["id"]): record for record in manifest["results"]} + + +def _load_audio(path: str, *, target_sample_rate: int): + import numpy as np + import soundfile as sf + from scipy.signal import resample_poly + + audio, sample_rate = sf.read(path, dtype="float32", always_2d=False) + if audio.ndim == 2: + audio = np.mean(audio, axis=1) + if sample_rate != target_sample_rate: + divisor = np.gcd(sample_rate, target_sample_rate) + audio = resample_poly(audio, target_sample_rate // divisor, sample_rate // divisor).astype(np.float32) + return audio + + +def _transcribe_paths(paths: list[str], model_spec: dict[str, object]) -> dict[str, str]: + import torch + from transformers import AutoModelForSpeechSeq2Seq, AutoProcessor + + model_info = model_spec["model"] + runtime = model_spec["runtime"] + if not isinstance(model_info, dict) or not isinstance(runtime, dict): + raise ValueError("ASR evaluator model and runtime specifications must be objects") + model_id = str(model_info["id"]) + revision = str(model_info["revision"]) + sample_rate = int(runtime["input_sample_rate"]) + + processor = AutoProcessor.from_pretrained(model_id, revision=revision) + model = AutoModelForSpeechSeq2Seq.from_pretrained( + model_id, + revision=revision, + dtype=torch.float16, + low_cpu_mem_usage=True, + use_safetensors=True, + ).to("cuda") + model.eval() + + transcripts: dict[str, str] = {} + for index, path in enumerate(paths, start=1): + print(f"[colab-asr] transcribing {index}/{len(paths)}: {path}", flush=True) + audio = _load_audio(path, target_sample_rate=sample_rate) + inputs = processor(audio, sampling_rate=sample_rate, return_tensors="pt") + input_features = inputs.input_features.to(device="cuda", dtype=torch.float16) + with torch.inference_mode(): + predicted_ids = model.generate(input_features) + transcripts[path] = processor.batch_decode(predicted_ids, skip_special_tokens=True)[0].strip() + return transcripts + + +def _downstream_records( + *, + references: dict[str, dict[str, object]], + baseline_results: dict[str, dict[str, object]], + enhanced_results: dict[str, dict[str, object]], + transcripts: dict[str, str], +) -> list[dict[str, object]]: + records: list[dict[str, object]] = [] + for item_id, reference in references.items(): + baseline_path = str(baseline_results[item_id]["degraded_path"]) + enhanced_path = str(enhanced_results[item_id]["enhanced_path"]) + records.append( + { + "id": item_id, + "reference_transcript": _normalize(str(reference["transcript"])), + "baseline_transcript": _normalize(transcripts[baseline_path]), + "enhanced_transcript": _normalize(transcripts[enhanced_path]), + "input_path": baseline_path, + "enhanced_path": enhanced_path, + "raw_reference_transcript": reference["transcript"], + "raw_baseline_transcript": transcripts[baseline_path], + "raw_enhanced_transcript": transcripts[enhanced_path], + } + ) + return records + + +def _aggregate(path: Path) -> dict[str, float]: + manifest = json.loads(path.read_text(encoding="utf-8")) + fields = ("baseline_wer", "wer", "wer_delta", "baseline_cer", "cer", "cer_delta") + return { + field: sum(float(record["scores"][field]) for record in manifest["records"]) / len(manifest["records"]) + for field in fields + } + + +def _archive() -> None: + with tarfile.open(ARCHIVE_PATH, "w:gz") as archive: + archive.add(EVIDENCE_DIR, arcname="evidence") + print(f"[colab-asr] evidence archive: {ARCHIVE_PATH}", flush=True) + + +def main() -> int: + if Path.cwd() != Path("/content"): + raise RuntimeError("This ASR workflow must run inside a Colab runtime rooted at /content") + if not (EVALSET_DIR / "manifest.json").is_file(): + raise FileNotFoundError("Run examples/colab_librispeech_eval.py in this session before downstream ASR") + + error: str | None = None + try: + model_spec = json.loads(MODEL_SPEC_PATH.read_text(encoding="utf-8")) + runtime = model_spec["runtime"] + if not isinstance(runtime, dict): + raise ValueError("ASR evaluator runtime specification must be an object") + _run( + "install-asr-evaluator", + [ + "uv", + "pip", + "install", + "--system", + f"transformers=={runtime['transformers_version']}", + "accelerate", + ], + ) + + dataset_manifest = json.loads((EVALSET_DIR / "manifest.json").read_text(encoding="utf-8")) + references = {str(record["id"]): record for record in dataset_manifest["records"]} + sinc_results = _load_run("sinc-resample") + lavasr_results = _load_run("lavasr-compat") + paths = sorted( + { + *(str(record["degraded_path"]) for record in sinc_results.values()), + *(str(record["enhanced_path"]) for record in sinc_results.values()), + *(str(record["enhanced_path"]) for record in lavasr_results.values()), + } + ) + transcripts = _transcribe_paths(paths, model_spec) + precomputed_path = EVIDENCE_DIR / "asr-precomputed-transcripts.json" + precomputed_path.write_text( + json.dumps( + { + "schema_version": 1, + "created_at": datetime.now(timezone.utc).isoformat(), + "dataset_id": dataset_manifest["dataset_id"], + "evaluator": model_spec, + "transcripts": transcripts, + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + + downstream_paths: dict[str, Path] = {} + for name, enhanced_results in (("sinc", sinc_results), ("lavasr", lavasr_results)): + dataset_path = EVIDENCE_DIR / f"asr-{name}-dataset.json" + dataset_path.write_text( + json.dumps( + { + "schema_version": 1, + "dataset_id": f"{dataset_manifest['dataset_id']}-{name}-asr", + "records": _downstream_records( + references=references, + baseline_results=sinc_results, + enhanced_results=enhanced_results, + transcripts=transcripts, + ), + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + output_path = EVIDENCE_DIR / f"asr-{name}-downstream.json" + _run( + f"evaluate-asr-{name}", + [ + "audio-super-res", + "eval", + "downstream", + "--dataset", + str(dataset_path), + "--output", + str(output_path), + ], + ) + downstream_paths[name] = output_path + + summary = { + "schema_version": 1, + "created_at": datetime.now(timezone.utc).isoformat(), + "status": "passed", + "dataset_id": dataset_manifest["dataset_id"], + "record_count": len(references), + "evaluator": model_spec, + "conditions": {name: _aggregate(path) for name, path in downstream_paths.items()}, + "error": None, + } + (EVIDENCE_DIR / "asr-summary.json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") + except Exception as exc: + error = f"{type(exc).__name__}: {exc}" + (EVIDENCE_DIR / "asr-summary.json").write_text( + json.dumps( + { + "schema_version": 1, + "created_at": datetime.now(timezone.utc).isoformat(), + "status": "failed", + "error": error, + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + finally: + _archive() + + if error: + print(f"[colab-asr] failed: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/tests/test_release_artifacts.py b/tests/test_release_artifacts.py index 3ad9f5a..0c3cfc5 100644 --- a/tests/test_release_artifacts.py +++ b/tests/test_release_artifacts.py @@ -68,5 +68,21 @@ def test_librispeech_evalset_spec_is_pinned_and_remote_only() -> None: assert storage_policy["run_location"] == "Colab CLI remote runtime" +def test_asr_evaluator_spec_is_pinned_and_not_a_package_dependency() -> None: + spec = _load_json("asr-evaluator-whisper-tiny-en.json") + model = spec["model"] + runtime = spec["runtime"] + integration_policy = spec["integration_policy"] + + assert spec["evaluator_id"] == "whisper-tiny-en-pinned-v1" + assert model["id"] == "openai/whisper-tiny.en" + assert model["revision"] == "87c7102498dcde7456f24cfd30239ca606ed9063" + assert model["license"] == "Apache-2.0" + assert runtime["transformers_version"] == "5.13.1" + assert runtime["device"] == "cuda" + assert integration_policy["package_dependency"] is False + assert integration_policy["package_input"] == "Precomputed reference, baseline, and enhanced transcript JSON only" + + def _load_json(name: str) -> dict[str, object]: return json.loads((ARTIFACTS / name).read_text(encoding="utf-8"))