From e5697702cff8301d96685ce7a6522bb35516ac24 Mon Sep 17 00:00:00 2001 From: Tinnci <23432137+Tinnci@users.noreply.github.com> Date: Sun, 12 Jul 2026 15:44:45 +0800 Subject: [PATCH] Add Colab listening evidence export --- CHANGELOG.md | 1 + ROADMAP.md | 9 +- docs/ACCELERATORS.md | 13 +++ docs/EVALUATION.md | 6 ++ examples/colab_listening_export.py | 150 +++++++++++++++++++++++++++++ 5 files changed, 177 insertions(+), 2 deletions(-) create mode 100644 examples/colab_listening_export.py diff --git a/CHANGELOG.md b/CHANGELOG.md index c1881c8..79054c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -23,6 +23,7 @@ This project follows semantic versioning once published to PyPI. Colab-only real-speech matrix workflow. - Added a pinned external Whisper evaluator specification and Colab-only transcript workflow for precomputed downstream WER/CER evidence without adding an ASR package dependency. +- Added a Colab-only verified MUSHRA bundle export for the licensed LibriSpeech sinc/LavaSR matrix. ## 0.6.0 - 2026-07-12 diff --git a/ROADMAP.md b/ROADMAP.md index 5fff2dc..9b088d4 100644 --- a/ROADMAP.md +++ b/ROADMAP.md @@ -49,7 +49,7 @@ Promotion criteria: - passing stability and parity evidence on the supported runtime path; - known limitations visible in model listings. -## Priority 2: Produce Real Evaluation Evidence +## Completed: Produce Real Evaluation Evidence The harness is implemented; the missing value is a repeatable evidence set. @@ -59,7 +59,7 @@ Next deliverables: - [x] Commit a threshold-policy example for release regression use. - [x] Record baseline matrices for `sinc-resample` and gated `lavasr-compat` runs. - [x] Add precomputed ASR transcript evidence before integrating any real ASR runtime. -- [ ] Export one blind listening bundle and document how results map back to objective and stability +- [x] Export one blind listening bundle and document how results map back to objective and stability tables. First real-speech finding: the eight-speaker LibriSpeech `dev-clean` T4 run had no stability @@ -71,6 +71,11 @@ Pinned Whisper `tiny.en` downstream evidence on the same eight utterances report WER (`0.1594`) and CER (`0.0757`) for degraded input, sinc output, and LavaSR output. LavaSR neither improved nor degraded this small ASR slice, so blind listening remains the final P2 evidence gap. +The deterministic MUSHRA export contains eight trials and 32 stimuli: reference, degraded anchor, +sinc output, and LavaSR output for each utterance. Public trial entries expose only blind IDs and +generic paths; role/backend/source mapping remains in the separately hashed answer key. Collecting +human ratings is a follow-up study, not a prerequisite for completing the export workflow. + Do not collapse quality, downstream usefulness, speed, stability, and governance into one ranking. The evaluation policy lives in [docs/EVALUATION.md](docs/EVALUATION.md). diff --git a/docs/ACCELERATORS.md b/docs/ACCELERATORS.md index e03b4e8..640d604 100644 --- a/docs/ACCELERATORS.md +++ b/docs/ACCELERATORS.md @@ -256,6 +256,19 @@ The external model metadata and revision are pinned in precomputed transcripts. The package's normal `eval downstream` command remains lightweight and computes WER/CER from JSON rather than importing or downloading an ASR model. +Export the blind listening bundle from the same real-speech matrices before stopping the session: + +```sh +colab exec -s asr-librispeech -f examples/colab_listening_export.py --timeout 900 +colab download -s asr-librispeech \ + /content/asr-listening-evidence.tar.gz \ + runs/asr-listening-evidence.tar.gz +``` + +The bundle uses MUSHRA metadata with seed `0` and four stimuli per trial: reference, degraded +anchor, sinc output, and LavaSR output. Public stimuli contain only blind IDs and generic paths; +backend, role, and source mapping remain in the separate answer key. + For upstream LavaSR parity, install upstream dependencies deliberately and use the environment gates in [../tests/README.md](../tests/README.md). Validate AudioSR separately because its external package owns checkpoint and accelerator behavior. diff --git a/docs/EVALUATION.md b/docs/EVALUATION.md index dac9c89..f8687e5 100644 --- a/docs/EVALUATION.md +++ b/docs/EVALUATION.md @@ -447,6 +447,12 @@ Mean WER was `0.1594` and mean CER was `0.0757` for all three conditions, with z delta on this eight-item slice. This rules out a measured ASR benefit here, but the sample is too small to prove general neutrality; it remains a reproducible downstream smoke baseline. +The matching MUSHRA export uses seed `0` and includes eight trials with four stimuli each: +reference, degraded anchor, sinc output, and LavaSR output. The verified public trial stimuli contain +only `blind_id` and generic `path`; 8 reference, 8 anchor, and 16 system mappings remain in the +external answer key. The downloaded bundle records SHA256 for both manifests so study results can +be tied back to the exact blind ordering. + ## Golden Compatibility Validation Golden validation compares a package-owned compatible backend with an upstream output or a diff --git a/examples/colab_listening_export.py b/examples/colab_listening_export.py new file mode 100644 index 0000000..1ce4a6a --- /dev/null +++ b/examples/colab_listening_export.py @@ -0,0 +1,150 @@ +"""Export and verify the LibriSpeech blind listening bundle in Colab. + +Run this after ``colab_librispeech_eval.py`` in the same remote session. The +script copies the reference, degraded anchor, sinc output, and LavaSR output into +deterministically blinded stimuli and writes a compact downloadable archive. +""" + +from __future__ import annotations + +import hashlib +import json +import os +import subprocess +import sys +import tarfile +from collections import Counter +from datetime import datetime, timezone +from pathlib import Path + +WORK_ROOT = Path(os.environ.get("ASR_LIBRISPEECH_ROOT", "/content/audio-super-resolution-librispeech")) +REPO_DIR = WORK_ROOT / "repo" +EVIDENCE_DIR = WORK_ROOT / "evidence" +LISTENING_DIR = EVIDENCE_DIR / "listening-mushra" +ARCHIVE_PATH = Path(os.environ.get("ASR_LISTENING_ARCHIVE", "/content/asr-listening-evidence.tar.gz")) + + +def _run(command: list[str]) -> None: + print(f"[colab-listening] {' '.join(command)}", flush=True) + subprocess.run(command, cwd=REPO_DIR, text=True, check=True) + + +def _sha256(path: Path) -> str: + digest = hashlib.sha256() + with path.open("rb") as source: + for chunk in iter(lambda: source.read(1024 * 1024), b""): + digest.update(chunk) + return digest.hexdigest() + + +def _verify_bundle() -> dict[str, object]: + public_path = LISTENING_DIR / "listening_manifest.json" + answer_path = LISTENING_DIR / "answer_key.json" + public = json.loads(public_path.read_text(encoding="utf-8")) + answer = json.loads(answer_path.read_text(encoding="utf-8")) + if public["trial_count"] != len(public["trials"]): + raise ValueError("public listening trial count does not match its trial list") + if len(public["trials"]) != len(answer["trials"]): + raise ValueError("public and answer-key trial counts differ") + + public_stimuli = [stimulus for trial in public["trials"] for stimulus in trial["stimuli"]] + answer_stimuli = [stimulus for trial in answer["trials"] for stimulus in trial["stimuli"]] + if any(set(stimulus) != {"blind_id", "path"} for stimulus in public_stimuli): + raise ValueError("public stimuli leaked fields beyond blind_id and path") + if {stimulus["blind_id"] for stimulus in public_stimuli} != {stimulus["blind_id"] for stimulus in answer_stimuli}: + raise ValueError("public and answer-key blind IDs differ") + + role_counts = Counter(str(stimulus["role"]) for stimulus in answer_stimuli) + backend_counts = Counter(str(stimulus["backend"]) for stimulus in answer_stimuli) + expected_roles = {"reference": len(public["trials"]), "anchor": len(public["trials"]), "system": 16} + if dict(role_counts) != expected_roles: + raise ValueError(f"unexpected listening roles: {dict(role_counts)}") + if backend_counts["sinc-resample"] != 8 or backend_counts["lavasr-compat"] != 8: + raise ValueError(f"unexpected backend counts: {dict(backend_counts)}") + + stimuli = sorted((LISTENING_DIR / "stimuli").glob("*.wav")) + if len(stimuli) != len(public_stimuli): + raise ValueError("stimulus file count does not match the public manifest") + return { + "schema_version": 1, + "created_at": datetime.now(timezone.utc).isoformat(), + "status": "passed", + "protocol": public["protocol"], + "seed": public["seed"], + "trial_count": len(public["trials"]), + "stimulus_count": len(public_stimuli), + "stimuli_per_trial": sorted({len(trial["stimuli"]) for trial in public["trials"]}), + "role_counts": dict(role_counts), + "backend_counts": dict(backend_counts), + "public_stimulus_fields": ["blind_id", "path"], + "answer_key_external": public["answer_key_external"], + "rating_dimensions": public["rating_dimensions"], + "manifest_sha256": _sha256(public_path), + "answer_key_sha256": _sha256(answer_path), + "error": None, + } + + +def _archive() -> None: + with tarfile.open(ARCHIVE_PATH, "w:gz") as archive: + archive.add(LISTENING_DIR, arcname="listening-mushra") + archive.add(EVIDENCE_DIR / "listening-summary.json", arcname="listening-summary.json") + print(f"[colab-listening] evidence archive: {ARCHIVE_PATH}", flush=True) + + +def main() -> int: + if Path.cwd() != Path("/content"): + raise RuntimeError("This listening workflow must run inside a Colab runtime rooted at /content") + sinc_manifest = EVIDENCE_DIR / "sinc-matrix" / "runs" / "sinc-resample__wideband_16k.json" + lavasr_manifest = EVIDENCE_DIR / "lavasr-matrix" / "runs" / "lavasr-compat__wideband_16k.json" + if not sinc_manifest.is_file() or not lavasr_manifest.is_file(): + raise FileNotFoundError("Run examples/colab_librispeech_eval.py before listening export") + + error: str | None = None + try: + _run( + [ + "audio-super-res", + "eval", + "listening-export", + "--manifest", + str(sinc_manifest), + "--manifest", + str(lavasr_manifest), + "--output-dir", + str(LISTENING_DIR), + "--protocol", + "mushra", + "--seed", + "0", + ] + ) + summary = _verify_bundle() + (EVIDENCE_DIR / "listening-summary.json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") + except Exception as exc: + error = f"{type(exc).__name__}: {exc}" + (EVIDENCE_DIR / "listening-summary.json").write_text( + json.dumps( + { + "schema_version": 1, + "created_at": datetime.now(timezone.utc).isoformat(), + "status": "failed", + "error": error, + }, + indent=2, + ) + + "\n", + encoding="utf-8", + ) + finally: + if LISTENING_DIR.is_dir(): + _archive() + + if error: + print(f"[colab-listening] failed: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main())