diff --git a/CHANGELOG.md b/CHANGELOG.md index 50b26f8..3ec6641 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -12,6 +12,13 @@ This project follows semantic versioning once published to PyPI. producing deterministic low-level output texture. - Made `audio-super-res eval matrix` return a non-zero exit status when any matrix run fails, so remote evidence workflows cannot report success for failed stability checks. +- Corrected `load_time_seconds` and `total_elapsed_seconds` comparison direction to + `lower_is_better` alongside the other runtime metrics. + +### Added + +- Added a versioned release regression threshold policy and a Colab CLI workflow for same-backend + baseline/candidate matrix evidence. ## 0.6.0 - 2026-07-12 diff --git a/docs/ACCELERATORS.md b/docs/ACCELERATORS.md index 291253f..bd87d4e 100644 --- a/docs/ACCELERATORS.md +++ b/docs/ACCELERATORS.md @@ -207,6 +207,24 @@ Set `ASR_GIT_REF`, `ASR_LAVASR_DEVICE`, or `ASR_COLAB_ARCHIVE` in the remote exe only when validating another immutable ref, device, or evidence destination. Do not run `examples/colab_lavasr_validation.py` locally: it intentionally refuses to run outside `/content`. +For same-backend release regression evidence across Git refs, use a separate session or a fresh +runtime after the stability workflow: + +```sh +colab new -s asr-regression --gpu T4 +colab exec -s asr-regression -f examples/colab_eval_regression.py --timeout 3600 +colab download -s asr-regression \ + /content/asr-eval-regression.tar.gz \ + runs/asr-eval-regression.tar.gz +colab stop -s asr-regression +``` + +The defaults compare `v0.6.0` with `main`, share one verified LavaSR cache, and run identical +`wideband_16k`/`lowpass_4k` matrices over the deterministic eight-item smoke evalset. The archive +also contains a `sinc-resample` matrix, the threshold policy, reports, resolved commits, and the +matrix comparison. Override `ASR_BASELINE_REF` and `ASR_CANDIDATE_REF` only with reviewable refs; +recorded evidence always includes the resolved commits. + For upstream LavaSR parity, install upstream dependencies deliberately and use the environment gates in [../tests/README.md](../tests/README.md). Validate AudioSR separately because its external package owns checkpoint and accelerator behavior. diff --git a/docs/EVALUATION.md b/docs/EVALUATION.md index 6ca86ea..6de669c 100644 --- a/docs/EVALUATION.md +++ b/docs/EVALUATION.md @@ -111,6 +111,12 @@ referenced run manifests into a portable evidence directory, optionally archived Use `eval matrix --reuse-existing` to keep completed run manifests, and `--fail-fast` when a combination-level failure should stop the whole matrix instead of being recorded as a failed run. +The maintained example policy is +[`examples/artifacts/eval-threshold-policy.json`](../examples/artifacts/eval-threshold-policy.json). +It covers same-backend quality and stability regressions. It deliberately excludes RTF, elapsed +time, and memory thresholds until repeated runs establish variance for a specific device/runtime +pair. Never compare performance numbers across unlike hardware with this policy. + `eval init-speech-bwe` creates a deterministic synthetic tiny evalset for smoke and regression testing. It is useful for CI and example commands, but it is not a substitute for a licensed real speech dataset when making backend quality claims. diff --git a/examples/artifacts/README.md b/examples/artifacts/README.md index f695714..2c5616c 100644 --- a/examples/artifacts/README.md +++ b/examples/artifacts/README.md @@ -5,5 +5,8 @@ These files are small, static examples of the JSON artifacts produced by the CLI - `sample-plan-manifest.json`: output from a dry-run batch plan. - `sample-completed-manifest.json`: output from a completed enhancement run. - `sample-quality-report.json`: output from `--quality-report-json`. +- `eval-threshold-policy.json`: conservative same-backend release regression limits for matrix + comparison. It intentionally excludes performance thresholds until repeated device-specific + variance is available. They are intended for release notes, documentation, and downstream CI examples. diff --git a/examples/artifacts/eval-threshold-policy.json b/examples/artifacts/eval-threshold-policy.json new file mode 100644 index 0000000..bdc86e6 --- /dev/null +++ b/examples/artifacts/eval-threshold-policy.json @@ -0,0 +1,21 @@ +{ + "schema_version": 1, + "name": "release-regression-v1", + "description": "Conservative per-item quality and stability regression limits for same-backend release comparisons.", + "thresholds": { + "si_sdr_db": 0.5, + "sdr_db": 0.5, + "lsd_db": 0.25, + "spectral_convergence": 0.05, + "highband_lsd_4_8k": 0.25, + "highband_lsd_8_16k": 0.25, + "mcd": 0.5, + "duration_drift_seconds": 0.01, + "clipped_fraction": 0.001 + }, + "notes": [ + "Use only when baseline and candidate share backend, dataset, degrader, device class, and runtime provider.", + "Performance limits are intentionally excluded until device-specific repeated-run variance is recorded.", + "Candidate failures, stability failures, and quality failures remain regressions independently of numeric thresholds." + ] +} diff --git a/examples/colab_eval_regression.py b/examples/colab_eval_regression.py new file mode 100644 index 0000000..824faa2 --- /dev/null +++ b/examples/colab_eval_regression.py @@ -0,0 +1,310 @@ +"""Collect same-backend baseline/candidate eval evidence in a Colab runtime. + +The workflow runs all model inference remotely, compares immutable Git refs with the +maintained threshold policy, and writes one downloadable archive. It intentionally +refuses to execute outside Colab's ``/content`` workspace. +""" + +from __future__ import annotations + +import json +import os +import shutil +import subprocess +import sys +import tarfile +from datetime import datetime, timezone +from pathlib import Path + +REPO_URL = os.environ.get("ASR_REPO_URL", "https://github.com/Tinnci/python-audio-super-resolution.git") +BASELINE_REF = os.environ.get("ASR_BASELINE_REF", "v0.6.0") +CANDIDATE_REF = os.environ.get("ASR_CANDIDATE_REF", "main") +DEVICE = os.environ.get("ASR_EVAL_DEVICE", "cuda") +WORK_ROOT = Path(os.environ.get("ASR_COLAB_EVAL_ROOT", "/content/audio-super-resolution-regression")) +BASELINE_DIR = WORK_ROOT / "baseline-repo" +CANDIDATE_DIR = WORK_ROOT / "candidate-repo" +CACHE_DIR = WORK_ROOT / "models" +EVIDENCE_DIR = WORK_ROOT / "evidence" +DATASET_DIR = EVIDENCE_DIR / "evalset" +ARCHIVE_PATH = Path(os.environ.get("ASR_COLAB_EVAL_ARCHIVE", "/content/asr-eval-regression.tar.gz")) + + +def _run(name: str, command: list[str], *, cwd: Path | None = None) -> None: + log_path = EVIDENCE_DIR / "logs" / f"{name}.log" + log_path.parent.mkdir(parents=True, exist_ok=True) + print(f"[colab-regression] {name}: {' '.join(command)}", flush=True) + with log_path.open("w", encoding="utf-8") as log: + subprocess.run( + command, + cwd=cwd, + stdout=log, + stderr=subprocess.STDOUT, + text=True, + check=True, + ) + + +def _clone(name: str, ref: str, destination: Path) -> str: + _run(f"clone-{name}", ["git", "clone", "--filter=blob:none", REPO_URL, str(destination)]) + _run(f"checkout-{name}", ["git", "checkout", ref], cwd=destination) + return subprocess.run( + ["git", "rev-parse", "HEAD"], + cwd=destination, + capture_output=True, + text=True, + check=True, + ).stdout.strip() + + +def _install_project(name: str, repository: Path) -> None: + _run( + f"install-{name}", + [ + "uv", + "pip", + "install", + "--system", + "--reinstall-package", + "audio-super-resolution", + "-e", + f"{repository}[lavasr,download]", + ], + ) + + +def _matrix_command(*, backend: str, output_dir: Path) -> list[str]: + command = [ + "audio-super-res", + "eval", + "matrix", + "--dataset", + str(DATASET_DIR / "speech_clean_48k"), + "--backend", + backend, + "--degrader", + "wideband_16k", + "--degrader", + "lowpass_4k", + "--optional-metric", + "mcd", + "--output-dir", + str(output_dir), + "--device", + DEVICE if backend == "lavasr-compat" else "cpu", + ] + if backend == "lavasr-compat": + command.extend( + [ + "--runtime-provider", + "torch-eager", + "--model-cache-dir", + str(CACHE_DIR), + ] + ) + return command + + +def _write_report(name: str, manifest: Path, output: Path, *, cwd: Path) -> None: + _run( + name, + ["audio-super-res", "eval", "report", "--manifest", str(manifest), "--output", str(output)], + cwd=cwd, + ) + + +def _matrix_summary(path: Path) -> dict[str, object]: + manifest = json.loads(path.read_text(encoding="utf-8")) + return { + "passed": manifest.get("passed"), + "run_count": manifest.get("run_count"), + "runs": [ + { + "backend": run.get("backend"), + "degrader": run.get("degrader"), + "passed": run.get("passed"), + "result_count": run.get("result_count"), + "failure_count": run.get("failure_count"), + } + for run in manifest.get("runs", []) + if isinstance(run, dict) + ], + } + + +def _write_summary( + *, + status: str, + baseline_commit: str | None, + candidate_commit: str | None, + error: str | None = None, +) -> None: + summary: dict[str, object] = { + "schema_version": 1, + "created_at": datetime.now(timezone.utc).isoformat(), + "status": status, + "repository": REPO_URL, + "baseline_ref": BASELINE_REF, + "baseline_commit": baseline_commit, + "candidate_ref": CANDIDATE_REF, + "candidate_commit": candidate_commit, + "device": DEVICE, + "error": error, + } + matrix_paths = { + "sinc_baseline": EVIDENCE_DIR / "sinc-baseline" / "matrix.json", + "lavasr_baseline": EVIDENCE_DIR / "lavasr-baseline" / "matrix.json", + "lavasr_candidate": EVIDENCE_DIR / "lavasr-candidate" / "matrix.json", + } + summary["matrices"] = {name: _matrix_summary(path) for name, path in matrix_paths.items() if path.is_file()} + comparison_path = EVIDENCE_DIR / "lavasr-comparison.json" + if comparison_path.is_file(): + comparison = json.loads(comparison_path.read_text(encoding="utf-8")) + summary["comparison"] = { + "passed": comparison.get("passed"), + "run_count": comparison.get("run_count"), + "regression_count": len(comparison.get("regressions", [])), + "regressions": comparison.get("regressions", []), + } + (EVIDENCE_DIR / "summary.json").write_text(json.dumps(summary, indent=2) + "\n", encoding="utf-8") + + +def _archive() -> None: + ARCHIVE_PATH.parent.mkdir(parents=True, exist_ok=True) + with tarfile.open(ARCHIVE_PATH, "w:gz") as archive: + archive.add(EVIDENCE_DIR, arcname="evidence") + print(f"[colab-regression] evidence archive: {ARCHIVE_PATH}", flush=True) + + +def main() -> int: + if Path.cwd() != Path("/content"): + raise RuntimeError("This regression workflow must run inside a Colab runtime rooted at /content") + + if WORK_ROOT.exists(): + shutil.rmtree(WORK_ROOT) + EVIDENCE_DIR.mkdir(parents=True) + + baseline_commit: str | None = None + candidate_commit: str | None = None + error: str | None = None + try: + _run("install-uv", [sys.executable, "-m", "pip", "install", "uv"]) + baseline_commit = _clone("baseline", BASELINE_REF, BASELINE_DIR) + candidate_commit = _clone("candidate", CANDIDATE_REF, CANDIDATE_DIR) + + _install_project("candidate-bootstrap", CANDIDATE_DIR) + _run( + "init-evalset", + [ + "audio-super-res", + "eval", + "init-speech-bwe", + "--output-dir", + str(DATASET_DIR), + "--count", + "8", + "--duration-seconds", + "0.2", + ], + cwd=CANDIDATE_DIR, + ) + + _install_project("baseline", BASELINE_DIR) + _run( + "prepare-lavasr-weights", + [ + "audio-super-res", + "--backend", + "lavasr-compat", + "--model-cache-dir", + str(CACHE_DIR), + "--download-weights", + "--prepare-model-cache", + ], + cwd=BASELINE_DIR, + ) + _run( + "lavasr-baseline-matrix", + _matrix_command(backend="lavasr-compat", output_dir=EVIDENCE_DIR / "lavasr-baseline"), + cwd=BASELINE_DIR, + ) + + _install_project("candidate", CANDIDATE_DIR) + _run( + "sinc-baseline-matrix", + _matrix_command(backend="sinc-resample", output_dir=EVIDENCE_DIR / "sinc-baseline"), + cwd=CANDIDATE_DIR, + ) + _run( + "lavasr-candidate-matrix", + _matrix_command(backend="lavasr-compat", output_dir=EVIDENCE_DIR / "lavasr-candidate"), + cwd=CANDIDATE_DIR, + ) + + policy_source = CANDIDATE_DIR / "examples" / "artifacts" / "eval-threshold-policy.json" + policy_copy = EVIDENCE_DIR / "eval-threshold-policy.json" + shutil.copy2(policy_source, policy_copy) + comparison_path = EVIDENCE_DIR / "lavasr-comparison.json" + _run( + "lavasr-matrix-compare", + [ + "audio-super-res", + "eval", + "matrix-compare", + str(EVIDENCE_DIR / "lavasr-baseline" / "matrix.json"), + str(EVIDENCE_DIR / "lavasr-candidate" / "matrix.json"), + "--threshold-policy", + str(policy_copy), + "--output", + str(comparison_path), + ], + cwd=CANDIDATE_DIR, + ) + + _write_report( + "sinc-baseline-report", + EVIDENCE_DIR / "sinc-baseline" / "matrix.json", + EVIDENCE_DIR / "sinc-baseline-report.md", + cwd=CANDIDATE_DIR, + ) + _write_report( + "lavasr-baseline-report", + EVIDENCE_DIR / "lavasr-baseline" / "matrix.json", + EVIDENCE_DIR / "lavasr-baseline-report.md", + cwd=CANDIDATE_DIR, + ) + _write_report( + "lavasr-candidate-report", + EVIDENCE_DIR / "lavasr-candidate" / "matrix.json", + EVIDENCE_DIR / "lavasr-candidate-report.md", + cwd=CANDIDATE_DIR, + ) + _write_report( + "lavasr-comparison-report", + comparison_path, + EVIDENCE_DIR / "lavasr-comparison-report.md", + cwd=CANDIDATE_DIR, + ) + _write_summary( + status="passed", + baseline_commit=baseline_commit, + candidate_commit=candidate_commit, + ) + except Exception as exc: + error = f"{type(exc).__name__}: {exc}" + _write_summary( + status="failed", + baseline_commit=baseline_commit, + candidate_commit=candidate_commit, + error=error, + ) + finally: + _archive() + + if error: + print(f"[colab-regression] failed: {error}", file=sys.stderr) + return 1 + return 0 + + +if __name__ == "__main__": + raise SystemExit(main()) diff --git a/src/audio_super_resolution/evaluation.py b/src/audio_super_resolution/evaluation.py index 76ffe0c..f38fdc0 100644 --- a/src/audio_super_resolution/evaluation.py +++ b/src/audio_super_resolution/evaluation.py @@ -193,8 +193,10 @@ "wer", "cer", "endpoint_error", + "load_time_seconds", "backend_init_seconds", "elapsed_seconds", + "total_elapsed_seconds", "rtf", "peak_rss_mb", "peak_rss_delta_mb", diff --git a/tests/test_eval.py b/tests/test_eval.py index 54ee8e4..9131b39 100644 --- a/tests/test_eval.py +++ b/tests/test_eval.py @@ -108,6 +108,34 @@ def test_compare_eval_manifests_applies_threshold_directions() -> None: assert ("rtf", "lower_is_better") in regressions +def test_compare_eval_manifests_treats_all_elapsed_times_as_lower_is_better() -> None: + baseline = _eval_manifest("lavasr-compat", si_sdr=10.0, lsd=3.0) + candidate = _eval_manifest("lavasr-compat", si_sdr=10.0, lsd=3.0) + baseline["results"][0]["performance"].update( + { + "load_time_seconds": 1.0, + "total_elapsed_seconds": 2.0, + } + ) + candidate["results"][0]["performance"].update( + { + "load_time_seconds": 1.5, + "total_elapsed_seconds": 2.5, + } + ) + + comparison = compare_eval_manifests( + baseline, + candidate, + thresholds={"load_time_seconds": 0.1, "total_elapsed_seconds": 0.1}, + ) + + assert comparison["passed"] is False + regressions = {(regression["field"], regression["direction"]) for regression in comparison["regressions"]} + assert ("load_time_seconds", "lower_is_better") in regressions + assert ("total_elapsed_seconds", "lower_is_better") in regressions + + def test_cli_eval_run_and_compare(tmp_path: Path) -> None: dataset = tmp_path / "dataset" dataset.mkdir() diff --git a/tests/test_release_artifacts.py b/tests/test_release_artifacts.py index a6fdf13..81e79db 100644 --- a/tests/test_release_artifacts.py +++ b/tests/test_release_artifacts.py @@ -1,6 +1,8 @@ import json from pathlib import Path +from audio_super_resolution.evaluation import load_threshold_policy + ROOT = Path(__file__).resolve().parents[1] ARTIFACTS = ROOT / "examples" / "artifacts" @@ -31,5 +33,24 @@ def test_sample_quality_report_is_valid_json_artifact() -> None: assert report["reports"][0]["sample_rate"] == 48000 +def test_eval_threshold_policy_is_valid_release_artifact() -> None: + policy_path = ARTIFACTS / "eval-threshold-policy.json" + policy = _load_json("eval-threshold-policy.json") + + assert policy["schema_version"] == 1 + assert policy["name"] == "release-regression-v1" + assert load_threshold_policy(policy_path) == { + "si_sdr_db": 0.5, + "sdr_db": 0.5, + "lsd_db": 0.25, + "spectral_convergence": 0.05, + "highband_lsd_4_8k": 0.25, + "highband_lsd_8_16k": 0.25, + "mcd": 0.5, + "duration_drift_seconds": 0.01, + "clipped_fraction": 0.001, + } + + def _load_json(name: str) -> dict[str, object]: return json.loads((ARTIFACTS / name).read_text(encoding="utf-8"))