test: make plugin test results mean something

plugins/baseball-scoreboard/test_baseball_plugin.py

@@ -384,13 +384,38 @@ def main():
 
 
 if __name__ == "__main__":
+    # Pre-flight. This script drives a real or emulated matrix and loads the
+    # core's config template relative to the working directory, so outside a
+    # LEDMatrix checkout it cannot run at all. Skip (exit 2) rather than fail
+    # (exit 1): reporting "not applicable" as a regression is what trained
+    # everyone to ignore this plugin's results. See
+    # scripts/run_plugin_tests.py for the exit-code convention.
+    # The runner passes the core checkout in LEDMATRIX_CORE; fall back to the
+    # working directory so a manual run from a core tree still works.
+    _core_root = os.environ.get("LEDMATRIX_CORE", "")
+    if not os.path.exists(os.path.join(_core_root, "config", "config.template.json")):
+        print("SKIP: needs the core config template — set LEDMATRIX_CORE or "
+              "run from a LEDMatrix checkout (looked in %s)"
+              % (_core_root or os.getcwd()))
+        sys.exit(2)
+
+    if _core_root:
+        # This script drives the core's DisplayManager and CacheManager, and
+        # they resolve config/ and assets/ relative to the working directory.
+        # The runner starts us in the plugin directory, so move to the core --
+        # the plugin itself is already importable via sys.path above.
+        os.chdir(_core_root)
+
     try:
         success = main()
         sys.exit(0 if success else 1)
     except KeyboardInterrupt:
         print("\n\n[STOP] Tests interrupted by user")
         sys.exit(1)
     except Exception as e:
+        if "fallback mode" in str(e).lower():
+            print(f"SKIP: needs a real or emulated LED matrix ({e})")
+            sys.exit(2)
         print(f"\n[FAIL] Unexpected error: {e}")
         import traceback
         traceback.print_exc()

plugins/baseball-scoreboard/test_test_mode_live_games.py

@@ -80,11 +80,16 @@ def fake_test_update():
     live._fetch_data = fake_fetch
     live._test_mode_update = fake_test_update
 
-    # Guard update(): without the fix it enters the live-fetch path and trips over
-    # attributes a bare instance doesn't have. Suppress only that expected
-    # AttributeError so an unrelated failure still surfaces; the invariants below
-    # assert the real behavior regardless of how far the buggy path gets.
-    with contextlib.suppress(AttributeError):
+    # Guard update() on two expected exits, and only those, so an unrelated
+    # failure still surfaces:
+    #   _StopAfterTestUpdate - the fixed path; fake_test_update raises it to stop
+    #     update() right after the short-circuit branch is taken.
+    #   AttributeError       - the buggy path; it enters the live fetch and trips
+    #     over attributes a bare instance doesn't have.
+    # The invariants below assert the real behaviour either way. Suppressing only
+    # AttributeError (as this did) meant the sentinel escaped uncaught and the
+    # test died before reaching a single assertion.
+    with contextlib.suppress(AttributeError, _StopAfterTestUpdate):
         live.update()
 
     assert called["test_update"], "expected _test_mode_update() to run in test mode"

plugins/football-scoreboard/test_football_plugin.py

@@ -423,5 +423,20 @@ def main():
         return 1
 
 
+def _skip_if_not_interactive(reason):
+    """Exit 2 (skip) rather than 1 (fail) when this script's prerequisites are
+    absent. It drives a real/emulated LED matrix and prompts on stdin, so a
+    headless or non-tty run is "not applicable", not "broken" -- and reporting
+    it as a failure trained everyone to ignore this plugin's test results.
+    See scripts/run_plugin_tests.py for the exit-code convention.
+    """
+    import sys as _sys
+    print(f"SKIP: {reason}")
+    _sys.exit(2)
+
+
 if __name__ == "__main__":
+    if not sys.stdin.isatty():
+        _skip_if_not_interactive(
+            "needs an interactive terminal (this script prompts on stdin)")
     sys.exit(main())

scripts/run_plugin_tests.py

@@ -0,0 +1,134 @@
+#!/usr/bin/env python3
+"""Run a plugin's `test_*.py` scripts and report pass / skip / fail honestly.
+
+    <core-venv>/bin/python scripts/run_plugin_tests.py baseball-scoreboard \
+        --core /path/to/LEDMatrix
+    <core-venv>/bin/python scripts/run_plugin_tests.py --all --core /path/to/LEDMatrix
+
+Invoke it with the interpreter that has the plugins' dependencies -- the core
+checkout's venv. Scripts are spawned with `sys.executable`, so running this
+under a bare `python3` that lacks e.g. pytz reports every suite as failed, when
+the only thing wrong is the interpreter. `--core` sets PYTHONPATH; it does not
+change which Python runs.
+
+## Why this exists
+
+These are standalone scripts, not a pytest suite, and they signal only through
+an exit code. Without a shared convention every non-zero exit looks the same,
+so a script that is *not applicable* here — it wants a tty, or an LED matrix,
+or a font that ships with the core — was indistinguishable from a real
+regression.
+
+That cost real time: during one session the same seven "failures" were
+re-baselined three separate times to prove a change hadn't caused them. Three
+of the seven were only a missing `RGBMatrixEmulator` in the runner's
+virtualenv, one was a deliberate skip, two were interactive scripts, and
+exactly one was a genuinely broken test — which had been silently
+non-functional, dying before its first assertion, for as long as it had been
+"failing".
+
+Noise that everyone learns to ignore is worse than no signal at all, because a
+real regression hides in it.
+
+## The convention
+
+    0  pass
+    2  skip — prerequisites absent (no tty, no matrix, no font). Not a failure.
+    1  fail — a genuine problem. Anything else is treated as a failure too.
+
+Scripts opt into skipping by printing `SKIP: <reason>` and exiting 2.
+"""
+
+from __future__ import annotations
+
+import argparse
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+REPO_ROOT = Path(__file__).resolve().parents[1]
+PLUGINS = REPO_ROOT / "plugins"
+
+PASS, FAIL, SKIP = 0, 1, 2
+
+
+def run_one(script: Path, core: Path | None, timeout: int) -> tuple[int, str]:
+    env = dict(os.environ)
+    if core:
+        core = core.resolve()
+        env["PYTHONPATH"] = f"{core}{os.pathsep}{env.get('PYTHONPATH', '')}"
+        # Children run with cwd set to the plugin directory, so a script that
+        # resolves a core asset relatively -- config/config.template.json, a
+        # bundled font -- looks in the wrong place and skips even though a core
+        # was supplied. PYTHONPATH alone cannot tell it where the core is.
+        # LEDMATRIX_CORE is that contract, and it is absolute.
+        env["LEDMATRIX_CORE"] = str(core)
+    try:
+        proc = subprocess.run(
+            [sys.executable, script.name],
+            cwd=script.parent, env=env, capture_output=True,
+            text=True, timeout=timeout, stdin=subprocess.DEVNULL,
+        )
+    except subprocess.TimeoutExpired:
+        return FAIL, f"timed out after {timeout}s"
+
+    if proc.returncode == SKIP:
+        for line in proc.stdout.splitlines():
+            if line.startswith("SKIP:"):
+                return SKIP, line[len("SKIP:"):].strip()
+        return SKIP, "skipped"
+    if proc.returncode == PASS:
+        return PASS, ""
+
+    tail = [ln for ln in (proc.stdout + proc.stderr).splitlines() if ln.strip()]
+    return FAIL, (tail[-1][:120] if tail else f"exit {proc.returncode}")
+
+
+def main() -> int:
+    ap = argparse.ArgumentParser(description=__doc__)
+    ap.add_argument("plugin_ids", nargs="*")
+    ap.add_argument("--all", action="store_true", help="Every plugin")
+    ap.add_argument("--core", type=Path, default=None,
+                    help="Path to a LEDMatrix checkout to put on PYTHONPATH")
+    ap.add_argument("--timeout", type=int, default=180)
+    args = ap.parse_args()
+
+    ids = (sorted(p.name for p in PLUGINS.iterdir() if p.is_dir())
+           if args.all else args.plugin_ids)
+    if not ids:
+        ap.error("give plugin ids or --all")
+
+    # A typo used to look like success: no scripts found, nothing run, exit 0.
+    unknown = [pid for pid in ids if not (PLUGINS / pid).is_dir()]
+    if unknown:
+        print(f"Unknown plugin id(s): {', '.join(unknown)}", file=sys.stderr)
+        return 1
+
+    totals = {PASS: 0, SKIP: 0, FAIL: 0}
+    failures: list[str] = []
+
+    for pid in ids:
+        scripts = sorted((PLUGINS / pid).glob("test_*.py"))
+        if not scripts:
+            continue
+        print(f"\n{pid}")
+        for script in scripts:
+            status, detail = run_one(script, args.core, args.timeout)
+            totals[status] += 1
+            label = {PASS: "pass", SKIP: "SKIP", FAIL: "FAIL"}[status]
+            print(f"  [{label}] {script.name}" + (f" -- {detail}" if detail else ""))
+            if status == FAIL:
+                failures.append(f"{pid}/{script.name}: {detail}")
+
+    print(f"\n{totals[PASS]} passed, {totals[SKIP]} skipped, {totals[FAIL]} failed")
+    if failures:
+        print("\nFailures:", file=sys.stderr)
+        for f in failures:
+            print(f"  - {f}", file=sys.stderr)
+        return 1
+    return 0
+
+
+if __name__ == "__main__":
+    sys.exit(main())