From 92f1412b320b4bed939cfc9ea3c7a11fa04f5b77 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Thu, 3 Sep 2026 22:50:10 +0000 Subject: [PATCH 1/4] unsloth: prove each shipped feature actually runs, not just that it merged pin_contract.py proves the merge did not lose a pin's code. That is a different question from whether the feature works, and neither implies the other. The Inkling banded-attention kernel merged against upstream's sparse attention is thirteen hunks of CUDA template parameter threading where a mistake gives wrong attention output, and every static check passes. feature-checks.json says, per feature, what proves it. Keyed by FEATURE with the pin as a binding, and not the other way round: when upstream absorbs a feature the pin is deleted, and deleting the check with it would put the blind spot back somewhere else. The feature is still in the release, it just arrives through the base tag now. The one thing this file has to get right is that a pass is positive evidence. Both harnesses exit 0 having done nothing: test-llama-archs -a diffusion-gemma excluded, prints SKIP, exits 0 test-backend-ops test -o TYPO matches nothing, exits 0 so every probe rejects skip markers and requires a non-zero count of cases it ran. Without that the whole layer is decoration, and the tests are mostly about this rather than about the happy path. No runner in the prebuild pipeline has a GPU, and test-backend-ops compares a backend against CPU, so with no accelerator it skips every backend and proves nothing. Those probes are DEFERRED, named and counted, never quietly passed: ok inkling: 1/2 device rows decoded and matched CPU; projector registry intact over 194 assertions [1 needs a GPU] all 6 features demonstrated on CPU, 6 check(s) need a GPU and were not run With --gpu on a B200, against the fully merged tree: ok glm5next: 2/3 device rows decoded and matched CPU; 144/144 cases ok inkling: 2/3 device rows; 13/13 cases; projector registry over 194 assertions ok iq1-narrow-grids: 13/13; 13/13; 13/13 cases matched the CPU reference ok kimi-k3: 2/3 device rows decoded and matched CPU; registry intact ok qwen4exp-mtp: 2/3 device rows decoded and matched CPU; 4/4 cases all 6 features demonstrated on GPU Every pin must be in `features` or in `unchecked`, and the lint fails otherwise. `unchecked` is a recorded reason, not a hole: it is how diffusion-gemma went five weeks with no coverage and no record of it. The lint earned its keep immediately by catching that the manifest still named #107 after the pin moved to #177. The preflight runs the matrix after its compile gate, appending to the same PROBLEMS accumulator that already feeds the alert. --- .github/workflows/unsloth-pin-preflight.yml | 22 ++- .github/workflows/unsloth-pr-set-lint.yml | 33 +++- scripts/unsloth/feature-checks.json | 89 +++++++++ scripts/unsloth/feature_matrix.py | 200 ++++++++++++++++++++ scripts/unsloth/test_feature_matrix.py | 153 +++++++++++++++ 5 files changed, 493 insertions(+), 4 deletions(-) create mode 100644 scripts/unsloth/feature-checks.json create mode 100644 scripts/unsloth/feature_matrix.py create mode 100644 scripts/unsloth/test_feature_matrix.py diff --git a/.github/workflows/unsloth-pin-preflight.yml b/.github/workflows/unsloth-pin-preflight.yml index 3f3318904e8e..09c91b9d9090 100644 --- a/.github/workflows/unsloth-pin-preflight.yml +++ b/.github/workflows/unsloth-pin-preflight.yml @@ -188,12 +188,28 @@ jobs: # build_attn_sparse still called the old signature. Nothing above # can see that. CPU only and the `llama` target only, which is where # that translation unit lives; 59s cold at -j4 with no ccache. + GATE_OK=1 if ! cmake -B "${RUNNER_TEMP}/gate" -DCMAKE_BUILD_TYPE=Release \ - -DGGML_CUDA=OFF -DLLAMA_BUILD_TESTS=OFF -DLLAMA_BUILD_SERVER=OFF \ - -DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_BUILD_TOOLS=OFF -DLLAMA_CURL=OFF > /dev/null \ - || ! cmake --build "${RUNNER_TEMP}/gate" --target llama -j "$(nproc)" ; then + -DGGML_CUDA=OFF -DLLAMA_BUILD_TESTS=ON -DLLAMA_BUILD_SERVER=OFF \ + -DLLAMA_BUILD_EXAMPLES=OFF -DLLAMA_CURL=OFF > /dev/null \ + || ! cmake --build "${RUNNER_TEMP}/gate" -j "$(nproc)" \ + --target llama test-llama-archs test-backend-ops test-mtmd-impl ; then + GATE_OK= PROBLEMS="${PROBLEMS}- the pins merge cleanly and the merged tree does not compile. See the run log for the file and line; this is the failure that only shows up in the CUDA leg once the nightly has fanned out.\n" fi + + # The last question, and the only one that needs a binary: does each + # feature we ship still work. Everything above is about the source. + # CPU only, because no runner in this pipeline has a GPU -- see the + # note in feature_matrix.py about what that does and does not prove. + if [ -n "$GATE_OK" ]; then + if ! python3 ../scripts/unsloth/feature_matrix.py \ + --build-dir "${RUNNER_TEMP}/gate" \ + --feature-checks ../scripts/unsloth/feature-checks.json \ + --report "${RUNNER_TEMP}/feature_matrix.json" ; then + PROBLEMS="${PROBLEMS}- the merged tree compiles and a feature we ship could not be shown to work. See the run log for which feature and which probe.\n" + fi + fi fi if [ -z "$PROBLEMS" ]; then diff --git a/.github/workflows/unsloth-pr-set-lint.yml b/.github/workflows/unsloth-pr-set-lint.yml index c4c5b1771a56..8a898539f022 100644 --- a/.github/workflows/unsloth-pr-set-lint.yml +++ b/.github/workflows/unsloth-pr-set-lint.yml @@ -120,13 +120,44 @@ jobs: scripts/unsloth/test_merge_checks.py \ scripts/unsloth/test_sync_deletes.py \ scripts/unsloth/test_carry_vintage.py \ - scripts/unsloth/test_pin_contract.py; do + scripts/unsloth/test_pin_contract.py \ + scripts/unsloth/test_feature_matrix.py; do echo "::group::$t" python3 "$t" || fail=1 echo "::endgroup::" done exit "$fail" + # A pin nobody decided about is the failure this whole file exists to stop. + # Being in `unchecked` with a reason is a fine answer; being in neither map + # is how DiffusionGemma went five weeks with no coverage and no record of it. + - name: Every pin is either checked or knowingly unchecked + run: | + set -euo pipefail + python3 - <<'PY' + import json, re, sys + pins = json.load(open("scripts/unsloth/pr-set.json"))["prs"] + doc = json.load(open("scripts/unsloth/feature-checks.json")) + owned = {f["owner"] for f in doc["features"].values() if f.get("owner")} + known = owned | set(doc.get("unchecked", {})) + fail = 0 + for entry in pins: + url = entry if isinstance(entry, str) else entry["url"] + m = re.match(r"https://github\.com/([^/]+)/llama\.cpp/pull/(\d+)/", url) + pin = f"{m.group(1)}#{m.group(2)}" + if pin not in known: + print(f"::error file=scripts/unsloth/feature-checks.json::{pin} is pinned " + "and appears in neither `features` nor `unchecked`; say which it is") + fail = 1 + for pin in sorted(owned & set(doc.get("unchecked", {}))): + print(f"::error file=scripts/unsloth/feature-checks.json::{pin} is in both " + "`features` and `unchecked`") + fail = 1 + print(f"{len(pins)} pin(s), {len(owned)} with a feature check, " + f"{len(doc.get('unchecked', {}))} knowingly unchecked") + sys.exit(fail) + PY + # An over-limit run: script makes the whole file uncompilable, and nothing else sees it: yaml, actionlint and GitHub's own parser all pass it. See check_workflow_scalars.py. - name: Check no workflow string is near GitHub's size limit run: python3 scripts/unsloth/check_workflow_scalars.py --root . diff --git a/scripts/unsloth/feature-checks.json b/scripts/unsloth/feature-checks.json new file mode 100644 index 000000000000..229d41ad3d0e --- /dev/null +++ b/scripts/unsloth/feature-checks.json @@ -0,0 +1,89 @@ +{ + "_doc": [ + "The test that proves each shipped feature works. Read by feature_matrix.py.", + "", + "Keyed by FEATURE, with the pin that currently carries it, and NOT the other", + "way round. When upstream absorbs a feature the pin is deleted, and deleting", + "the check with it would put the blind spot back somewhere else: the feature", + "is still in the release, it just arrives through the base tag now. So an", + "entry outlives its `owner`, and `owner` becomes null rather than the entry", + "being removed.", + "", + "This is the half that cannot be derived. pin_contract.py reads a pin's own", + "diff and proves the merge kept it, which needs no upkeep but can only ever", + "prove the MERGE lost nothing -- a regression inside the pin regenerates a", + "smaller contract that passes. What a feature has to DO is a human sentence.", + "", + "Every pin in pr-set.json must appear in `features` or in `unchecked`. The", + "lint enforces that, so adding a pin forces a decision instead of a silence.", + "`unchecked` is a recorded reason, not a hole.", + "", + "kinds:", + " arch test-llama-archs -a builds a synthetic model of", + " the architecture, decodes 128", + " tokens on every device and", + " compares against CPU", + " backend-op test-backend-ops test -o runs the op against the CPU", + " reference implementation", + " mtmd test-mtmd-impl projector registry, no model", + "", + "A probe that exits 0 having run nothing is a failure, not a pass: both", + "harnesses do exactly that for an excluded arch or a misspelled op name.", + "feature_matrix.py rejects skip markers and requires a non-zero case count." + ], + "schema": 1, + "features": { + "inkling": { + "owner": "unslothai#172", + "checks": [ + { "kind": "arch", "arch": "inkling" }, + { "kind": "backend-op", "op": "FLASH_ATTN_EXT_BANDED" }, + { "kind": "mtmd", "projector": "inkling" } + ] + }, + "glm5next": { + "owner": "unslothai#173", + "checks": [ + { "kind": "arch", "arch": "glm5next" }, + { "kind": "backend-op", "op": "LIGHTNING_INDEXER" } + ] + }, + "kimi-k3": { + "owner": "unslothai#70", + "checks": [ + { "kind": "arch", "arch": "kimi-k3" }, + { "kind": "mtmd", "projector": "kimik3" } + ] + }, + "iq1-narrow-grids": { + "owner": "unslothai#171", + "checks": [ + { "kind": "backend-op", "op": "MUL_MAT", "params": "type_a=iq1_xs" }, + { "kind": "backend-op", "op": "MUL_MAT", "params": "type_a=iq1_xxs" }, + { "kind": "backend-op", "op": "MUL_MAT", "params": "type_a=iq1_xxxs" } + ] + }, + "qwen4exp-mtp": { + "owner": "unslothai#144", + "checks": [ + { "kind": "arch", "arch": "qwen4exp" }, + { "kind": "backend-op", "op": "TOPK_QSA" } + ] + }, + "projector-registry": { + "owner": "unslothai#176", + "checks": [ + { "kind": "mtmd", "projector": "*" } + ] + } + }, + "unchecked": { + "unslothai#177": "diffusion-gemma: the fixture is in place and save_models emits the model, but the backend comparison aborts in the arch's own llm_graph_input_attn_diffusion::set_input with a null self_kq_mask->buffer. Left to the DiffusionGemma authors; see the exclusion comment in tests/test-llama-archs.cpp.", + "unslothai#95": "sampling penalties indexed by token id; behaviour is covered by test-sampling, and there is no feature surface of its own to probe", + "unslothai#137": "batched readahead for lazily read gather tables; a throughput change with no observable output difference", + "unslothai#149": "GGML_CUDA_ENABLE_UNIFIED_MEMORY=0 env parsing; needs a CUDA or HIP host, and no runner in the pipeline has one", + "unslothai#152": "per-run mmap of a context's tensors; a memory-layout change with no observable output difference", + "unslothai#157": "cudaMemcpyDefault in the ggml_cuda_cpy 2D fast path; needs a CUDA host", + "unslothai#158": "ROCm_Host compute buffer type on HIP integrated GPUs; needs a ROCm host" + } +} diff --git a/scripts/unsloth/feature_matrix.py b/scripts/unsloth/feature_matrix.py new file mode 100644 index 000000000000..00b392ca1ec5 --- /dev/null +++ b/scripts/unsloth/feature_matrix.py @@ -0,0 +1,200 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. +"""Run the test that proves each shipped feature works, against a built tree. + +pin_contract.py proves the merge did not lose a pin's code. That is a different +question from whether the feature works, and neither one implies the other: the +Inkling banded-attention kernel merged against upstream's sparse attention is +thirteen hunks of CUDA template parameter threading, where a mistake gives +wrong attention output and every static check passes. + +Keyed by FEATURE, not by pin. When upstream absorbs a feature and the pin is +deleted, removing the check with it would put the blind spot back in a +different place -- the feature is still in the release, it just arrives through +the base tag now. So the manifest binds a feature to its current pin and +survives that pin going away. + +A PASS HAS TO BE POSITIVE EVIDENCE. Both harnesses exit 0 having done nothing: + + test-llama-archs -a diffusion-gemma # excluded -> prints SKIP, exits 0 + test-backend-ops test -o TYPO # matches nothing, exits 0 + +so every probe rejects skip markers and requires a non-zero count of cases it +actually ran. Without that this file is decoration. + +CPU only under CUDA_VISIBLE_DEVICES="" is what CI can do, since no runner in +the prebuild pipeline has a GPU. Run it with the variable unset on a GPU box to +get the comparison that matters for kernels. +""" + +from __future__ import annotations + +import argparse +import json +import re +import subprocess +import sys +from pathlib import Path + +# Output that means "this did not run" from a process that exited 0. +SKIP_RE = re.compile(r"\bSKIP\b|not supported|unsupported|no tests|0 tests", re.I) + + +class Unproven(Exception): + """The probe exited 0 without demonstrating anything.""" + + +class NeedsGPU(Exception): + """Nothing is wrong; this check cannot be answered on this machine. + + test-backend-ops compares a backend against the CPU reference, so with no + accelerator present it has nothing to compare and prints "Skipping CPU + backend". Reporting that as a pass would be a lie and reporting it as a + failure would block every nightly, since no runner in the prebuild pipeline + has a GPU. It is counted and named instead. + """ + + +def bins(build_dir: Path) -> Path: + for c in (build_dir / "bin", build_dir): + if (c / "test-backend-ops").exists() or (c / "test-llama-archs").exists(): + return c + raise SystemExit(f"no test binaries under {build_dir}") + + +def run(cmd: list[str], cwd: Path, gpu: bool) -> tuple[int, str]: + env = None + if not gpu: + import os + env = dict(os.environ, CUDA_VISIBLE_DEVICES="", HIP_VISIBLE_DEVICES="") + r = subprocess.run(cmd, cwd=cwd, capture_output=True, text=True, env=env) + return r.returncode, (r.stdout or "") + (r.stderr or "") + + +def probe_arch(check: dict, b: Path, gpu: bool) -> str: + """A synthetic model of this architecture decodes, and matches CPU.""" + arch = check["arch"] + rc, out = run([str(b / "test-llama-archs"), "-a", arch, "-s", "1234"], b, gpu) + if rc != 0: + raise Unproven(f"test-llama-archs -a {arch} exited {rc}") + # The arch's own rows, not the header and not another arch's. + rows = [ln for ln in out.splitlines() if ln.strip().startswith("|") and f"|{arch:>16}|" in ln + or (ln.strip().startswith("|") and ln.split("|")[1].strip() == arch)] + if not rows: + raise Unproven(f"test-llama-archs printed no row for {arch}; it is not in the harness") + ok = [r for r in rows if "OK" in r] + if not ok: + raise Unproven(f"every {arch} row was skipped, so nothing was decoded: {rows[0].strip()}") + return f"{len(ok)}/{len(rows)} device rows decoded and matched CPU" + + +def probe_backend_op(check: dict, b: Path, gpu: bool) -> str: + """The op exists in the backend and matches the CPU reference.""" + if not gpu: + raise NeedsGPU("test-backend-ops compares against CPU, so with no " + "accelerator it skips every backend and proves nothing") + cmd = [str(b / "test-backend-ops"), "test", "-o", check["op"]] + if check.get("params"): + cmd += ["-p", check["params"]] + rc, out = run(cmd, b, gpu) + if rc != 0: + raise Unproven(f"{' '.join(cmd[1:])} exited {rc}") + m = re.search(r"(\d+)/(\d+) tests passed", out) + if not m: + raise Unproven(f"{check['op']} produced no test count; the filter matched nothing") + passed, total = int(m.group(1)), int(m.group(2)) + if total == 0: + raise Unproven(f"{check['op']} matched 0 cases; the op name is stale") + if passed != total: + raise Unproven(f"{check['op']}: {passed}/{total} passed") + return f"{passed}/{total} cases matched the CPU reference" + + +def probe_mtmd(check: dict, b: Path, gpu: bool) -> str: + """The projector registry is intact, including this projector's entry.""" + rc, out = run([str(b / "test-mtmd-impl"), "test_projector_registry"], b, gpu) + if rc != 0: + raise Unproven(f"test-mtmd-impl exited {rc}") + m = re.search(r"assertions\s*:\s*(\d+)", out) + if not m or int(m.group(1)) == 0: + raise Unproven("test_projector_registry ran no assertions; the filter matched nothing") + # The registry test walks the whole enum, so it proves the table is sound. + # That the specific projector is IN the enum is pin_contract.py's job. + return f"projector registry intact over {m.group(1)} assertions" + + +PROBES = {"arch": probe_arch, "backend-op": probe_backend_op, "mtmd": probe_mtmd} + + +def main() -> int: + ap = argparse.ArgumentParser(description=__doc__.split("\n\n")[0]) + ap.add_argument("--build-dir", required=True) + ap.add_argument("--feature-checks", required=True) + ap.add_argument("--only", help="one feature id") + ap.add_argument("--gpu", action="store_true", + help="let the probes see the GPU; CI has none, so the default " + "hides it and the comparison is CPU-only") + ap.add_argument("--report") + args = ap.parse_args() + + b = bins(Path(args.build_dir).resolve()) + doc = json.loads(Path(args.feature_checks).read_text()) + report: dict = {"gpu": args.gpu, "features": [], "ok": False, "deferred": 0} + failed = 0 + deferred = 0 + + for name, feat in sorted(doc["features"].items()): + if args.only and name != args.only: + continue + entry = {"feature": name, "owner": feat.get("owner"), + "results": [], "problems": [], "deferred": []} + for check in feat["checks"]: + kind = check["kind"] + label = f"{kind}:{check.get('arch') or check.get('op') or check.get('projector')}" + try: + if kind not in PROBES: + raise Unproven(f"unknown check kind {kind!r}") + entry["results"].append({"check": label, "evidence": PROBES[kind](check, b, args.gpu)}) + except NeedsGPU as e: + entry["deferred"].append(f"{label}: {e}") + deferred += 1 + except Unproven as e: + entry["problems"].append(f"{label}: {e}") + except OSError as e: + entry["problems"].append(f"{label}: cannot run: {e}") + report["features"].append(entry) + if entry["problems"]: + failed += 1 + print(f"FAIL {name}", file=sys.stderr) + for p in entry["problems"]: + print(f" {p}", file=sys.stderr) + elif entry["results"]: + print(f"ok {name}: " + "; ".join(r["evidence"] for r in entry["results"]) + + (f" [{len(entry['deferred'])} needs a GPU]" if entry["deferred"] else "")) + else: + # Nothing was shown either way. Not a failure here, but it must not + # read as one of the ok lines. + print(f"-- {name}: nothing provable without a GPU " + f"({len(entry['deferred'])} check(s) deferred)") + + for pin, why in sorted(doc.get("unchecked", {}).items()): + print(f"note {pin} has no runtime check: {why}") + + report["ok"] = failed == 0 + report["deferred"] = deferred + if args.report: + Path(args.report).write_text(json.dumps(report, indent=2)) + if failed: + print(f"\n{failed} feature(s) could not be shown to work", file=sys.stderr) + return 1 + # Say what was NOT proven in the same breath as what was. A run that only + # ever prints a success line teaches the reader that green means covered. + tail = f", {deferred} check(s) need a GPU and were not run" if deferred else "" + print(f"\nall {len(report['features'])} features demonstrated" + + (" on GPU" if args.gpu else " on CPU") + tail) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/scripts/unsloth/test_feature_matrix.py b/scripts/unsloth/test_feature_matrix.py new file mode 100644 index 000000000000..ee050f7f761e --- /dev/null +++ b/scripts/unsloth/test_feature_matrix.py @@ -0,0 +1,153 @@ +#!/usr/bin/env python3 +# SPDX-License-Identifier: AGPL-3.0-only +# Copyright 2026-present the Unsloth AI Inc. team. All rights reserved. +"""Tests for feature_matrix.py. Run: python3 scripts/unsloth/test_feature_matrix.py + +The thing worth testing here is not that a passing probe passes. It is that a +probe which exits 0 having proved NOTHING is reported as a failure, because both +real harnesses do exactly that: + + test-llama-archs -a prints SKIP, exits 0 + test-backend-ops test -o matches nothing, exits 0 + +So the fakes below are the real output shapes, verbatim, and the assertions are +about what the script refuses to call a pass. +""" +import json +import os +import stat +import subprocess +import sys +import tempfile +from pathlib import Path + +SCRIPT = Path(__file__).resolve().parent / "feature_matrix.py" +FAILS = [] + + +def check(name, cond, extra=""): + print(f"{'PASS' if cond else 'FAIL'} {name}" + (f" :: {extra}" if extra and not cond else "")) + if not cond: + FAILS.append(name) + + +def fake(dirp: Path, name: str, stdout: str, rc: int = 0): + p = dirp / name + p.write_text("#!/bin/sh\ncat <<'XEOF'\n" + stdout + "\nXEOF\nexit " + str(rc) + "\n") + p.chmod(p.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH) + + +# Real output shapes, copied from actual runs. +ARCHS_OK = """main: using seed 1234 +| Model arch.| Device|Config| NMSE vs. CPU|Roundtrip| +|----------------|-------------------------------|------|---------------|---------| +| inkling| NVIDIA B200| MoE| OK (1.82e-11)| SKIP| +| inkling|Intel(R) Xeon(R) Platinum 8559C| MoE| OK (0.00e+00)| SKIP|""" +ARCHS_ALL_SKIP = """main: using seed 1234 +| Model arch.| Device|Config| NMSE vs. CPU|Roundtrip| +|----------------|-------------------------------|------|---------------|---------| +| inkling| NVIDIA B200| Dense|SKIP | SKIP|""" +ARCHS_ABSENT = """main: using seed 1234 +| Model arch.| Device|Config| NMSE vs. CPU|Roundtrip| +|----------------|-------------------------------|------|---------------|---------|""" +OPS_OK = """ FLASH_ATTN_EXT_BANDED(hsk=64): OK + 13/13 tests passed + Backend CUDA0: OK""" +OPS_NOTHING = """Backend 1/2: CUDA0 +Backend 2/2: CPU + Skipping CPU backend +2/2 backends passed +OK""" +MTMD_OK = """test_projector_registry (185 assertion(s)) [PASS] + +tests : 1 +assertions : 185 +failures : 0""" +MTMD_NOTHING = """tests : 0 +assertions : 0 +failures : 0""" + + +def build(archs=ARCHS_OK, ops=OPS_OK, mtmd=MTMD_OK, checks=None): + d = Path(tempfile.mkdtemp(prefix="fm_")) + (d / "bin").mkdir() + fake(d / "bin", "test-llama-archs", archs) + fake(d / "bin", "test-backend-ops", ops) + fake(d / "bin", "test-mtmd-impl", mtmd) + manifest = d / "feature-checks.json" + manifest.write_text(json.dumps({ + "schema": 1, + "features": {"inkling": {"owner": "unslothai#172", "checks": checks or [ + {"kind": "arch", "arch": "inkling"}, + {"kind": "backend-op", "op": "FLASH_ATTN_EXT_BANDED"}, + ]}}, + "unchecked": {"unslothai#95": "no feature surface"}, + })) + return d, manifest + + +def run(d, manifest, *extra): + rep = d / "r.json" + p = subprocess.run([sys.executable, str(SCRIPT), "--build-dir", str(d), + "--feature-checks", str(manifest), "--report", str(rep), *extra], + capture_output=True, text=True) + return p.returncode, (json.loads(rep.read_text()) if rep.exists() else {}), p.stdout + p.stderr + + +# --- 1. everything genuinely ran ------------------------------------------ +d, m = build() +rc, rep, out = run(d, m, "--gpu") +check("a real pass passes", rc == 0 and rep["ok"], out) +check("the evidence is recorded, not just the verdict", + "1.82e-11" not in out and "2/2 device rows" in out, out) + +# --- 2. the arch harness skipped the arch and exited 0 --------------------- +d, m = build(archs=ARCHS_ALL_SKIP) +rc, rep, out = run(d, m, "--gpu") +check("an all-SKIP arch run is a failure", rc == 1, out) +check("and says nothing was decoded", "nothing was decoded" in out, out) + +# --- 3. the arch is not in the harness at all ----------------------------- +d, m = build(archs=ARCHS_ABSENT) +rc, rep, out = run(d, m, "--gpu") +check("an arch with no row at all is a failure", rc == 1, out) +check("and says it is not in the harness", "not in the harness" in out, out) + +# --- 4. the op filter matched nothing ------------------------------------- +d, m = build(ops=OPS_NOTHING) +rc, rep, out = run(d, m, "--gpu") +check("an op filter that matched nothing is a failure", rc == 1, out) +check("and says the filter matched nothing", "matched nothing" in out, out) + +# --- 5. the op ran and failed --------------------------------------------- +d, m = build(ops=" 11/13 tests passed\n Backend CUDA0: FAIL") +rc, rep, out = run(d, m, "--gpu") +check("a failing op is a failure", rc == 1 and "11/13" in out, out) + +# --- 6. no GPU: op probes are deferred, not passed and not failed --------- +d, m = build() +rc, rep, out = run(d, m) +check("without a GPU the op probe is deferred", rc == 0 and rep["deferred"] == 1, out) +check("deferral is stated in the summary", "need a GPU" in out, out) +check("deferral is not counted as evidence", + len(rep["features"][0]["results"]) == 1, rep) + +# --- 7. a feature with nothing but GPU checks reads as unproven, not ok --- +d, m = build(checks=[{"kind": "backend-op", "op": "FLASH_ATTN_EXT_BANDED"}]) +rc, rep, out = run(d, m) +check("a wholly deferred feature does not print ok", + rc == 0 and "nothing provable without a GPU" in out and "\nok inkling" not in out, out) + +# --- 8. the mtmd probe ran no assertions ---------------------------------- +d, m = build(mtmd=MTMD_NOTHING, checks=[{"kind": "mtmd", "projector": "kimik3"}]) +rc, rep, out = run(d, m, "--gpu") +check("an mtmd run with zero assertions is a failure", rc == 1, out) + +# --- 9. unchecked pins are reported, not hidden --------------------------- +d, m = build() +rc, rep, out = run(d, m, "--gpu") +check("knowingly unchecked pins are printed", "unslothai#95 has no runtime check" in out, out) + +print() +print(f"{len(FAILS)} failure(s)" + (": " + ", ".join(FAILS) if FAILS else "")) +sys.exit(1 if FAILS else 0) From 834e3ff52327c9050fce39bc61becf3794427e87 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Thu, 3 Sep 2026 23:07:34 +0000 Subject: [PATCH 2/4] unsloth: record where the GPU checks actually run, and correct the diffusion-gemma reason The manifest said save_models emits the diffusion-gemma fixture. It does not any more: the saver cannot write attention.sliding_window_pattern, which that arch reads as required, so the model is excluded there too. Both blockers are outside the fixture and both are now named. Also says, in the file someone reads when adding a feature, that the backend-op checks never run in CI and what to do instead: build on a GPU box, run feature_matrix.py --gpu, paste the output into the carry PR. --- scripts/unsloth/feature-checks.json | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/scripts/unsloth/feature-checks.json b/scripts/unsloth/feature-checks.json index 229d41ad3d0e..21b34a0e4b65 100644 --- a/scripts/unsloth/feature-checks.json +++ b/scripts/unsloth/feature-checks.json @@ -29,7 +29,17 @@ "", "A probe that exits 0 having run nothing is a failure, not a pass: both", "harnesses do exactly that for an excluded arch or a misspelled op name.", - "feature_matrix.py rejects skip markers and requires a non-zero case count." + "feature_matrix.py rejects skip markers and requires a non-zero case count.", + "", + "No runner in the prebuild pipeline has a GPU, so the nightly runs this on", + "CPU and every backend-op check is DEFERRED there: named and counted, never", + "reported as passing. The kernels are exactly where a merge goes wrong", + "silently, so before accepting a carry PR that touches one, build it on a", + "GPU box and run:", + "", + " python3 scripts/unsloth/feature_matrix.py --build-dir build --gpu", + "", + "and paste the output into the PR. That is the only place those checks run." ], "schema": 1, "features": { @@ -78,7 +88,7 @@ } }, "unchecked": { - "unslothai#177": "diffusion-gemma: the fixture is in place and save_models emits the model, but the backend comparison aborts in the arch's own llm_graph_input_attn_diffusion::set_input with a null self_kq_mask->buffer. Left to the DiffusionGemma authors; see the exclusion comment in tests/test-llama-archs.cpp.", + "unslothai#177": "diffusion-gemma: the fixture is in place and the model loads, but nothing runs it yet. The backend comparison aborts in the arch's own llm_graph_input_attn_diffusion::set_input with a null self_kq_mask->buffer, and save_models cannot emit it because the saver has no way to write attention.sliding_window_pattern, which this arch reads as required. Both are outside the fixture; see the two exclusion comments in tests/test-llama-archs.cpp.", "unslothai#95": "sampling penalties indexed by token id; behaviour is covered by test-sampling, and there is no feature surface of its own to probe", "unslothai#137": "batched readahead for lazily read gather tables; a throughput change with no observable output difference", "unslothai#149": "GGML_CUDA_ENABLE_UNIFIED_MEMORY=0 env parsing; needs a CUDA or HIP host, and no runner in the pipeline has one", From 77501948c1d171b0ac86fa64c5c5c5ade58cc545 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Fri, 4 Sep 2026 00:42:29 +0000 Subject: [PATCH 3/4] unsloth: DiffusionGemma is checked now, not knowingly unchecked The blocker was the fixture, not the arch: see the pin. It decodes and matches CPU on every device, so it moves out of `unchecked` and into `features` with an arch probe, which is the whole point of keeping that map honest rather than letting a gap sit unnamed. --- scripts/unsloth/feature-checks.json | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/scripts/unsloth/feature-checks.json b/scripts/unsloth/feature-checks.json index 21b34a0e4b65..721f386a2763 100644 --- a/scripts/unsloth/feature-checks.json +++ b/scripts/unsloth/feature-checks.json @@ -58,6 +58,12 @@ { "kind": "backend-op", "op": "LIGHTNING_INDEXER" } ] }, + "diffusion-gemma": { + "owner": "unslothai#177", + "checks": [ + { "kind": "arch", "arch": "diffusion-gemma" } + ] + }, "kimi-k3": { "owner": "unslothai#70", "checks": [ @@ -88,7 +94,6 @@ } }, "unchecked": { - "unslothai#177": "diffusion-gemma: the fixture is in place and the model loads, but nothing runs it yet. The backend comparison aborts in the arch's own llm_graph_input_attn_diffusion::set_input with a null self_kq_mask->buffer, and save_models cannot emit it because the saver has no way to write attention.sliding_window_pattern, which this arch reads as required. Both are outside the fixture; see the two exclusion comments in tests/test-llama-archs.cpp.", "unslothai#95": "sampling penalties indexed by token id; behaviour is covered by test-sampling, and there is no feature surface of its own to probe", "unslothai#137": "batched readahead for lazily read gather tables; a throughput change with no observable output difference", "unslothai#149": "GGML_CUDA_ENABLE_UNIFIED_MEMORY=0 env parsing; needs a CUDA or HIP host, and no runner in the pipeline has one", From 9af58354922f687dc44ed34d6195dede83e83604 Mon Sep 17 00:00:00 2001 From: danielhanchen Date: Fri, 4 Sep 2026 11:27:23 +0000 Subject: [PATCH 4/4] unsloth: point the feature checks at the PRs that own each feature now --- scripts/unsloth/feature-checks.json | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/scripts/unsloth/feature-checks.json b/scripts/unsloth/feature-checks.json index 721f386a2763..93fe8d3c18f5 100644 --- a/scripts/unsloth/feature-checks.json +++ b/scripts/unsloth/feature-checks.json @@ -44,7 +44,7 @@ "schema": 1, "features": { "inkling": { - "owner": "unslothai#172", + "owner": "ggml-org#25731", "checks": [ { "kind": "arch", "arch": "inkling" }, { "kind": "backend-op", "op": "FLASH_ATTN_EXT_BANDED" }, @@ -52,14 +52,14 @@ ] }, "glm5next": { - "owner": "unslothai#173", + "owner": "ggml-org#27754", "checks": [ { "kind": "arch", "arch": "glm5next" }, { "kind": "backend-op", "op": "LIGHTNING_INDEXER" } ] }, "diffusion-gemma": { - "owner": "unslothai#177", + "owner": "ggml-org#24423", "checks": [ { "kind": "arch", "arch": "diffusion-gemma" } ] @@ -72,7 +72,7 @@ ] }, "iq1-narrow-grids": { - "owner": "unslothai#171", + "owner": "unslothai#61", "checks": [ { "kind": "backend-op", "op": "MUL_MAT", "params": "type_a=iq1_xs" }, { "kind": "backend-op", "op": "MUL_MAT", "params": "type_a=iq1_xxs" },