diff --git a/CMakeLists.txt b/CMakeLists.txt index 7b6821e3e..551c5b40e 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -342,6 +342,22 @@ add_test(NAME bench_preflight COMMAND bash ${CMAKE_SOURCE_DIR}/tools/tests/test- # the same offset from the data section returns the tensor, from byte 8 it does not. add_python_test(hf_tensor_self ${CMAKE_SOURCE_DIR}/tools/hf_tensor.py --self-test) add_python_test(moe_placement ${CMAKE_SOURCE_DIR}/tools/moe_placement.py --self-test) +# engine#2018: eight `tests/*_test.py` had NO runner at all -- the third file kind for this sweep (`tests/*.sh` is +# engine#1824, `tools/*.py` self-tests is engine#1911), and the kind most of this tree's newer tests are written in. A PR +# that modifies one of them changes code that never executes. All eight pass on this box (`python3 tests/` rc 0, checked +# before registering), so they are rows rather than allow-list entries; `tools/ci/tests_referenced_check.py` now fails on the +# state, so a ninth cannot appear unnoticed. +add_python_test(check_pins ${CMAKE_SOURCE_DIR}/tests/check_pins_test.py) +add_python_test(gguf_to_q4nx ${CMAKE_SOURCE_DIR}/tests/gguf_to_q4nx_test.py) +add_python_test(hrx_long_prompt ${CMAKE_SOURCE_DIR}/tests/hrx_long_prompt_test.py) +add_python_test(hyperloom_benchmark ${CMAKE_SOURCE_DIR}/tests/hyperloom_benchmark_test.py) +# ... and it SKIPS (77) on a box with no GTT counters, because its assertions are about a real device: the first version of +# this row failed CI's build runner in 0.06 s on `gtt_used_gib is not None` (engine#2018). A skip, not a weakened assertion. +set_tests_properties(hyperloom_benchmark PROPERTIES SKIP_RETURN_CODE 77) +add_python_test(moe_expert_chunks_accum ${CMAKE_SOURCE_DIR}/tests/moe_expert_chunks_accum_test.py) +add_python_test(mtp_draft_vocab ${CMAKE_SOURCE_DIR}/tests/mtp_draft_vocab_test.py) +add_python_test(repack_verify_source ${CMAKE_SOURCE_DIR}/tests/repack_verify_source_test.py) +add_python_test(ternary_to_q4_0 ${CMAKE_SOURCE_DIR}/tests/ternary_to_q4_0_test.py) # engine #1279: the version a package is built with (scripts/pack-version.sh) -- a release tag as # it stands, a candidate mapped into the ISO week of its own date with a suffix that sorts below that # release, and the ordering a candidate needs: previous release < candidate < its release. The diff --git a/tests/hyperloom_benchmark_test.py b/tests/hyperloom_benchmark_test.py index d477ab535..817774514 100644 --- a/tests/hyperloom_benchmark_test.py +++ b/tests/hyperloom_benchmark_test.py @@ -44,6 +44,15 @@ def load(): def main(): bench = load() + # engine#2018: this file's assertions are about a REAL device (`box_conditions()` must return numbers for every field + # it reads), so on a box with no amdgpu -- CI's build runner -- there is nothing for them to be true about. Skipped with + # 77 rather than weakened: the numbers are the point where a GPU exists, and a test that passes on a null box would + # stop being evidence. Registered as a ctest row with SKIP_RETURN_CODE 77 (CMakeLists.txt, engine#2018). + _box = bench.box_conditions() + if _box["gtt_used_gib"] is None or _box["gtt_total_gib"] is None: + print("skip: this box exposes no GTT counters (no amdgpu) -- the assertions below are about a real device") + return 77 + # The filler prompt: measured at ~4.85 characters per token on this box (a 75,200-character prompt # came back as 15,492 prompt tokens), so 18,800 tokens needs >= 18,800 * 4.85 characters, and the # same request twice has to be byte-identical or candidates are not comparable. diff --git a/tools/ci/tests_referenced_check.py b/tools/ci/tests_referenced_check.py index 2df91ec28..c9a460a3d 100755 --- a/tools/ci/tests_referenced_check.py +++ b/tools/ci/tests_referenced_check.py @@ -135,6 +135,52 @@ def unregistered_selftest_tools(root): return drift +def unregistered_test_scripts(root): + """tests/*.py that are tests and that no registration names (engine#2018). + + The third file kind for this sweep: `tests/*.sh` is engine#1824, `tools/*.py` self-tests is engine#1911, and the newer + tests here are written in Python -- measured at filing, **8 of the tree's 41 `tests/*_test.py` had no runner at all**, so a + PR that modifies one changes code that never executes (tests/gguf_to_q4nx_test.py was the named example). + + A file is a test by the tree's own convention (`*_test.py`) or by declaring a self-test flag. `have` is a registration in + CMakeLists.txt/cmake/*.cmake, or another `tests/*.py` that imports it -- a registered test importing a helper does run it, + and that is how several module-level arms in this tree are written. + """ + tests = root / "tests" + if not tests.is_dir(): + return [] + need = [] + for path in sorted(tests.glob("*.py")): + try: + text = path.read_text(errors="replace") + except OSError: + continue + if path.name.endswith("_test.py") or SELFTEST.search(text): + need.append(path.name) + if not need: + return [] + named = {n: False for n in need} + for path in [root / "CMakeLists.txt"] + sorted((root / "cmake").glob("*.cmake")): + if not path.is_file(): + continue + text = path.read_text(errors="replace") + for n in need: + if n in text: + named[n] = True + if not all(named.values()): + for path in sorted(tests.glob("*.py")): + try: + text = path.read_text(errors="replace") + except OSError: + continue + for n in need: + if path.name == n: + continue # a test naming itself is not a runner + if n in text or pathlib.PurePosixPath(n).stem in text: + named[n] = True + return [n for n in need if not named[n]] + + def check(root): tests = root / "tests" if not tests.is_dir(): @@ -158,7 +204,11 @@ def check(root): for rel in tools_drift: print("tests_referenced: %s declares a self-test and no runner names it -- add a ctest row (add_test in " "CMakeLists.txt or cmake/*.cmake) or a workflow step, or the arm is dead code" % rel) - if drift or gone or idle or tools_drift: + py_drift = unregistered_test_scripts(root) + for name in py_drift: + print("tests_referenced: tests/%s is a test and nothing runs it -- register it (add_python_test in " + "CMakeLists.txt), or list it in %s with the reason it must not be a row (engine#2018)" % (name, ALLOW)) + if drift or gone or idle or tools_drift or py_drift: return 1 print("tests_referenced: all %d test script(s) are referenced%s; %d tool self-test(s) are registered" % ( len(need), "" if not allow else " (%d deliberately not a row: %s)" % (len(allow), ", ".join(sorted(allow))), @@ -239,6 +289,24 @@ def root_with(files): "tools/registered.py": "# --self-test\n"}) cases.append(("a registered tool self-test is clean", check(r3), 0, None)) + # 8b. engine#2018: a tests/*.py that is a test and that nothing runs + r = root_with({"tests/one.sh": "#!/bin/sh\n", + "CMakeLists.txt": "add_test(NAME one COMMAND one.sh)\n", + "tests/orphan_test.py": "print('x')\n"}) + cases.append(("a tests/*_test.py with no runner is drift", check(r), 1, "orphan_test.py")) + cases_root["a tests/*_test.py with no runner is drift"] = r + # ... and it is not drift once a registration names it + r = root_with({"tests/one.sh": "#!/bin/sh\n", + "CMakeLists.txt": "add_test(NAME one COMMAND one.sh)\nadd_python_test(orphan tests/orphan_test.py)\n", + "tests/orphan_test.py": "print('x')\n"}) + cases.append(("a registered tests/*_test.py is clean", check(r), 0, None)) + # ... and a registered test that IMPORTS it runs it, which is how several module-level arms are written here + r = root_with({"tests/one.sh": "#!/bin/sh\n", + "CMakeLists.txt": "add_test(NAME one COMMAND one.sh)\nadd_python_test(one_py tests/one_test.py)\n", + "tests/one_test.py": "import orphan_test\n", + "tests/orphan_test.py": "print('x')\n"}) + cases.append(("a tests/*.py another registered test imports is not drift", check(r), 0, None)) + # 9. the entry may be written either way: path form and basename mean the same line r = root_with({"tests/keeper2.sh": "#!/bin/sh\n", "CMakeLists.txt": "", str(ALLOW): "tests/keeper2.sh # path form is accepted\n"})