Skip to content
Closed
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 16 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -342,6 +342,22 @@ add_test(NAME bench_preflight COMMAND bash ${CMAKE_SOURCE_DIR}/tools/tests/test-
# the same offset from the data section returns the tensor, from byte 8 it does not.
add_python_test(hf_tensor_self ${CMAKE_SOURCE_DIR}/tools/hf_tensor.py --self-test)
add_python_test(moe_placement ${CMAKE_SOURCE_DIR}/tools/moe_placement.py --self-test)
# engine#2018: eight `tests/*_test.py` had NO runner at all -- the third file kind for this sweep (`tests/*.sh` is
# engine#1824, `tools/*.py` self-tests is engine#1911), and the kind most of this tree's newer tests are written in. A PR
# that modifies one of them changes code that never executes. All eight pass on this box (`python3 tests/<f>` rc 0, checked
# before registering), so they are rows rather than allow-list entries; `tools/ci/tests_referenced_check.py` now fails on the
# state, so a ninth cannot appear unnoticed.
add_python_test(check_pins ${CMAKE_SOURCE_DIR}/tests/check_pins_test.py)
add_python_test(gguf_to_q4nx ${CMAKE_SOURCE_DIR}/tests/gguf_to_q4nx_test.py)
add_python_test(hrx_long_prompt ${CMAKE_SOURCE_DIR}/tests/hrx_long_prompt_test.py)
add_python_test(hyperloom_benchmark ${CMAKE_SOURCE_DIR}/tests/hyperloom_benchmark_test.py)
# ... and it SKIPS (77) on a box with no GTT counters, because its assertions are about a real device: the first version of
# this row failed CI's build runner in 0.06 s on `gtt_used_gib is not None` (engine#2018). A skip, not a weakened assertion.
set_tests_properties(hyperloom_benchmark PROPERTIES SKIP_RETURN_CODE 77)
add_python_test(moe_expert_chunks_accum ${CMAKE_SOURCE_DIR}/tests/moe_expert_chunks_accum_test.py)
add_python_test(mtp_draft_vocab ${CMAKE_SOURCE_DIR}/tests/mtp_draft_vocab_test.py)
add_python_test(repack_verify_source ${CMAKE_SOURCE_DIR}/tests/repack_verify_source_test.py)
add_python_test(ternary_to_q4_0 ${CMAKE_SOURCE_DIR}/tests/ternary_to_q4_0_test.py)
# engine #1279: the version a package is built with (scripts/pack-version.sh) -- a release tag as
# it stands, a candidate mapped into the ISO week of its own date with a suffix that sorts below that
# release, and the ordering a candidate needs: previous release < candidate < its release. The
Expand Down
9 changes: 9 additions & 0 deletions tests/hyperloom_benchmark_test.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,15 @@ def load():
def main():
bench = load()

# engine#2018: this file's assertions are about a REAL device (`box_conditions()` must return numbers for every field
# it reads), so on a box with no amdgpu -- CI's build runner -- there is nothing for them to be true about. Skipped with
# 77 rather than weakened: the numbers are the point where a GPU exists, and a test that passes on a null box would
# stop being evidence. Registered as a ctest row with SKIP_RETURN_CODE 77 (CMakeLists.txt, engine#2018).
_box = bench.box_conditions()
if _box["gtt_used_gib"] is None or _box["gtt_total_gib"] is None:
print("skip: this box exposes no GTT counters (no amdgpu) -- the assertions below are about a real device")
return 77

# The filler prompt: measured at ~4.85 characters per token on this box (a 75,200-character prompt
# came back as 15,492 prompt tokens), so 18,800 tokens needs >= 18,800 * 4.85 characters, and the
# same request twice has to be byte-identical or candidates are not comparable.
Expand Down
70 changes: 69 additions & 1 deletion tools/ci/tests_referenced_check.py
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,52 @@ def unregistered_selftest_tools(root):
return drift


def unregistered_test_scripts(root):
"""tests/*.py that are tests and that no registration names (engine#2018).

The third file kind for this sweep: `tests/*.sh` is engine#1824, `tools/*.py` self-tests is engine#1911, and the newer
tests here are written in Python -- measured at filing, **8 of the tree's 41 `tests/*_test.py` had no runner at all**, so a
PR that modifies one changes code that never executes (tests/gguf_to_q4nx_test.py was the named example).

A file is a test by the tree's own convention (`*_test.py`) or by declaring a self-test flag. `have` is a registration in
CMakeLists.txt/cmake/*.cmake, or another `tests/*.py` that imports it -- a registered test importing a helper does run it,
and that is how several module-level arms in this tree are written.
"""
tests = root / "tests"
if not tests.is_dir():
return []
need = []
for path in sorted(tests.glob("*.py")):
try:
text = path.read_text(errors="replace")
except OSError:
continue
if path.name.endswith("_test.py") or SELFTEST.search(text):
need.append(path.name)
if not need:
return []
named = {n: False for n in need}
for path in [root / "CMakeLists.txt"] + sorted((root / "cmake").glob("*.cmake")):
if not path.is_file():
continue
text = path.read_text(errors="replace")
for n in need:
if n in text:
named[n] = True
if not all(named.values()):
for path in sorted(tests.glob("*.py")):
try:
text = path.read_text(errors="replace")
except OSError:
continue
for n in need:
if path.name == n:
continue # a test naming itself is not a runner
if n in text or pathlib.PurePosixPath(n).stem in text:
named[n] = True
return [n for n in need if not named[n]]


def check(root):
tests = root / "tests"
if not tests.is_dir():
Expand All @@ -158,7 +204,11 @@ def check(root):
for rel in tools_drift:
print("tests_referenced: %s declares a self-test and no runner names it -- add a ctest row (add_test in "
"CMakeLists.txt or cmake/*.cmake) or a workflow step, or the arm is dead code" % rel)
if drift or gone or idle or tools_drift:
py_drift = unregistered_test_scripts(root)
for name in py_drift:
print("tests_referenced: tests/%s is a test and nothing runs it -- register it (add_python_test in "
"CMakeLists.txt), or list it in %s with the reason it must not be a row (engine#2018)" % (name, ALLOW))
if drift or gone or idle or tools_drift or py_drift:
return 1
print("tests_referenced: all %d test script(s) are referenced%s; %d tool self-test(s) are registered" % (
len(need), "" if not allow else " (%d deliberately not a row: %s)" % (len(allow), ", ".join(sorted(allow))),
Expand Down Expand Up @@ -239,6 +289,24 @@ def root_with(files):
"tools/registered.py": "# --self-test\n"})
cases.append(("a registered tool self-test is clean", check(r3), 0, None))

# 8b. engine#2018: a tests/*.py that is a test and that nothing runs
r = root_with({"tests/one.sh": "#!/bin/sh\n",
"CMakeLists.txt": "add_test(NAME one COMMAND one.sh)\n",
"tests/orphan_test.py": "print('x')\n"})
cases.append(("a tests/*_test.py with no runner is drift", check(r), 1, "orphan_test.py"))
cases_root["a tests/*_test.py with no runner is drift"] = r
# ... and it is not drift once a registration names it
r = root_with({"tests/one.sh": "#!/bin/sh\n",
"CMakeLists.txt": "add_test(NAME one COMMAND one.sh)\nadd_python_test(orphan tests/orphan_test.py)\n",
"tests/orphan_test.py": "print('x')\n"})
cases.append(("a registered tests/*_test.py is clean", check(r), 0, None))
# ... and a registered test that IMPORTS it runs it, which is how several module-level arms are written here
r = root_with({"tests/one.sh": "#!/bin/sh\n",
"CMakeLists.txt": "add_test(NAME one COMMAND one.sh)\nadd_python_test(one_py tests/one_test.py)\n",
"tests/one_test.py": "import orphan_test\n",
"tests/orphan_test.py": "print('x')\n"})
cases.append(("a tests/*.py another registered test imports is not drift", check(r), 0, None))

# 9. the entry may be written either way: path form and basename mean the same line
r = root_with({"tests/keeper2.sh": "#!/bin/sh\n", "CMakeLists.txt": "",
str(ALLOW): "tests/keeper2.sh # path form is accepted\n"})
Expand Down
Loading