Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
18 changes: 11 additions & 7 deletions tools/ci/hrx_f16_accumulators.txt
Original file line number Diff line number Diff line change
Expand Up @@ -25,10 +25,14 @@
# regenerates this file and tools/ci/hrx_f16_narrowing.txt; a new entry is drift the ratchet
# refuses, and a line whose file no longer has a mul-updated f16 fragment is stale and should be
# dropped as the corpus is fixed.
loom-libs/motifs/mul_mat_id_f16_f16_wmma_core.loom
loom-libs/motifs/mul_mat_id_swiglu_f16_f16_wmma_body.loom
loom-libs/ops/flash_attention_f32_f16_wmma.loom
loom-libs/ops/mul_mat_q6_k_packed_f16_wmma.loom
qwen_moe/qwen3_moe/flash_attention_decode_f32_f16_wmma.loom
qwen_moe/qwen3_moe/flash_attention_f32_f16_wmma.loom
qwen_moe/qwen3_moe/routed_linear_q4k_f16_wmma.loom
loom-libs/motifs/mul_mat_id_f16_f16_wmma_core.loom 4
loom-libs/motifs/mul_mat_id_swiglu_f16_f16_wmma_body.loom 2
loom-libs/ops/flash_attention_decode_split_f32_f16_wmma.loom 2
loom-libs/ops/flash_attention_f32_f16_wmma.loom 2
loom-libs/ops/mul_mat_q6_k_packed_f16_wmma.loom 1
qwen_moe/qwen3_moe/dense_linear_quantized_f16_wmma.loom 4
qwen_moe/qwen3_moe/flash_attention_decode_f32_f16_wmma.loom 2
qwen_moe/qwen3_moe/flash_attention_decode_split_f32_f16_wmma.loom 2
qwen_moe/qwen3_moe/flash_attention_f32_f16_wmma.loom 2
qwen_moe/qwen3_moe/routed_down_quantized_f16_wmma.loom 4
qwen_moe/qwen3_moe/routed_linear_q4k_f16_wmma.loom 6
172 changes: 165 additions & 7 deletions tools/ci/hrx_f16_narrowing.py
Original file line number Diff line number Diff line change
Expand Up @@ -54,10 +54,13 @@
# that have one, so a new one is drift.
ACC_BASELINE = pathlib.Path("tools/ci/hrx_f16_accumulators.txt")
ACC_INIT = re.compile(r"vector\.fragment<init>[^\n]*vector<(\d+)xf16>")
ACC_MUL = re.compile(r"vector\.mulf|template\.apply<@[^>]*wmma[^>]*>|vector\.(fma|contract|dot|outerproduct)\b|wmma\.(mma|load|store)")
ACC_MUL = re.compile(r"vector\.mulf|vector\.mma\b|template\.apply<@[^>]*wmma[^>]*>|vector\.(fma|contract|dot|outerproduct)\b|wmma\.(mma|load|store)")
# Measured (engine#2167): at 120 lines past the init, `mul_mat_id_f16_f16_wmma_core`'s four accumulators read as
# unaffected, because their update sits ~250 lines below in a long Q4_K/Q6 loop body. 250 finds them.
ACC_WINDOW = 250
# A shrink fails the check, not only a grow: `shrank` is in the same failure set as `grew`. That is deliberate -- the
# ratchet is a ratchet, so when a count goes DOWN, `--write` the smaller list in the same change and say why the count
# moved. An improvement therefore cannot land without editing the baseline in front of a reviewer (engine#2244 review).


def corpus_fingerprint(corpus):
Expand All @@ -83,9 +86,33 @@ def corpus_fingerprint(corpus):
GUARD = ("scale_log2", "prescale")


def baseline_entries(text):
"""`path` or `path count` per line -> {path: count or None}. A path-only line is still accepted, so an older
baseline keeps working; a line with a count is the stronger form and is what `--write` emits."""
out = {}
for line in text.splitlines():
line = line.strip()
if not line or line.lstrip().startswith("#"):
continue
parts = line.split()
out[parts[0]] = int(parts[1]) if len(parts) > 1 else None
return out


def baseline_paths(text):
return {line.strip() for line in text.splitlines()
if line.strip() and not line.lstrip().startswith("#")}
return set(baseline_entries(text))


def narrowing_lines(corpus):
"""{path: number of lines that narrow f32 to f16}. The body list alone cannot see a new narrowing written inside a
body it already names -- 51 of the corpus's bodies are listed, so that is the common case, not the rare one."""
found = {}
for path in corpus.rglob("*.loom"):
text = path.read_text(encoding="utf-8", errors="replace")
n = sum(1 for line in text.splitlines() if NARROW.search(line))
if n:
found[path.relative_to(corpus).as_posix()] = n
return found


def narrowing_bodies(corpus):
Expand Down Expand Up @@ -116,6 +143,22 @@ def unclassified_truncations(corpus):
return out


def accumulator_counts(corpus):
"""{path: number of mul-updated f16 fragment accumulators}; measured at the pin: 14 across 7 bodies."""
found = {}
for path in corpus.rglob("*.loom"):
lines = path.read_text(encoding="utf-8", errors="replace").splitlines()
n = 0
for i, line in enumerate(lines):
if not ACC_INIT.search(line):
continue
if any(ACC_MUL.search(later) for later in lines[i:i + ACC_WINDOW]):
n += 1
if n:
found[path.relative_to(corpus).as_posix()] = n
return found


def accumulator_bodies(corpus):
"""Every .loom with an f16 fragment accumulator updated by a multiply or a WMMA/contract accumulate.

Expand Down Expand Up @@ -149,6 +192,29 @@ def drift(baseline, found):
return sorted(found - baseline), sorted(baseline - found)


def count_drift(entries, counts, corpus):
"""Per-body counts, three ways rather than two.

`counts` omits every file that carries no narrowing line, so "absent from counts" used to be treated as "nothing
to compare" and skipped -- which made the best outcome (a body that went to zero) the one thing this ratchet
could not see, and let a baseline line rot after its file was deleted. A file that is GONE has to lose its line;
a file that is PRESENT with no lines has shrunk to zero (engine#2244 review).
"""
grew, shrank, lost = [], [], []
for rel, want in entries.items():
if want is None:
continue
if not (corpus / rel).is_file():
lost.append((rel, want))
continue
got = counts.get(rel, 0)
if got > want:
grew.append((rel, want, got))
elif got < want:
shrank.append((rel, want, got))
return grew, shrank, lost


def check(root, corpus=None):
corpus = pathlib.Path(corpus) if corpus else root / CORPUS
if not corpus.is_dir() or not any(corpus.rglob("*.loom")):
Expand All @@ -161,6 +227,9 @@ def check(root, corpus=None):
baseline = baseline_paths(base_text)
found = narrowing_bodies(corpus)
new, stale = drift(baseline, found)
entries = baseline_entries(base_text)
counts = narrowing_lines(corpus)
grew, shrank, lost = count_drift(entries, counts, corpus)
# engine#2186 item 1 / engine#2167: the accumulator class, which no truncation scan can see.
acc_base_file = root / ACC_BASELINE
if not acc_base_file.is_file():
Expand All @@ -170,6 +239,9 @@ def check(root, corpus=None):
acc_baseline = baseline_paths(acc_base_file.read_text(encoding="utf-8"))
acc_found = accumulator_bodies(corpus)
acc_new, acc_stale = drift(acc_baseline, acc_found)
acc_entries = baseline_entries(acc_base_file.read_text(encoding="utf-8"))
acc_counts = accumulator_counts(corpus)
acc_grew, acc_shrank, acc_lost = count_drift(acc_entries, acc_counts, corpus)
guard = guarded(found, corpus)
unknown = unclassified_truncations(corpus)
for where, text in unknown:
Expand All @@ -188,7 +260,26 @@ def check(root, corpus=None):
for rel in acc_stale:
print(f"hrx_f16_narrowing: {rel} is in the accumulator baseline but has no mul-updated f16 fragment -- drop "
f"the line, the list should shrink")
if new or stale or unknown or acc_new or acc_stale:
for rel, want, got in grew:
print(f"hrx_f16_narrowing: MORE narrowing LINES in {rel} ({want} -> {got}) -- the body is already listed, so "
f"the body check alone would pass (engine#1962)")
for rel, want in lost:
print(f"hrx_f16_narrowing: {rel} is in the baseline with {want} narrowing line(s) but the file is gone from the "
f"corpus -- drop the line, the list should shrink")
for rel, want, got in shrank:
suffix = (f"-- that is the outcome this ratchet exists for, so record it: drop the line"
if got == 0 else "-- drop the count")
print(f"hrx_f16_narrowing: {rel} narrows fewer lines than the baseline ({want} -> {got}) {suffix}")
for rel, want, got in acc_grew:
print(f"hrx_f16_narrowing: MORE f16 accumulators in {rel} ({want} -> {got}) -- a second accumulator inside a "
f"known body (engine#2186, engine#2167)")
for rel, want in acc_lost:
print(f"hrx_f16_narrowing: {rel} is in the accumulator baseline with {want} accumulator(s) but the file is gone "
f"from the corpus -- drop the line")
for rel, want, got in acc_shrank:
suffix = "-- drop the line, the list should shrink" if got == 0 else "-- drop the count"
print(f"hrx_f16_narrowing: {rel} has fewer f16 accumulators ({want} -> {got}) {suffix}")
if new or stale or unknown or acc_new or acc_stale or grew or shrank or lost or acc_grew or acc_shrank or acc_lost:
print(f"hrx_f16_narrowing: FAIL ({len(new)} new, {len(stale)} stale of {len(baseline)}, "
f"{len(unknown)} unclassified fptrunc; {len(acc_new)} new, {len(acc_stale)} stale of "
f"{len(acc_baseline)} accumulator(s))")
Expand All @@ -207,9 +298,12 @@ def write(root, corpus=None):
return 77
head = "".join(line for line in (root / BASELINE).read_text(encoding="utf-8").splitlines(True)
if line.lstrip().startswith("#"))
bodies = sorted(narrowing_bodies(corpus))
(root / BASELINE).write_text(head + "\n".join(bodies) + "\n", encoding="utf-8")
accs = sorted(accumulator_bodies(corpus))
counts = narrowing_lines(corpus)
lines = [f"{rel}\t{n}" for rel, n in sorted(counts.items())]
(root / BASELINE).write_text(head + "\n".join(lines) + "\n", encoding="utf-8")
bodies = sorted(counts)
acc_counts = accumulator_counts(corpus)
accs = [f"{rel}\t{n}" for rel, n in sorted(acc_counts.items())]
acc_head = "".join(line for line in (root / ACC_BASELINE).read_text(encoding="utf-8").splitlines(True)
if line.lstrip().startswith("#")) if (root / ACC_BASELINE).is_file() else ""
(root / ACC_BASELINE).write_text(acc_head + "\n".join(accs) + "\n", encoding="utf-8")
Expand Down Expand Up @@ -276,6 +370,40 @@ def check_that(ok, what):
empty = root / "empty"
empty.mkdir()
check_that(check(empty) == 77, "a missing corpus returns 77, not 0")
# engine#2244 review: nothing reached check()'s FAIL condition, so deleting `grew` from it still passed
# --self-test. The failure set is the one thing this tool exists to enforce, so it needs an arm of its own:
# a listed body that gains a narrowing line must come back rc 1.
ck = root / "ck"
ck_corpus = ck / CORPUS
ck_corpus.mkdir(parents=True)
(ck / BASELINE).parent.mkdir(parents=True, exist_ok=True)
(ck_corpus / "listed.loom").write_text(" %x = scalar.fptrunc %y : f32 to f16\n")
(ck / BASELINE).write_text("# c\nlisted.loom 1\n")
(ck / ACC_BASELINE).write_text("# c\n")
check_that(check(ck) == 0, "check(): a corpus that matches its baseline is rc 0")
(ck_corpus / "listed.loom").write_text(
" %x = scalar.fptrunc %y : f32 to f16\n %z = scalar.fptrunc %w : f32 to f16\n")
check_that(check(ck) == 1,
"check(): one extra narrowing LINE in a listed body is rc 1 -- the fail condition is armed")
(ck_corpus / "listed.loom").write_text(" %x = scalar.constant 0.0 : f32\n")
check_that(check(ck) == 1,
"check(): a body that drops to zero is also rc 1, not a silent pass")
# engine#2244 review: the two cases "absent from counts" used to collapse into one `continue`. A body that
# went to ZERO narrowing lines is the outcome this ratchet exists for, and a body whose file was deleted
# must lose its baseline line instead of rotting quietly.
cdr = root / "cd"
cdr.mkdir()
(cdr / "clean.loom").write_text(" %x = scalar.constant 0.0 : f32\n")
(cdr / "still.loom").write_text(" %x = scalar.fptrunc %y : f32 to f16\n")
entries = {"clean.loom": 3, "still.loom": 1, "gone.loom": 2}
counts = narrowing_lines(cdr)
grew, shrank, lost = count_drift(entries, counts, cdr)
check_that(shrank == [("clean.loom", 3, 0)],
"a body that drops to zero narrowing lines is reported as a shrink, not as silence")
check_that(lost == [("gone.loom", 2)],
"a baseline body whose file is gone is reported stale, not skipped")
check_that(grew == [] and ("still.loom", 1, 1) not in shrank,
"an unchanged body is neither growth nor shrink")
# engine#2186 item 1 / engine#2167: the accumulator class. A mul-updated f16 fragment is found; an add-only one
# is not; a WMMA/contract update counts as a multiply; and an update far below the init is still found -- the
# last is the fixture that would have caught the 120-line window, which reported four real accumulators as
Expand Down Expand Up @@ -312,6 +440,36 @@ def check_that(ok, what):
check_that(fp != corpus_fingerprint(c), "fingerprint: a one-byte edit changes it")
(c / "a.loom").rename(c / "s" / "a.loom")
check_that(fp != corpus_fingerprint(c), "fingerprint: a move changes it too (the path is hashed)")
with tempfile.TemporaryDirectory() as tmp:
root2 = pathlib.Path(tmp)
c2 = root2 / CORPUS
c2.mkdir(parents=True)
(c2 / "a.loom").write_text(" %x = scalar.fptrunc %y : f32 to f16\n")
check_that(narrowing_lines(c2) == {"a.loom": 1}, "the narrowing-line count of a listed body is counted")
(c2 / "a.loom").write_text(" %x = scalar.fptrunc %y : f32 to f16\n %z = scalar.fptrunc %w : f32 to f16\n")
check_that(narrowing_lines(c2) == {"a.loom": 2},
"a second narrowing INSIDE an already-listed body raises the count (the body check alone would pass)")
(root2 / BASELINE).parent.mkdir(parents=True, exist_ok=True)
(root2 / BASELINE).write_text("a.loom\t2\n")
check_that(baseline_entries((root2 / BASELINE).read_text()) == {"a.loom": 2},
"a 'path count' baseline line parses to both")
(root2 / BASELINE).write_text("a.loom\n")
check_that(baseline_entries((root2 / BASELINE).read_text()) == {"a.loom": None},
"a path-only line still parses, so an older baseline keeps working")
c2b = root2 / "acc2"
c2b.mkdir()
(c2b / "one.loom").write_text(
" %init = vector.fragment<init> %z shape [%m, %n] : vector<8xf16>\n"
" %next = vector.mulf %acc, %k : vector<8xf16>\n")
check_that(accumulator_counts(c2b) == {"one.loom": 1}, "an accumulator count of one is counted")
(c2b / "one.loom").write_text(
" %init = vector.fragment<init> %z shape [%m, %n] : vector<8xf16>\n"
" %next = vector.mulf %acc, %k : vector<8xf16>\n"
" %init2 = vector.fragment<init> %z2 shape [%m, %n] : vector<16xf16>\n"
" %next2 = vector.mulf %acc2, %k2 : vector<16xf16>\n")
check_that(accumulator_counts(c2b) == {"one.loom": 2},
"a SECOND accumulator inside a listed body raises the count (the body check alone would pass)")

print(f"hrx_f16_narrowing self-test: {'ok' if not failures else f'{failures} failure(s)'}")
return 1 if failures else 0

Expand Down
Loading
Loading