diff --git a/tools/ci/hrx_f16_accumulators.txt b/tools/ci/hrx_f16_accumulators.txt index e3158e67b..040b637ab 100644 --- a/tools/ci/hrx_f16_accumulators.txt +++ b/tools/ci/hrx_f16_accumulators.txt @@ -25,10 +25,14 @@ # regenerates this file and tools/ci/hrx_f16_narrowing.txt; a new entry is drift the ratchet # refuses, and a line whose file no longer has a mul-updated f16 fragment is stale and should be # dropped as the corpus is fixed. -loom-libs/motifs/mul_mat_id_f16_f16_wmma_core.loom -loom-libs/motifs/mul_mat_id_swiglu_f16_f16_wmma_body.loom -loom-libs/ops/flash_attention_f32_f16_wmma.loom -loom-libs/ops/mul_mat_q6_k_packed_f16_wmma.loom -qwen_moe/qwen3_moe/flash_attention_decode_f32_f16_wmma.loom -qwen_moe/qwen3_moe/flash_attention_f32_f16_wmma.loom -qwen_moe/qwen3_moe/routed_linear_q4k_f16_wmma.loom +loom-libs/motifs/mul_mat_id_f16_f16_wmma_core.loom 4 +loom-libs/motifs/mul_mat_id_swiglu_f16_f16_wmma_body.loom 2 +loom-libs/ops/flash_attention_decode_split_f32_f16_wmma.loom 2 +loom-libs/ops/flash_attention_f32_f16_wmma.loom 2 +loom-libs/ops/mul_mat_q6_k_packed_f16_wmma.loom 1 +qwen_moe/qwen3_moe/dense_linear_quantized_f16_wmma.loom 4 +qwen_moe/qwen3_moe/flash_attention_decode_f32_f16_wmma.loom 2 +qwen_moe/qwen3_moe/flash_attention_decode_split_f32_f16_wmma.loom 2 +qwen_moe/qwen3_moe/flash_attention_f32_f16_wmma.loom 2 +qwen_moe/qwen3_moe/routed_down_quantized_f16_wmma.loom 4 +qwen_moe/qwen3_moe/routed_linear_q4k_f16_wmma.loom 6 diff --git a/tools/ci/hrx_f16_narrowing.py b/tools/ci/hrx_f16_narrowing.py index d110d7ed4..c2fb44b01 100755 --- a/tools/ci/hrx_f16_narrowing.py +++ b/tools/ci/hrx_f16_narrowing.py @@ -54,10 +54,13 @@ # that have one, so a new one is drift. ACC_BASELINE = pathlib.Path("tools/ci/hrx_f16_accumulators.txt") ACC_INIT = re.compile(r"vector\.fragment[^\n]*vector<(\d+)xf16>") -ACC_MUL = re.compile(r"vector\.mulf|template\.apply<@[^>]*wmma[^>]*>|vector\.(fma|contract|dot|outerproduct)\b|wmma\.(mma|load|store)") +ACC_MUL = re.compile(r"vector\.mulf|vector\.mma\b|template\.apply<@[^>]*wmma[^>]*>|vector\.(fma|contract|dot|outerproduct)\b|wmma\.(mma|load|store)") # Measured (engine#2167): at 120 lines past the init, `mul_mat_id_f16_f16_wmma_core`'s four accumulators read as # unaffected, because their update sits ~250 lines below in a long Q4_K/Q6 loop body. 250 finds them. ACC_WINDOW = 250 +# A shrink fails the check, not only a grow: `shrank` is in the same failure set as `grew`. That is deliberate -- the +# ratchet is a ratchet, so when a count goes DOWN, `--write` the smaller list in the same change and say why the count +# moved. An improvement therefore cannot land without editing the baseline in front of a reviewer (engine#2244 review). def corpus_fingerprint(corpus): @@ -83,9 +86,33 @@ def corpus_fingerprint(corpus): GUARD = ("scale_log2", "prescale") +def baseline_entries(text): + """`path` or `path count` per line -> {path: count or None}. A path-only line is still accepted, so an older + baseline keeps working; a line with a count is the stronger form and is what `--write` emits.""" + out = {} + for line in text.splitlines(): + line = line.strip() + if not line or line.lstrip().startswith("#"): + continue + parts = line.split() + out[parts[0]] = int(parts[1]) if len(parts) > 1 else None + return out + + def baseline_paths(text): - return {line.strip() for line in text.splitlines() - if line.strip() and not line.lstrip().startswith("#")} + return set(baseline_entries(text)) + + +def narrowing_lines(corpus): + """{path: number of lines that narrow f32 to f16}. The body list alone cannot see a new narrowing written inside a + body it already names -- 51 of the corpus's bodies are listed, so that is the common case, not the rare one.""" + found = {} + for path in corpus.rglob("*.loom"): + text = path.read_text(encoding="utf-8", errors="replace") + n = sum(1 for line in text.splitlines() if NARROW.search(line)) + if n: + found[path.relative_to(corpus).as_posix()] = n + return found def narrowing_bodies(corpus): @@ -116,6 +143,22 @@ def unclassified_truncations(corpus): return out +def accumulator_counts(corpus): + """{path: number of mul-updated f16 fragment accumulators}; measured at the pin: 14 across 7 bodies.""" + found = {} + for path in corpus.rglob("*.loom"): + lines = path.read_text(encoding="utf-8", errors="replace").splitlines() + n = 0 + for i, line in enumerate(lines): + if not ACC_INIT.search(line): + continue + if any(ACC_MUL.search(later) for later in lines[i:i + ACC_WINDOW]): + n += 1 + if n: + found[path.relative_to(corpus).as_posix()] = n + return found + + def accumulator_bodies(corpus): """Every .loom with an f16 fragment accumulator updated by a multiply or a WMMA/contract accumulate. @@ -149,6 +192,29 @@ def drift(baseline, found): return sorted(found - baseline), sorted(baseline - found) +def count_drift(entries, counts, corpus): + """Per-body counts, three ways rather than two. + + `counts` omits every file that carries no narrowing line, so "absent from counts" used to be treated as "nothing + to compare" and skipped -- which made the best outcome (a body that went to zero) the one thing this ratchet + could not see, and let a baseline line rot after its file was deleted. A file that is GONE has to lose its line; + a file that is PRESENT with no lines has shrunk to zero (engine#2244 review). + """ + grew, shrank, lost = [], [], [] + for rel, want in entries.items(): + if want is None: + continue + if not (corpus / rel).is_file(): + lost.append((rel, want)) + continue + got = counts.get(rel, 0) + if got > want: + grew.append((rel, want, got)) + elif got < want: + shrank.append((rel, want, got)) + return grew, shrank, lost + + def check(root, corpus=None): corpus = pathlib.Path(corpus) if corpus else root / CORPUS if not corpus.is_dir() or not any(corpus.rglob("*.loom")): @@ -161,6 +227,9 @@ def check(root, corpus=None): baseline = baseline_paths(base_text) found = narrowing_bodies(corpus) new, stale = drift(baseline, found) + entries = baseline_entries(base_text) + counts = narrowing_lines(corpus) + grew, shrank, lost = count_drift(entries, counts, corpus) # engine#2186 item 1 / engine#2167: the accumulator class, which no truncation scan can see. acc_base_file = root / ACC_BASELINE if not acc_base_file.is_file(): @@ -170,6 +239,9 @@ def check(root, corpus=None): acc_baseline = baseline_paths(acc_base_file.read_text(encoding="utf-8")) acc_found = accumulator_bodies(corpus) acc_new, acc_stale = drift(acc_baseline, acc_found) + acc_entries = baseline_entries(acc_base_file.read_text(encoding="utf-8")) + acc_counts = accumulator_counts(corpus) + acc_grew, acc_shrank, acc_lost = count_drift(acc_entries, acc_counts, corpus) guard = guarded(found, corpus) unknown = unclassified_truncations(corpus) for where, text in unknown: @@ -188,7 +260,26 @@ def check(root, corpus=None): for rel in acc_stale: print(f"hrx_f16_narrowing: {rel} is in the accumulator baseline but has no mul-updated f16 fragment -- drop " f"the line, the list should shrink") - if new or stale or unknown or acc_new or acc_stale: + for rel, want, got in grew: + print(f"hrx_f16_narrowing: MORE narrowing LINES in {rel} ({want} -> {got}) -- the body is already listed, so " + f"the body check alone would pass (engine#1962)") + for rel, want in lost: + print(f"hrx_f16_narrowing: {rel} is in the baseline with {want} narrowing line(s) but the file is gone from the " + f"corpus -- drop the line, the list should shrink") + for rel, want, got in shrank: + suffix = (f"-- that is the outcome this ratchet exists for, so record it: drop the line" + if got == 0 else "-- drop the count") + print(f"hrx_f16_narrowing: {rel} narrows fewer lines than the baseline ({want} -> {got}) {suffix}") + for rel, want, got in acc_grew: + print(f"hrx_f16_narrowing: MORE f16 accumulators in {rel} ({want} -> {got}) -- a second accumulator inside a " + f"known body (engine#2186, engine#2167)") + for rel, want in acc_lost: + print(f"hrx_f16_narrowing: {rel} is in the accumulator baseline with {want} accumulator(s) but the file is gone " + f"from the corpus -- drop the line") + for rel, want, got in acc_shrank: + suffix = "-- drop the line, the list should shrink" if got == 0 else "-- drop the count" + print(f"hrx_f16_narrowing: {rel} has fewer f16 accumulators ({want} -> {got}) {suffix}") + if new or stale or unknown or acc_new or acc_stale or grew or shrank or lost or acc_grew or acc_shrank or acc_lost: print(f"hrx_f16_narrowing: FAIL ({len(new)} new, {len(stale)} stale of {len(baseline)}, " f"{len(unknown)} unclassified fptrunc; {len(acc_new)} new, {len(acc_stale)} stale of " f"{len(acc_baseline)} accumulator(s))") @@ -207,9 +298,12 @@ def write(root, corpus=None): return 77 head = "".join(line for line in (root / BASELINE).read_text(encoding="utf-8").splitlines(True) if line.lstrip().startswith("#")) - bodies = sorted(narrowing_bodies(corpus)) - (root / BASELINE).write_text(head + "\n".join(bodies) + "\n", encoding="utf-8") - accs = sorted(accumulator_bodies(corpus)) + counts = narrowing_lines(corpus) + lines = [f"{rel}\t{n}" for rel, n in sorted(counts.items())] + (root / BASELINE).write_text(head + "\n".join(lines) + "\n", encoding="utf-8") + bodies = sorted(counts) + acc_counts = accumulator_counts(corpus) + accs = [f"{rel}\t{n}" for rel, n in sorted(acc_counts.items())] acc_head = "".join(line for line in (root / ACC_BASELINE).read_text(encoding="utf-8").splitlines(True) if line.lstrip().startswith("#")) if (root / ACC_BASELINE).is_file() else "" (root / ACC_BASELINE).write_text(acc_head + "\n".join(accs) + "\n", encoding="utf-8") @@ -276,6 +370,40 @@ def check_that(ok, what): empty = root / "empty" empty.mkdir() check_that(check(empty) == 77, "a missing corpus returns 77, not 0") + # engine#2244 review: nothing reached check()'s FAIL condition, so deleting `grew` from it still passed + # --self-test. The failure set is the one thing this tool exists to enforce, so it needs an arm of its own: + # a listed body that gains a narrowing line must come back rc 1. + ck = root / "ck" + ck_corpus = ck / CORPUS + ck_corpus.mkdir(parents=True) + (ck / BASELINE).parent.mkdir(parents=True, exist_ok=True) + (ck_corpus / "listed.loom").write_text(" %x = scalar.fptrunc %y : f32 to f16\n") + (ck / BASELINE).write_text("# c\nlisted.loom 1\n") + (ck / ACC_BASELINE).write_text("# c\n") + check_that(check(ck) == 0, "check(): a corpus that matches its baseline is rc 0") + (ck_corpus / "listed.loom").write_text( + " %x = scalar.fptrunc %y : f32 to f16\n %z = scalar.fptrunc %w : f32 to f16\n") + check_that(check(ck) == 1, + "check(): one extra narrowing LINE in a listed body is rc 1 -- the fail condition is armed") + (ck_corpus / "listed.loom").write_text(" %x = scalar.constant 0.0 : f32\n") + check_that(check(ck) == 1, + "check(): a body that drops to zero is also rc 1, not a silent pass") + # engine#2244 review: the two cases "absent from counts" used to collapse into one `continue`. A body that + # went to ZERO narrowing lines is the outcome this ratchet exists for, and a body whose file was deleted + # must lose its baseline line instead of rotting quietly. + cdr = root / "cd" + cdr.mkdir() + (cdr / "clean.loom").write_text(" %x = scalar.constant 0.0 : f32\n") + (cdr / "still.loom").write_text(" %x = scalar.fptrunc %y : f32 to f16\n") + entries = {"clean.loom": 3, "still.loom": 1, "gone.loom": 2} + counts = narrowing_lines(cdr) + grew, shrank, lost = count_drift(entries, counts, cdr) + check_that(shrank == [("clean.loom", 3, 0)], + "a body that drops to zero narrowing lines is reported as a shrink, not as silence") + check_that(lost == [("gone.loom", 2)], + "a baseline body whose file is gone is reported stale, not skipped") + check_that(grew == [] and ("still.loom", 1, 1) not in shrank, + "an unchanged body is neither growth nor shrink") # engine#2186 item 1 / engine#2167: the accumulator class. A mul-updated f16 fragment is found; an add-only one # is not; a WMMA/contract update counts as a multiply; and an update far below the init is still found -- the # last is the fixture that would have caught the 120-line window, which reported four real accumulators as @@ -312,6 +440,36 @@ def check_that(ok, what): check_that(fp != corpus_fingerprint(c), "fingerprint: a one-byte edit changes it") (c / "a.loom").rename(c / "s" / "a.loom") check_that(fp != corpus_fingerprint(c), "fingerprint: a move changes it too (the path is hashed)") + with tempfile.TemporaryDirectory() as tmp: + root2 = pathlib.Path(tmp) + c2 = root2 / CORPUS + c2.mkdir(parents=True) + (c2 / "a.loom").write_text(" %x = scalar.fptrunc %y : f32 to f16\n") + check_that(narrowing_lines(c2) == {"a.loom": 1}, "the narrowing-line count of a listed body is counted") + (c2 / "a.loom").write_text(" %x = scalar.fptrunc %y : f32 to f16\n %z = scalar.fptrunc %w : f32 to f16\n") + check_that(narrowing_lines(c2) == {"a.loom": 2}, + "a second narrowing INSIDE an already-listed body raises the count (the body check alone would pass)") + (root2 / BASELINE).parent.mkdir(parents=True, exist_ok=True) + (root2 / BASELINE).write_text("a.loom\t2\n") + check_that(baseline_entries((root2 / BASELINE).read_text()) == {"a.loom": 2}, + "a 'path count' baseline line parses to both") + (root2 / BASELINE).write_text("a.loom\n") + check_that(baseline_entries((root2 / BASELINE).read_text()) == {"a.loom": None}, + "a path-only line still parses, so an older baseline keeps working") + c2b = root2 / "acc2" + c2b.mkdir() + (c2b / "one.loom").write_text( + " %init = vector.fragment %z shape [%m, %n] : vector<8xf16>\n" + " %next = vector.mulf %acc, %k : vector<8xf16>\n") + check_that(accumulator_counts(c2b) == {"one.loom": 1}, "an accumulator count of one is counted") + (c2b / "one.loom").write_text( + " %init = vector.fragment %z shape [%m, %n] : vector<8xf16>\n" + " %next = vector.mulf %acc, %k : vector<8xf16>\n" + " %init2 = vector.fragment %z2 shape [%m, %n] : vector<16xf16>\n" + " %next2 = vector.mulf %acc2, %k2 : vector<16xf16>\n") + check_that(accumulator_counts(c2b) == {"one.loom": 2}, + "a SECOND accumulator inside a listed body raises the count (the body check alone would pass)") + print(f"hrx_f16_narrowing self-test: {'ok' if not failures else f'{failures} failure(s)'}") return 1 if failures else 0 diff --git a/tools/ci/hrx_f16_narrowing.txt b/tools/ci/hrx_f16_narrowing.txt index 2c6eba434..8b7441ec5 100644 --- a/tools/ci/hrx_f16_narrowing.txt +++ b/tools/ci/hrx_f16_narrowing.txt @@ -19,54 +19,54 @@ # perfectly correct. What the list buys is that a NEW narrowing body cannot appear without someone # writing it down and saying what feeds it -- and that is the open question on engine#1962: which of # these are protected by a producer-side prescale cannot be answered by reading them. -loom-libs/motifs/bf16_f16.loom -loom-libs/motifs/dequant.loom -loom-libs/motifs/dequant_1bit.loom -loom-libs/motifs/dequant_prism.loom -loom-libs/motifs/f32_f16.loom -loom-libs/motifs/llm_attention_qkv_matmul_postops.loom -loom-libs/motifs/mul_mat_f32_f32_wmma_core.loom -loom-libs/motifs/mul_mat_id_f16_f16_wmma_core.loom -loom-libs/motifs/mul_mat_id_f32_f32_wmma_core.loom -loom-libs/motifs/mul_mat_id_q4k_f16_wmma_projection.loom -loom-libs/motifs/mul_mat_id_q5k_f16_wmma_projection.loom -loom-libs/motifs/mul_mat_id_q6k_f16_wmma_projection.loom -loom-libs/motifs/mul_mat_id_swiglu_f16_f16_accumulate.loom -loom-libs/motifs/mul_mat_id_swiglu_f16_f16_wmma_body.loom -loom-libs/motifs/mul_mat_quantized_f16_prefill.loom -loom-libs/motifs/publish_f32.loom -loom-libs/motifs/q4_k_f16.loom -loom-libs/motifs/q6_k_f16.loom -loom-libs/motifs/q8_0_f16.loom -loom-libs/motifs/q8_1_f16.loom -loom-libs/motifs/quantize_q8_1_x4.loom -loom-libs/ops/copy_f32.loom -loom-libs/ops/flash_attention_decode_split_f32_f16_wmma.loom -loom-libs/ops/flash_attention_f32_f16_wmma.loom -loom-libs/ops/gated_delta_net_f32_wmma.loom -loom-libs/ops/llm_attention_k_matmul_rope_set_rows_decode_f32_f32.loom -loom-libs/ops/llm_attention_v_matmul_set_rows_decode_f32_f32.loom -loom-libs/ops/mul_mat_id_swiglu_f32_f32_wmma.loom -loom-libs/ops/mul_mat_q6_k_packed_f16_wmma.loom -loom-libs/ops/mul_mat_swiglu_f32_f32_wmma.loom -loom-libs/ops/mul_mat_swiglu_symmetric_i4_wmma.loom -loom-libs/ops/mul_mat_symmetric_i4_wmma.loom -loom-libs/ops/mul_mat_symmetric_i8_wmma.loom -loom-libs/ops/rmsnorm_binary_f32.loom -loom-libs/ops/rope_set_rows_f32.loom -loom-libs/ops/set_rows.loom -loom-libs/ops/small_rows_f32.loom -qwen_moe/ggml/linear_q6k_q8_1_x4.loom -qwen_moe/ggml/quantize_q8_1_x4.loom -qwen_moe/qwen3_moe/attention_postprocess_f32_f16.loom -qwen_moe/qwen3_moe/attention_prepare_quantized.loom -qwen_moe/qwen3_moe/dense_linear_quantized_f16_wmma.loom -qwen_moe/qwen3_moe/flash_attention_decode_f32_f16_wmma.loom -qwen_moe/qwen3_moe/flash_attention_decode_q128_f32_f16_wmma.loom -qwen_moe/qwen3_moe/flash_attention_decode_split_f32_f16_wmma.loom -qwen_moe/qwen3_moe/flash_attention_decode_split_next_q8_test.loom -qwen_moe/qwen3_moe/flash_attention_f32_f16_wmma.loom -qwen_moe/qwen3_moe/routed_down_quantized_f16_wmma.loom -qwen_moe/qwen3_moe/routed_down_weighted_reduce_next_rmsnorm_q8_1_x4.loom -qwen_moe/qwen3_moe/routed_gate_up_swiglu_q4k.loom -qwen_moe/qwen3_moe/routed_linear_q4k_f16_wmma.loom +loom-libs/motifs/bf16_f16.loom 1 +loom-libs/motifs/dequant.loom 20 +loom-libs/motifs/dequant_1bit.loom 5 +loom-libs/motifs/dequant_prism.loom 2 +loom-libs/motifs/f32_f16.loom 1 +loom-libs/motifs/llm_attention_qkv_matmul_postops.loom 1 +loom-libs/motifs/mul_mat_f32_f32_wmma_core.loom 3 +loom-libs/motifs/mul_mat_id_f16_f16_wmma_core.loom 5 +loom-libs/motifs/mul_mat_id_f32_f32_wmma_core.loom 1 +loom-libs/motifs/mul_mat_id_q4k_f16_wmma_projection.loom 5 +loom-libs/motifs/mul_mat_id_q5k_f16_wmma_projection.loom 5 +loom-libs/motifs/mul_mat_id_q6k_f16_wmma_projection.loom 2 +loom-libs/motifs/mul_mat_id_swiglu_f16_f16_accumulate.loom 1 +loom-libs/motifs/mul_mat_id_swiglu_f16_f16_wmma_body.loom 1 +loom-libs/motifs/mul_mat_quantized_f16_prefill.loom 4 +loom-libs/motifs/publish_f32.loom 1 +loom-libs/motifs/q4_k_f16.loom 4 +loom-libs/motifs/q6_k_f16.loom 5 +loom-libs/motifs/q8_0_f16.loom 1 +loom-libs/motifs/q8_1_f16.loom 1 +loom-libs/motifs/quantize_q8_1_x4.loom 2 +loom-libs/ops/copy_f32.loom 2 +loom-libs/ops/flash_attention_decode_split_f32_f16_wmma.loom 2 +loom-libs/ops/flash_attention_f32_f16_wmma.loom 9 +loom-libs/ops/gated_delta_net_f32_wmma.loom 56 +loom-libs/ops/llm_attention_k_matmul_rope_set_rows_decode_f32_f32.loom 1 +loom-libs/ops/llm_attention_v_matmul_set_rows_decode_f32_f32.loom 2 +loom-libs/ops/mul_mat_id_swiglu_f32_f32_wmma.loom 1 +loom-libs/ops/mul_mat_q6_k_packed_f16_wmma.loom 16 +loom-libs/ops/mul_mat_swiglu_f32_f32_wmma.loom 5 +loom-libs/ops/mul_mat_swiglu_symmetric_i4_wmma.loom 5 +loom-libs/ops/mul_mat_symmetric_i4_wmma.loom 1 +loom-libs/ops/mul_mat_symmetric_i8_wmma.loom 1 +loom-libs/ops/rmsnorm_binary_f32.loom 2 +loom-libs/ops/rope_set_rows_f32.loom 2 +loom-libs/ops/set_rows.loom 2 +loom-libs/ops/small_rows_f32.loom 1 +qwen_moe/ggml/linear_q6k_q8_1_x4.loom 1 +qwen_moe/ggml/quantize_q8_1_x4.loom 2 +qwen_moe/qwen3_moe/attention_postprocess_f32_f16.loom 3 +qwen_moe/qwen3_moe/attention_prepare_quantized.loom 2 +qwen_moe/qwen3_moe/dense_linear_quantized_f16_wmma.loom 5 +qwen_moe/qwen3_moe/flash_attention_decode_f32_f16_wmma.loom 11 +qwen_moe/qwen3_moe/flash_attention_decode_q128_f32_f16_wmma.loom 3 +qwen_moe/qwen3_moe/flash_attention_decode_split_f32_f16_wmma.loom 4 +qwen_moe/qwen3_moe/flash_attention_decode_split_next_q8_test.loom 1 +qwen_moe/qwen3_moe/flash_attention_f32_f16_wmma.loom 9 +qwen_moe/qwen3_moe/routed_down_quantized_f16_wmma.loom 1 +qwen_moe/qwen3_moe/routed_down_weighted_reduce_next_rmsnorm_q8_1_x4.loom 2 +qwen_moe/qwen3_moe/routed_gate_up_swiglu_q4k.loom 2 +qwen_moe/qwen3_moe/routed_linear_q4k_f16_wmma.loom 8