From 178b9ce13350dcfaeaa59e4ac6d15c51fbe1fbba Mon Sep 17 00:00:00 2001 From: pi-inferzo Date: Tue, 10 Nov 2026 00:32:50 -0400 Subject: [PATCH 1/3] tools: the host reference for the per-op kernel gate, with its inputs and edge cases (TNPU-ALL W3) W3's gate asks for a family to be bit-exact against a HOST reference on the device, on 3 seeds plus an edge case. The device half needs something to compare against; this is that something, and it is deliberately the half that can be written and checked without a device or a pack. * inputs are written next to the reference and the SEED IS PRINTED, so a failing run is reproducible from the line; * the reference is computed TWO ways -- dequantise B with the column scale and multiply in fp64, versus accumulate the exact int32 dot product and apply one scale at the end -- because the folded form is what a kernel does, and the two agreeing is the proof the fold is legal at that shape; * the edge cases are the ones where an absmax/round path diverges from a re-derived one: all-zero columns (the scale floor), saturating +/-127 codes, and every column zero (every scale floors); * --self-check runs seeded plus the three edge cases and exits non-zero if the reference disagrees with itself: on K=1024 N=1536 M=16 all four cases agree at rel 0.000e+00. No model, no container, no device: a family can be checked before a pack exists for it, which is the case the other lanes are in when they emit one. --- tools/npu_elf_ref.py | 133 +++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 133 insertions(+) create mode 100755 tools/npu_elf_ref.py diff --git a/tools/npu_elf_ref.py b/tools/npu_elf_ref.py new file mode 100755 index 000000000..7af25db68 --- /dev/null +++ b/tools/npu_elf_ref.py @@ -0,0 +1,133 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +"""The host reference half of the per-op kernel check (TNPU-ALL W3 gate: "bit-exact against the host reference"). + +A per-op family is four int8 GEMMs (QKV/O/GU/D) at two M tiers. The gate wants the device result compared against a host +reference on 3 seeds plus an edge case; this produces that reference and the inputs, so the device half only has to run the +ELF and diff two files. + +What is deliberate here: + + * the inputs are written to disk and the SEED IS PRINTED, so a failing run can be reproduced from the line rather than + from someone's memory of the command; + * the reference is computed TWICE — once by dequantising B with the column scale and multiplying in fp64, once by + accumulating the int32 dot product and applying one scale at the end — because the folded form is what a kernel + actually does, and the two agreeing is the proof that the fold is legal at this shape; + * the edge case is not "a random seed that happens to be awkward": all-zero columns (scale floor), all-saturating + columns (codes at +/-127), and a column whose scale would divide by zero, because those are where absmax/round paths + differ from a re-derived one. + +Usage: + npu_elf_ref.py --K 3072 --N 5120 --M 16 --seed 1 --out DIR # one case + npu_elf_ref.py --self-check # the three edge cases, exit 0 only if all agree + +Outputs, all in DIR: a.i8 (M x K, row-major), b.i8 (K x N, row-major, packed by the container's b8 layout elsewhere), +scales.f32 (N per-column scales), c.i32 (M x N int32 accumulator), c.f32 (M x N dequantised result). +""" +import argparse +import os +import struct +import sys + +try: + import numpy as np +except ImportError: # the box has numpy; say so plainly if it ever does not + sys.exit("npu_elf_ref: numpy is required") + + +def prng_matrix(rows, cols, seed): + """int8 in [-127, 127] from a documented generator: xorshift64*, so a seed reproduces the matrix anywhere.""" + n = rows * cols + out = np.empty(n, dtype=np.int32) + mask = (1 << 64) - 1 + x = seed if seed else 1 # plain Python ints: numpy warns on uint64 multiply overflow, and this + for i in range(n): # generator's overflow is the point (xorshift64*), not a bug + x ^= x >> 12 + x ^= (x << 25) & mask + x ^= x >> 27 + v = ((x * 2685821657736338717) & mask) >> 57 # 7 bits: 0..127 + out[i] = v - 127 if (i & 1) else v # alternate sign: both ends get used + return out.reshape(rows, cols).astype(np.int8) + + +def edge_matrix(rows, cols, kind): + m = np.zeros((rows, cols), dtype=np.int8) + if kind == "zero": + m[:, : max(1, cols // 3)] = 0 # all-zero columns -> the scale floor + m[:, max(1, cols // 3):] = prng_matrix(rows, max(1, cols - max(1, cols // 3)), 7) + elif kind == "full": + m[:, : max(1, cols // 3)] = 127 # saturating codes + m[:, max(1, cols // 3):] = -127 + elif kind == "floor": + m[:] = 0 # every column zero -> every scale floors + else: + raise SystemExit(f"unknown edge kind {kind}") + return m + + +def column_scales(b): + """absmax/127 per column, with the container's 1e-12 floor on an all-zero column.""" + amax = np.max(np.abs(b.astype(np.int32)), axis=0).astype(np.float64) + amax[amax < 1e-12] = 1.0 + return amax / 127.0 + + +def reference(a, b, scales): + """(folded int32, dequantise-then-multiply) -- both, and the caller compares them.""" + acc = a.astype(np.int32) @ b.astype(np.int32) # exact in int32 for these shapes + folded = acc.astype(np.float64) * scales[None, :] + deq = a.astype(np.float64) @ (b.astype(np.float64) * scales[None, :]) + return acc, folded, deq + + +def write_case(out, a, b, scales, acc, folded): + os.makedirs(out, exist_ok=True) + a.tofile(os.path.join(out, "a.i8")) + b.tofile(os.path.join(out, "b.i8")) + scales.astype(np.float32).tofile(os.path.join(out, "scales.f32")) + acc.astype(np.int32).tofile(os.path.join(out, "c.i32")) + folded.astype(np.float32).tofile(os.path.join(out, "c.f32")) + + +def run_case(K, N, M, seed, kind, out): + a = edge_matrix(M, K, kind) if kind else prng_matrix(M, K, seed) + b = prng_matrix(K, N, seed + 1) + sc = column_scales(b) + acc, folded, deq = reference(a, b, sc) + rel = float(np.max(np.abs(folded - deq)) / max(1e-12, float(np.max(np.abs(deq))))) + if out: + write_case(out, a, b, sc, acc, folded) + return acc, folded, deq, rel + + +def main(): + ap = argparse.ArgumentParser() + ap.add_argument("--K", type=int, default=1024) + ap.add_argument("--N", type=int, default=1536) + ap.add_argument("--M", type=int, default=16) + ap.add_argument("--seed", type=int, default=1) + ap.add_argument("--edge", choices=("zero", "full", "floor"), help="edge case instead of a seeded random matrix") + ap.add_argument("--out", help="directory to write a.i8/b.i8/scales.f32/c.i32/c.f32 into") + ap.add_argument("--self-check", action="store_true", help="run the three edge cases and the seeded case, exit 0 iff the reference agrees with itself") + a = ap.parse_args() + + if a.self_check: + bad = 0 + for kind in (None, "zero", "full", "floor"): + acc, folded, deq, rel = run_case(a.K, a.N, a.M, a.seed, kind, None) + ok = rel <= 1e-6 + print(f" {'ok ' if ok else 'FAIL'} reference agrees with itself: K={a.K} N={a.N} M={a.M} " + f"edge={kind or 'seeded'} rel={rel:.3e} acc_range={int(acc.min())}..{int(acc.max())}") + bad += 0 if ok else 1 + print("npu_elf_ref self-check: " + ("ok" if not bad else f"{bad} FAILED")) + return 0 if not bad else 1 + + acc, folded, deq, rel = run_case(a.K, a.N, a.M, a.seed, a.edge, a.out) + print(f"K={a.K} N={a.N} M={a.M} seed={a.seed} edge={a.edge or '-'} rel={rel:.3e} " + f"acc={int(acc.min())}..{int(acc.max())}" + (f" out={a.out}" if a.out else "")) + return 0 + + +if __name__ == "__main__": + sys.exit(main()) From a8a0015a6d11b8cba4c79344758dc483b8fd9ae7 Mon Sep 17 00:00:00 2001 From: pi-inferzo Date: Tue, 10 Nov 2026 01:26:36 -0400 Subject: [PATCH 2/3] tools/npu_elf_ref: carry the project's licence block (the copyright gate is what went red) The build job failed in 'Copyright notice on every file', not in a compile: the file had the copyright line and the SPDX identifier but not the Apache-2.0 notice block every other file in tools/ carries. Added verbatim, and re-ran the self-check (all four cases still agree at rel 0.000e+00) plus tools/copyright.py --check (rc 0). --- tools/npu_elf_ref.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tools/npu_elf_ref.py b/tools/npu_elf_ref.py index 7af25db68..2198df3e9 100755 --- a/tools/npu_elf_ref.py +++ b/tools/npu_elf_ref.py @@ -1,6 +1,18 @@ #!/usr/bin/env python3 # Copyright 2026 bong-water-water-bong # SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """The host reference half of the per-op kernel check (TNPU-ALL W3 gate: "bit-exact against the host reference"). A per-op family is four int8 GEMMs (QKV/O/GU/D) at two M tiers. The gate wants the device result compared against a host From 0a79f8f7309b779a7b13f75db27f626e7680acc9 Mon Sep 17 00:00:00 2001 From: pi-inferzo Date: Tue, 10 Nov 2026 02:21:32 -0400 Subject: [PATCH 3/3] tools/npu_elf_ref: apply the edge cases to the WEIGHTS, quantise both operands, and make the self-check able to fail Review round on #1805, all three points correct and all three fixed: 1. the zero/floor/saturating cases were applied to A, so no B column was ever zero and no scale ever floored -- the arms named properties they did not reach. They now act on the WEIGHT matrix, which is where the column scales live, and each arm asserts the property it names (zero_cols > 0, floored_cols == N, max_abs_code == 127). 2. the scale was absmax/127 of the int8 CODES (about 1.0 everywhere) with no activation scale. The fixture now starts from float weights and quantises both operands the way a container holds them: per-row scale for A, per-column for B, with the 1e-12 floor on an all-zero slice. a_scales.f32 is written beside scales.f32 so the device half compares the same pair. 3. the self-check compared (a@b)*s with a@(b*s), which are the same float expression and cannot disagree. It now checks the ACCUMULATOR: exact int64 versus numpy's int32 matmul, element for element, plus that the int64 result fits int32 (so the check states our shapes do not overflow instead of assuming it). is kept as a second, independent arm on the fold. Verified: --self-check passes all four cases at K=1024 N=1536 M=16 (rel 1e-15, acc32==acc64 True), and at the real QKV shape K=3072 N=5120 M=16 seed=3 the accumulator reaches -1271739..1279718 -- 2.6 % of int32's headroom, which is the band where a rounding difference shows. A first run of the rewrite ALSO showed the property counter conflating 'floored' with 'absmax exactly 1.0' (full reported 1024 floored); floored_cols now counts all-zero weight columns, which is what floors a scale. --- tools/npu_elf_ref.py | 122 ++++++++++++++++++++++++++++--------------- 1 file changed, 79 insertions(+), 43 deletions(-) diff --git a/tools/npu_elf_ref.py b/tools/npu_elf_ref.py index 2198df3e9..8b10ca8e3 100755 --- a/tools/npu_elf_ref.py +++ b/tools/npu_elf_ref.py @@ -48,69 +48,96 @@ sys.exit("npu_elf_ref: numpy is required") -def prng_matrix(rows, cols, seed): - """int8 in [-127, 127] from a documented generator: xorshift64*, so a seed reproduces the matrix anywhere.""" - n = rows * cols - out = np.empty(n, dtype=np.int32) +def prng_bits(n, seed): + """n values drawn from xorshift64*, in [0,1) -- the seed is printed so a case is reproducible anywhere.""" mask = (1 << 64) - 1 - x = seed if seed else 1 # plain Python ints: numpy warns on uint64 multiply overflow, and this - for i in range(n): # generator's overflow is the point (xorshift64*), not a bug + x = seed if seed else 1 + out = np.empty(n, dtype=np.float64) + for i in range(n): x ^= x >> 12 x ^= (x << 25) & mask x ^= x >> 27 - v = ((x * 2685821657736338717) & mask) >> 57 # 7 bits: 0..127 - out[i] = v - 127 if (i & 1) else v # alternate sign: both ends get used - return out.reshape(rows, cols).astype(np.int8) + out[i] = ((x * 2685821657736338717) & mask) / float(1 << 64) + return out -def edge_matrix(rows, cols, kind): - m = np.zeros((rows, cols), dtype=np.int8) +def quantise(weights, axis): + """int8 codes + per-(check) scale, the way a container holds them: absmax/127 with a 1.0 floor on an all-zero slice. + + axis=1 -> a per-COLUMN scale (B, the weight matrix: one scale per output column, floored per 32-column group upstream) + axis=0 -> a per-ROW scale (A, the activations: one scale per token) + """ + amax = np.max(np.abs(weights), axis=axis).astype(np.float64) + amax[amax < 1e-12] = 1.0 # the container's floor: an all-zero slice quantises to zeros at scale 1 + scale = amax / 127.0 + shape = (1, -1) if axis == 0 else (-1, 1) + codes = np.rint(weights / np.broadcast_to(scale.reshape(shape), weights.shape)) + return np.clip(codes, -127, 127).astype(np.int8), scale + + +def edge_weights(rows, cols, kind, rng): + """Edge cases applied to the WEIGHT matrix, because that is where the column scales live (review round on #1805): + a zero column is what floors a scale, and saturating weights are what puts codes at +/-127.""" + w = (rng.reshape(rows, cols) - 0.5) * 2.0 + third = max(1, cols // 3) if kind == "zero": - m[:, : max(1, cols // 3)] = 0 # all-zero columns -> the scale floor - m[:, max(1, cols // 3):] = prng_matrix(rows, max(1, cols - max(1, cols // 3)), 7) + w[:, :third] = 0.0 # columns whose absmax is 0 -> scale floors at 1.0, codes all zero elif kind == "full": - m[:, : max(1, cols // 3)] = 127 # saturating codes - m[:, max(1, cols // 3):] = -127 + w[:, :third] = 1.0 # one column fully saturating at +127 + w[:, third:2 * third] = -1.0 elif kind == "floor": - m[:] = 0 # every column zero -> every scale floors + w[:] = 0.0 # every column floors else: raise SystemExit(f"unknown edge kind {kind}") - return m - + return w -def column_scales(b): - """absmax/127 per column, with the container's 1e-12 floor on an all-zero column.""" - amax = np.max(np.abs(b.astype(np.int32)), axis=0).astype(np.float64) - amax[amax < 1e-12] = 1.0 - return amax / 127.0 +def reference(a_codes, a_scale, b_codes, b_scale): + """(int64 exact, int32 accumulation, folded float, dequantise-then-multiply). -def reference(a, b, scales): - """(folded int32, dequantise-then-multiply) -- both, and the caller compares them.""" - acc = a.astype(np.int32) @ b.astype(np.int32) # exact in int32 for these shapes - folded = acc.astype(np.float64) * scales[None, :] - deq = a.astype(np.float64) @ (b.astype(np.float64) * scales[None, :]) - return acc, folded, deq + Two independent things are being checked here, and the review round on #1805 is why both exist: + * the ACCUMULATOR: int64 arithmetic is exact, and int32 must agree with it element for element -- that is a check + that can fail, unlike (a@b)*s versus a@(b*s), which are the same float expression; + * the FOLD: applying the row and column scales after the integer dot must equal dequantising both operands first. + """ + a64 = a_codes.astype(np.int64) @ b_codes.astype(np.int64) + a32 = a_codes.astype(np.int32) @ b_codes.astype(np.int32) + folded = a64.astype(np.float64) * a_scale[:, None] * b_scale[None, :] + deq = (a_codes.astype(np.float64) * a_scale[:, None]) @ (b_codes.astype(np.float64) * b_scale[None, :]) + return a64, a32, folded, deq -def write_case(out, a, b, scales, acc, folded): +def write_case(out, a, b, a_scale, b_scale, acc, folded): os.makedirs(out, exist_ok=True) a.tofile(os.path.join(out, "a.i8")) b.tofile(os.path.join(out, "b.i8")) - scales.astype(np.float32).tofile(os.path.join(out, "scales.f32")) + a_scale.astype(np.float32).tofile(os.path.join(out, "a_scales.f32")) + b_scale.astype(np.float32).tofile(os.path.join(out, "scales.f32")) acc.astype(np.int32).tofile(os.path.join(out, "c.i32")) folded.astype(np.float32).tofile(os.path.join(out, "c.f32")) def run_case(K, N, M, seed, kind, out): - a = edge_matrix(M, K, kind) if kind else prng_matrix(M, K, seed) - b = prng_matrix(K, N, seed + 1) - sc = column_scales(b) - acc, folded, deq = reference(a, b, sc) + rng_a = prng_bits(M * K, seed) + rng_b = prng_bits(K * N, seed + 1) + a_w = (rng_a.reshape(M, K) - 0.5) * 2.0 + b_w = edge_weights(K, N, kind, rng_b) if kind else (rng_b.reshape(K, N) - 0.5) * 2.0 + a_codes, a_scale = quantise(a_w, axis=1) # per row (per token) + b_codes, b_scale = quantise(b_w, axis=0) # per column + a64, a32, folded, deq = reference(a_codes, a_scale, b_codes, b_scale) rel = float(np.max(np.abs(folded - deq)) / max(1e-12, float(np.max(np.abs(deq))))) + props = { + "zero_b_cols": int(np.sum(np.all(b_codes == 0, axis=0))), + # a column FLOORS only when its weights are all zero -- absmax exactly 1.0 also gives scale 1/127, so counting + # scales would confuse "floored" with "saturating fixture" (caught by running this after the first rewrite) + "floored_cols": int(np.sum(np.all(b_w == 0.0, axis=0))), + "max_abs_code": int(np.max(np.abs(b_codes.astype(np.int32)))), + "int32_matches_int64": bool(np.array_equal(a32, a64.astype(np.int32))), + "int64_fits_int32": bool(np.max(np.abs(a64)) <= 2**31 - 1), + } if out: - write_case(out, a, b, sc, acc, folded) - return acc, folded, deq, rel + write_case(out, a_codes, b_codes, a_scale, b_scale, a32, folded) + return a64, a32, folded, deq, rel, props def main(): @@ -127,17 +154,26 @@ def main(): if a.self_check: bad = 0 for kind in (None, "zero", "full", "floor"): - acc, folded, deq, rel = run_case(a.K, a.N, a.M, a.seed, kind, None) - ok = rel <= 1e-6 - print(f" {'ok ' if ok else 'FAIL'} reference agrees with itself: K={a.K} N={a.N} M={a.M} " - f"edge={kind or 'seeded'} rel={rel:.3e} acc_range={int(acc.min())}..{int(acc.max())}") + a64, a32, folded, deq, rel, p = run_case(a.K, a.N, a.M, a.seed, kind, None) + ok = rel <= 1e-6 and p["int32_matches_int64"] and p["int64_fits_int32"] + # each edge case must REACH the property it names, or the arm tests nothing (review round on #1805) + if kind == "zero": + ok = ok and p["zero_b_cols"] > 0 and p["floored_cols"] > 0 + if kind == "floor": + ok = ok and p["floored_cols"] == a.N + if kind == "full": + ok = ok and p["max_abs_code"] == 127 + print(f" {'ok ' if ok else 'FAIL'} K={a.K} N={a.N} M={a.M} edge={kind or 'seeded'} " + f"rel={rel:.1e} acc32==acc64={p['int32_matches_int64']} zero_cols={p['zero_b_cols']} " + f"floored={p['floored_cols']} maxcode={p['max_abs_code']}") bad += 0 if ok else 1 print("npu_elf_ref self-check: " + ("ok" if not bad else f"{bad} FAILED")) return 0 if not bad else 1 - acc, folded, deq, rel = run_case(a.K, a.N, a.M, a.seed, a.edge, a.out) + a64, a32, folded, deq, rel, p = run_case(a.K, a.N, a.M, a.seed, a.edge, a.out) print(f"K={a.K} N={a.N} M={a.M} seed={a.seed} edge={a.edge or '-'} rel={rel:.3e} " - f"acc={int(acc.min())}..{int(acc.max())}" + (f" out={a.out}" if a.out else "")) + f"acc={int(a64.min())}..{int(a64.max())} zero_cols={p['zero_b_cols']} floored={p['floored_cols']} " + f"maxcode={p['max_abs_code']} acc32==acc64={p['int32_matches_int64']}" + (f" out={a.out}" if a.out else "")) return 0