From 0e1d7c918960f0f9e4c1d91fdcd3792caa2521c8 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 13:33:41 -0400 Subject: [PATCH] npu: NPU_INFER_DUMP_OPS prints one magnitude line per projection and norm (refs #1503, follows #1822) #1822's instrument, re-applied on current main with its open review answered: the variable is read once per process, the mean/absmax arithmetic and the line's text are tested, and the line is printed on the host arms and the device arm of both npu_gemm() and npu_gemm_rows(), so a host run and a device run can be compared line for line. Non-finite values are counted instead of vanishing from absmax. Co-Authored-By: Claude Fable 5.2 --- CMakeLists.txt | 8 +++ npu/forward/include/ops-dump.h | 94 +++++++++++++++++++++++++++ npu/forward/src/q4nx_forward.cpp | 21 +++++- tests/ops_dump_test.cpp | 107 +++++++++++++++++++++++++++++++ 4 files changed, 228 insertions(+), 2 deletions(-) create mode 100644 npu/forward/include/ops-dump.h create mode 100644 tests/ops_dump_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 71b63011b..304e7a1e5 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -1201,6 +1201,14 @@ target_link_libraries(npu_pack_test PRIVATE onebit_npu) # golden fixture that pins them. target_include_directories(npu_pack_test PRIVATE npu/forward/include ${CMAKE_SOURCE_DIR}/tests) +# NPU_INFER_DUMP_OPS (npu/forward/include/ops-dump.h, engine#1503): the on/off rule, the mean|x|/absmax arithmetic and +# the line's text. Header-only, no device. `add_dependencies(onebit ...)` is what gets it built in CI, whose target +# list is explicit (tools/ci/ci_targets_check.py). +add_executable(ops_dump_test tests/ops_dump_test.cpp) +target_include_directories(ops_dump_test PRIVATE npu/forward/include) +add_test(NAME ops_dump COMMAND ops_dump_test) +add_dependencies(onebit ops_dump_test) + # The device weight layout contract (q4nx_weight_layout): pure index math, no device # and no runtime. CI builds an explicit target list, so the check compiles itself with # the build's compiler at test time (tests/npu_blocked_layout_test.sh) rather than being diff --git a/npu/forward/include/ops-dump.h b/npu/forward/include/ops-dump.h new file mode 100644 index 000000000..1019bfa0b --- /dev/null +++ b/npu/forward/include/ops-dump.h @@ -0,0 +1,94 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ops-dump.h -- NPU_INFER_DUMP_OPS=1 prints mean|x| and absmax after every projection and every weighted RMSNorm, +// tagged with the call site's own name: +// +// [op] L00 rmsnorm rows=1 |x| mean 0.0105 absmax 0.0598 +// [op] L00 attn_qkv rows=1 |x| mean 1.3877 absmax 31.8909 +// +// Why it exists: NPU_INFER_DUMP_DIR (residual-dump.h) writes the residual stream at LAYER boundaries, which says +// that a model diverges and not which op did it (engine#1503: a residual already 1e4 too large at layer 0 while +// the container matched its pack tensor for tensor). The insertion points are `rmsnorm()`, `npu_gemm()` and +// `npu_gemm_rows()`; the tag each projection already carries is the label, so there is no second name table. +// The line is printed on the host arms and on the device arm alike, so the same run under NPU_INFER_HOME_GEMM=1 +// and without it can be compared line for line. +// +// Row 0 only (the first N values of a batched result): one line per call, which is what localises a divergence. +// +// Off unless the variable is set to something other than "" or "0". The variable is read ONCE per process +// (ops_dump_enabled() is called per projection per token, and off has to cost a load and a branch, not a +// getenv). It can never fail a run: no allocation, no file, stderr only. +#pragma once + +#include +#include +#include +#include + +namespace onebit::npu { + +// The rule itself, on a value: what the tests pin (the cached read below cannot be re-armed inside one process). +inline bool ops_dump_value_on(const char* v) { return v && *v && std::strcmp(v, "0") != 0; } + +inline bool ops_dump_enabled() { + static const bool on = ops_dump_value_on(std::getenv("NPU_INFER_DUMP_OPS")); + return on; +} + +struct OpsStats { + double mean = 0.0; // mean of |x[i]| + float absmax = 0.0f; // max of |x[i]| + int nonfinite = 0; // NaN/Inf count: they are counted, and left out of mean and absmax +}; + +// The whole content of the diagnostic. A NaN would otherwise vanish from absmax (`a > mx` is false) and turn the +// mean into "nan" with no count, so non-finite values are reported as a number of their own. +inline OpsStats ops_stats(const float* x, int n) { + OpsStats s; + if (!x || n <= 0) return s; + double sum = 0.0; + int finite = 0; + for (int i = 0; i < n; ++i) { + if (!std::isfinite(x[i])) { ++s.nonfinite; continue; } + const float a = std::fabs(x[i]); + sum += a; + if (a > s.absmax) s.absmax = a; + ++finite; + } + if (finite > 0) s.mean = sum / finite; + return s; +} + +// Formats the line into `buf` (always NUL-terminated when cap > 0) and returns snprintf's count; 0 when there is +// nothing to say. Separate from the print so a test can match the text. +inline int ops_dump_line(char* buf, size_t cap, const char* tag, int layer, int rows, const float* x, int n) { + if (!buf || cap == 0) return 0; + buf[0] = '\0'; + if (!x || n <= 0) return 0; + const OpsStats s = ops_stats(x, n); + if (s.nonfinite) + return std::snprintf(buf, cap, " [op] L%02d %-10s rows=%d |x| mean %.6g absmax %.6g nonfinite %d/%d\n", layer, + tag ? tag : "?", rows, s.mean, (double)s.absmax, s.nonfinite, n); + return std::snprintf(buf, cap, " [op] L%02d %-10s rows=%d |x| mean %.6g absmax %.6g\n", layer, tag ? tag : "?", + rows, s.mean, (double)s.absmax); +} + +inline void ops_dump(const char* tag, int layer, int rows, const float* x, int n) { + char buf[192]; + if (ops_dump_line(buf, sizeof buf, tag, layer, rows, x, n) > 0) std::fputs(buf, stderr); +} + +} // namespace onebit::npu diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 09cba1ef7..971965d2c 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -34,6 +34,7 @@ #include #include #include "batch_check_compare.h" +#include "ops-dump.h" #include "residual-dump.h" #include #include @@ -3207,6 +3208,14 @@ void Q4nxNpuForward::host_gemm_i8(const float* A, int a_k, int M, int K, int N, } } +// NPU_INFER_DUMP_OPS (ops-dump.h): one line per projection, tagged with the tag its caller already passes +// ("attn_qkv", "ssm_out", ...). Called where a projection's result is final -- both host arms and the device arm -- +// so a host run and a device run print the same lines. Row 0 of the result; off is one cached bool. +static inline void op_line(const char* tag, int layer, int rows, const std::vector& out, int n) { + if (onebit::npu::ops_dump_enabled() && n > 0 && out.size() >= (size_t)n) + onebit::npu::ops_dump(tag, layer, rows, out.data(), n); +} + void Q4nxNpuForward::npu_gemm(int role, const float* A, int K, int N, const float* Bmat, std::vector& out, int wkey, const char* sc, int a_k) { if (host_gemm_) { @@ -3217,9 +3226,11 @@ void Q4nxNpuForward::npu_gemm(int role, const float* A, int K, int N, const floa out.assign((size_t)N, 0.0f); if (host_gemm_i8_) { host_gemm_i8(A, a_k, 1, K, N, Bmat, wkey, out.data()); + op_line(sc, cur_layer_, 1, out, N); return; } q4nx_host_gemm_f32_padded(A, a_k, 1, K, N, Bmat, out.data()); + op_line(sc, cur_layer_, 1, out, N); return; } npu::target::GemvArgs a; @@ -3228,7 +3239,8 @@ void Q4nxNpuForward::npu_gemm(int role, const float* A, int K, int N, const floa a.a_k = a_k; a.tiles = &Q4nxNpuForward::gemv_tile_cb; a.tiles_self = this; std::string err; - if (!xd_ || !xd_->gemv(a, out, err)) err_ = err.empty() ? "npu_gemm: no target" : err; + if (!xd_ || !xd_->gemv(a, out, err)) { err_ = err.empty() ? "npu_gemm: no target" : err; return; } + op_line(sc, cur_layer_, 1, out, N); } void Q4nxNpuForward::npu_gemm_rows(int role, const float* A, int M, int K, int N, @@ -3240,9 +3252,11 @@ void Q4nxNpuForward::npu_gemm_rows(int role, const float* A, int M, int K, int N out.assign((size_t)M * N, 0.0f); if (host_gemm_i8_) { host_gemm_i8(A, a_k, M, K, N, Bmat, wkey, out.data()); + op_line(sc, cur_layer_, M, out, N); return; } q4nx_host_gemm_f32_padded(A, a_k, M, K, N, Bmat, out.data()); + op_line(sc, cur_layer_, M, out, N); return; } // The device's activation BO holds exactly max_batch_rows() rows (the M=128 prefill design), and a wider @@ -3282,7 +3296,8 @@ void Q4nxNpuForward::npu_gemm_rows(int role, const float* A, int M, int K, int N a.rows = true; a.decode_twin = decode_twin; a.tiles = &Q4nxNpuForward::gemv_tile_cb; a.tiles_self = this; std::string err; - if (!xd_ || !xd_->gemv(a, out, err)) err_ = err.empty() ? "npu_gemm_rows: no target" : err; + if (!xd_ || !xd_->gemv(a, out, err)) { err_ = err.empty() ? "npu_gemm_rows: no target" : err; return; } + op_line(sc, cur_layer_, M, out, N); } // The folded container's sign table: magic "HAD1", u32 count, count x (u32 width, u32 offset), @@ -4028,6 +4043,8 @@ float Q4nxNpuForward::rmsnorm(const float* x, const float* w, int n, float* out) } else { for (int i = 0; i < n; i++) out[i] = x[i] * inv * w[i]; } + // NPU_INFER_DUMP_OPS: the norm's own output, so a divergence can be put on the norm and not on its layer. + if (onebit::npu::ops_dump_enabled()) onebit::npu::ops_dump("rmsnorm", cur_layer_, 1, out, n); return inv; } diff --git a/tests/ops_dump_test.cpp b/tests/ops_dump_test.cpp new file mode 100644 index 000000000..971f92964 --- /dev/null +++ b/tests/ops_dump_test.cpp @@ -0,0 +1,107 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// ops_dump_test -- NPU_INFER_DUMP_OPS (ops-dump.h): the on/off rule, the arithmetic of the line, its text, and +// that the variable is read once per process. No device, no container, no model. +// +// ops_dump_test the checks below +// ops_dump_test --enabled prints 1 or 0: what ops_dump_enabled() says in a process started with this environment +#include "ops-dump.h" + +#include +#include +#include +#include +#include +#include + +static int failures = 0; +static void check(bool ok, const std::string& what) { + if (ok) return; + ++failures; + std::printf("FAIL %s\n", what.c_str()); +} + +// What a fresh process decides for one environment: the cached read cannot be re-armed in this one. +static int child_says(const char* self, const char* env_prefix) { + const std::string cmd = std::string(env_prefix) + " '" + self + "' --enabled"; + FILE* p = popen(cmd.c_str(), "r"); + if (!p) return -1; + int c = std::fgetc(p); + pclose(p); + return c == '1' ? 1 : c == '0' ? 0 : -1; +} + +int main(int argc, char** argv) { + using namespace onebit::npu; + if (argc > 1 && !std::strcmp(argv[1], "--enabled")) { + std::printf("%d\n", ops_dump_enabled() ? 1 : 0); + return 0; + } + + // The rule. + check(!ops_dump_value_on(nullptr), "unset is off"); + check(!ops_dump_value_on(""), "an empty value is off"); + check(!ops_dump_value_on("0"), "\"0\" is off"); + check(ops_dump_value_on("1"), "\"1\" is on"); + check(ops_dump_value_on("00"), "anything else is on"); + + // The rule as a process sees it. + check(child_says(argv[0], "env -u NPU_INFER_DUMP_OPS") == 0, "process: unset is off"); + check(child_says(argv[0], "env NPU_INFER_DUMP_OPS=") == 0, "process: empty is off"); + check(child_says(argv[0], "env NPU_INFER_DUMP_OPS=0") == 0, "process: 0 is off"); + check(child_says(argv[0], "env NPU_INFER_DUMP_OPS=1") == 1, "process: 1 is on"); + + // Read once: whatever this process decided first, a later change of the variable does not move it. + const bool first = ops_dump_enabled(); + setenv("NPU_INFER_DUMP_OPS", first ? "0" : "1", 1); + check(ops_dump_enabled() == first, "the variable is read once per process"); + + // The arithmetic: mean of |x| over n, and the largest |x|. + const float x[] = {1.0f, -3.0f, 2.0f, 0.0f}; + OpsStats s = ops_stats(x, 4); + check(s.mean == 1.5, "mean|x| of {1,-3,2,0} is 1.5"); + check(s.absmax == 3.0f, "absmax of {1,-3,2,0} is 3 (the negative one)"); + check(s.nonfinite == 0, "no non-finite value in a finite array"); + s = ops_stats(x, 2); + check(s.mean == 2.0 && s.absmax == 3.0f, "n bounds the read: {1,-3} gives mean 2, absmax 3"); + s = ops_stats(x, 0); + check(s.mean == 0.0 && s.absmax == 0.0f, "n = 0 is all zeros, not a division by zero"); + s = ops_stats(nullptr, 4); + check(s.mean == 0.0 && s.absmax == 0.0f, "a null array is all zeros"); + + // Non-finite values are counted and kept out of the two numbers. + const float bad[] = {1.0f, std::numeric_limits::quiet_NaN(), -5.0f, std::numeric_limits::infinity()}; + s = ops_stats(bad, 4); + check(s.nonfinite == 2, "NaN and Inf are counted"); + check(s.mean == 3.0 && s.absmax == 5.0f, "mean and absmax are over the finite values"); + + // The text. + char buf[192]; + int n = ops_dump_line(buf, sizeof buf, "attn_qkv", 7, 16, x, 4); + check(std::string(buf) == " [op] L07 attn_qkv rows=16 |x| mean 1.5 absmax 3\n", std::string("line text, got: ") + buf); + check(n == (int)std::strlen(buf), "the count is the line's length"); + ops_dump_line(buf, sizeof buf, "rmsnorm", 0, 1, bad, 4); + check(std::string(buf) == " [op] L00 rmsnorm rows=1 |x| mean 3 absmax 5 nonfinite 2/4\n", + std::string("non-finite line text, got: ") + buf); + check(ops_dump_line(buf, sizeof buf, "t", 0, 1, nullptr, 4) == 0 && buf[0] == '\0', "a null array prints nothing"); + check(ops_dump_line(buf, sizeof buf, "t", 0, 1, x, 0) == 0 && buf[0] == '\0', "n = 0 prints nothing"); + char small[8]; + ops_dump_line(small, sizeof small, "attn_qkv", 7, 16, x, 4); + check(std::strlen(small) == sizeof small - 1, "a short buffer is truncated and terminated"); + + std::printf("%s\n", failures ? "FAILED" : "ops_dump: ok"); + return failures ? 1 : 0; +}