From ddcb0ee3a06569b0f39652f31771238c2272b141 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 18:09:16 -0400 Subject: [PATCH] npu: NPU_INFER_DUMP_OPS says so once at model load, so a stale binary is visible (engine#2028) engine#2028 reported the per-op dump silent on the design path. It was a binary linked before #1977 (cl-npupack-b measured 869/1061 [op] lines on one that has it, and 0 on the filer's, whose strings lack the variable's name), but four runs went to the wrong mechanism first, because "the dump prints nothing" and "this binary has no dump" look identical. ops_dump_announce() prints [ops-dump] NPU_INFER_DUMP_OPS is on: an [op] line follows every projection and weighted RMSNorm, on stderr once per process when the dump is on, from Q4nxNpuForward::init_impl (serve and npu-run, host and device). No line with the variable set means the binary predates the dump. Pinned in ops_dump_test: on prints it once (second call silent), unset and 0 print nothing. On a two-layer container the line comes before the first of 316 [op] lines; with the variable off there is none. Co-Authored-By: Claude Opus 5.5 --- docs/npu.md | 5 ++++- npu/forward/include/ops-dump.h | 18 ++++++++++++++++++ npu/forward/src/q4nx_forward.cpp | 3 +++ tests/ops_dump_test.cpp | 30 ++++++++++++++++++++++++++++++ 4 files changed, 55 insertions(+), 1 deletion(-) diff --git a/docs/npu.md b/docs/npu.md index 2448c851c..0ee142606 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -2186,7 +2186,10 @@ can fail a run. | `NPU_INFER_DUMP_DIR=` | `/layer_.f32` per layer: the residual stream after that layer, M x H little-endian f32, row-major. The directory must exist; a file that cannot be written is reported on stderr and the run goes on. | which **layer** two runs part at (per-layer cosine against a reference trace) | | `NPU_INFER_DUMP_OPS=1` | one stderr line per projection and per weighted RMSNorm: `[op] L rows= \|x\| mean absmax ` | which **op** inside the layer | -`NPU_INFER_DUMP_OPS` is on for any value other than empty or `0`, and is read once per process. +`NPU_INFER_DUMP_OPS` is on for any value other than empty or `0`, and is read once per process. When it is on, the forward prints +`[ops-dump] NPU_INFER_DUMP_OPS is on: …` once at model load, before any request (engine#2028). A run that sets the +variable and never sees that line is running a binary built before the dump existed (#1977): rebuild it, rather than +looking for a code path that skips the dump. ``` [op] L00 rmsnorm rows=1 |x| mean 0.0143171 absmax 3.01935 diff --git a/npu/forward/include/ops-dump.h b/npu/forward/include/ops-dump.h index 42b3d0fd6..9fb15babc 100644 --- a/npu/forward/include/ops-dump.h +++ b/npu/forward/include/ops-dump.h @@ -36,6 +36,7 @@ // getenv). It can never fail a run: no allocation, no file, stderr only. #pragma once +#include #include #include #include @@ -109,4 +110,21 @@ inline void ops_dump(const char* tag, int layer, int rows, const float* x, int n if (ops_dump_line(buf, sizeof buf, tag, layer, rows, x, n) > 0) std::fputs(buf, stderr); } +// engine#2028: the line a process prints once, at model load, when the dump is armed. A run that sets the variable and +// does not see this line before its first request is running a binary from before the instrument (#1977): four silent +// phi-4 runs were spent on exactly that, and blamed on the prompt shape and then on the design path before the binary's +// age was checked. Separate from the print so a test can match the text. +inline const char* ops_dump_banner() { + return "[ops-dump] NPU_INFER_DUMP_OPS is on: an [op] line follows every projection and weighted RMSNorm, on stderr\n"; +} + +// Prints the banner once per process (a serve can load more than one forward), and only when the dump is on. Returns +// whether this call printed it. +inline bool ops_dump_announce() { + static std::atomic said{false}; + if (!ops_dump_enabled() || said.exchange(true)) return false; + std::fputs(ops_dump_banner(), stderr); + return true; +} + } // namespace onebit::npu diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 18deb4d7b..b638ad8dd 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -436,6 +436,9 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, // every row measured so far used. host_gemm_i8_ = (hg[0] == 'i' && hg[1] == '8'); } + // engine#2028: say at load, before any request, that NPU_INFER_DUMP_OPS is armed (once per process). No line with + // the variable set means the binary predates the dump. + onebit::npu::ops_dump_announce(); // The launch layer. A1: everything below that touches a device talks to this, and the settings the // container declares (weight layout, pb16i image dir, Hadamard fold) are pushed into it as they are // parsed. A2 turns this into NpuTarget::open(plan, kernels_dir). diff --git a/tests/ops_dump_test.cpp b/tests/ops_dump_test.cpp index 71496d7d4..758baa2a4 100644 --- a/tests/ops_dump_test.cpp +++ b/tests/ops_dump_test.cpp @@ -18,6 +18,7 @@ // // ops_dump_test the checks below // ops_dump_test --enabled prints 1 or 0: what ops_dump_enabled() says in a process started with this environment +// ops_dump_test --announce calls ops_dump_announce() twice and prints the two results (engine#2028's load line) #include "ops-dump.h" #include @@ -46,12 +47,31 @@ static int child_says(const char* self, const char* env_prefix) { return c == '1' ? 1 : c == '0' ? 0 : -1; } +// Everything a fresh process writes, stderr included, for one environment and one mode. +static std::string child_output(const char* self, const char* env_prefix, const char* mode) { + const std::string cmd = std::string(env_prefix) + " '" + self + "' " + mode + " 2>&1"; + std::string got; + if (FILE* p = popen(cmd.c_str(), "r")) { + char line[256]; + while (std::fgets(line, sizeof line, p)) got += line; + pclose(p); + } + return got; +} + int main(int argc, char** argv) { using namespace onebit::npu; if (argc > 1 && !std::strcmp(argv[1], "--enabled")) { std::printf("%d\n", ops_dump_enabled() ? 1 : 0); return 0; } + if (argc > 1 && !std::strcmp(argv[1], "--announce")) { + const bool first = ops_dump_announce(); + const bool second = ops_dump_announce(); + std::fflush(stderr); + std::printf("%d%d\n", first ? 1 : 0, second ? 1 : 0); + return 0; + } // The rule. check(!ops_dump_value_on(nullptr), "unset is off"); @@ -66,6 +86,16 @@ int main(int argc, char** argv) { check(child_says(argv[0], "env NPU_INFER_DUMP_OPS=0") == 0, "process: 0 is off"); check(child_says(argv[0], "env NPU_INFER_DUMP_OPS=1") == 1, "process: 1 is on"); + // The load line (engine#2028): printed once when the dump is on, never when it is off. Its absence with the variable + // set is how a run tells that its binary predates the dump. + check(child_output(argv[0], "env NPU_INFER_DUMP_OPS=1", "--announce") == std::string(ops_dump_banner()) + "10\n", + "on: the banner once, and the second call is silent; got: " + + child_output(argv[0], "env NPU_INFER_DUMP_OPS=1", "--announce")); + check(child_output(argv[0], "env -u NPU_INFER_DUMP_OPS", "--announce") == "00\n", "unset: no banner"); + check(child_output(argv[0], "env NPU_INFER_DUMP_OPS=0", "--announce") == "00\n", "0: no banner"); + check(std::string(ops_dump_banner()).find("NPU_INFER_DUMP_OPS") != std::string::npos, + "the banner names the variable, so a grep for it finds the line"); + // Read once: whatever this process decided first, a later change of the variable does not move it. const bool first = ops_dump_enabled(); setenv("NPU_INFER_DUMP_OPS", first ? "0" : "1", 1);