diff --git a/docs/npu.md b/docs/npu.md index 2448c851c..0ee142606 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -2186,7 +2186,10 @@ can fail a run. | `NPU_INFER_DUMP_DIR=` | `/layer_.f32` per layer: the residual stream after that layer, M x H little-endian f32, row-major. The directory must exist; a file that cannot be written is reported on stderr and the run goes on. | which **layer** two runs part at (per-layer cosine against a reference trace) | | `NPU_INFER_DUMP_OPS=1` | one stderr line per projection and per weighted RMSNorm: `[op] L rows= \|x\| mean absmax ` | which **op** inside the layer | -`NPU_INFER_DUMP_OPS` is on for any value other than empty or `0`, and is read once per process. +`NPU_INFER_DUMP_OPS` is on for any value other than empty or `0`, and is read once per process. When it is on, the forward prints +`[ops-dump] NPU_INFER_DUMP_OPS is on: …` once at model load, before any request (engine#2028). A run that sets the +variable and never sees that line is running a binary built before the dump existed (#1977): rebuild it, rather than +looking for a code path that skips the dump. ``` [op] L00 rmsnorm rows=1 |x| mean 0.0143171 absmax 3.01935 diff --git a/npu/forward/include/ops-dump.h b/npu/forward/include/ops-dump.h index 42b3d0fd6..9fb15babc 100644 --- a/npu/forward/include/ops-dump.h +++ b/npu/forward/include/ops-dump.h @@ -36,6 +36,7 @@ // getenv). It can never fail a run: no allocation, no file, stderr only. #pragma once +#include #include #include #include @@ -109,4 +110,21 @@ inline void ops_dump(const char* tag, int layer, int rows, const float* x, int n if (ops_dump_line(buf, sizeof buf, tag, layer, rows, x, n) > 0) std::fputs(buf, stderr); } +// engine#2028: the line a process prints once, at model load, when the dump is armed. A run that sets the variable and +// does not see this line before its first request is running a binary from before the instrument (#1977): four silent +// phi-4 runs were spent on exactly that, and blamed on the prompt shape and then on the design path before the binary's +// age was checked. Separate from the print so a test can match the text. +inline const char* ops_dump_banner() { + return "[ops-dump] NPU_INFER_DUMP_OPS is on: an [op] line follows every projection and weighted RMSNorm, on stderr\n"; +} + +// Prints the banner once per process (a serve can load more than one forward), and only when the dump is on. Returns +// whether this call printed it. +inline bool ops_dump_announce() { + static std::atomic said{false}; + if (!ops_dump_enabled() || said.exchange(true)) return false; + std::fputs(ops_dump_banner(), stderr); + return true; +} + } // namespace onebit::npu diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp index 18deb4d7b..b638ad8dd 100644 --- a/npu/forward/src/q4nx_forward.cpp +++ b/npu/forward/src/q4nx_forward.cpp @@ -436,6 +436,9 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path, // every row measured so far used. host_gemm_i8_ = (hg[0] == 'i' && hg[1] == '8'); } + // engine#2028: say at load, before any request, that NPU_INFER_DUMP_OPS is armed (once per process). No line with + // the variable set means the binary predates the dump. + onebit::npu::ops_dump_announce(); // The launch layer. A1: everything below that touches a device talks to this, and the settings the // container declares (weight layout, pb16i image dir, Hadamard fold) are pushed into it as they are // parsed. A2 turns this into NpuTarget::open(plan, kernels_dir). diff --git a/tests/ops_dump_test.cpp b/tests/ops_dump_test.cpp index 71496d7d4..758baa2a4 100644 --- a/tests/ops_dump_test.cpp +++ b/tests/ops_dump_test.cpp @@ -18,6 +18,7 @@ // // ops_dump_test the checks below // ops_dump_test --enabled prints 1 or 0: what ops_dump_enabled() says in a process started with this environment +// ops_dump_test --announce calls ops_dump_announce() twice and prints the two results (engine#2028's load line) #include "ops-dump.h" #include @@ -46,12 +47,31 @@ static int child_says(const char* self, const char* env_prefix) { return c == '1' ? 1 : c == '0' ? 0 : -1; } +// Everything a fresh process writes, stderr included, for one environment and one mode. +static std::string child_output(const char* self, const char* env_prefix, const char* mode) { + const std::string cmd = std::string(env_prefix) + " '" + self + "' " + mode + " 2>&1"; + std::string got; + if (FILE* p = popen(cmd.c_str(), "r")) { + char line[256]; + while (std::fgets(line, sizeof line, p)) got += line; + pclose(p); + } + return got; +} + int main(int argc, char** argv) { using namespace onebit::npu; if (argc > 1 && !std::strcmp(argv[1], "--enabled")) { std::printf("%d\n", ops_dump_enabled() ? 1 : 0); return 0; } + if (argc > 1 && !std::strcmp(argv[1], "--announce")) { + const bool first = ops_dump_announce(); + const bool second = ops_dump_announce(); + std::fflush(stderr); + std::printf("%d%d\n", first ? 1 : 0, second ? 1 : 0); + return 0; + } // The rule. check(!ops_dump_value_on(nullptr), "unset is off"); @@ -66,6 +86,16 @@ int main(int argc, char** argv) { check(child_says(argv[0], "env NPU_INFER_DUMP_OPS=0") == 0, "process: 0 is off"); check(child_says(argv[0], "env NPU_INFER_DUMP_OPS=1") == 1, "process: 1 is on"); + // The load line (engine#2028): printed once when the dump is on, never when it is off. Its absence with the variable + // set is how a run tells that its binary predates the dump. + check(child_output(argv[0], "env NPU_INFER_DUMP_OPS=1", "--announce") == std::string(ops_dump_banner()) + "10\n", + "on: the banner once, and the second call is silent; got: " + + child_output(argv[0], "env NPU_INFER_DUMP_OPS=1", "--announce")); + check(child_output(argv[0], "env -u NPU_INFER_DUMP_OPS", "--announce") == "00\n", "unset: no banner"); + check(child_output(argv[0], "env NPU_INFER_DUMP_OPS=0", "--announce") == "00\n", "0: no banner"); + check(std::string(ops_dump_banner()).find("NPU_INFER_DUMP_OPS") != std::string::npos, + "the banner names the variable, so a grep for it finds the line"); + // Read once: whatever this process decided first, a later change of the variable does not move it. const bool first = ops_dump_enabled(); setenv("NPU_INFER_DUMP_OPS", first ? "0" : "1", 1);