diff --git a/docs/npu.md b/docs/npu.md
index 2448c851c..0ee142606 100644
--- a/docs/npu.md
+++ b/docs/npu.md
@@ -2186,7 +2186,10 @@ can fail a run.
| `NPU_INFER_DUMP_DIR=
` | `/layer_.f32` per layer: the residual stream after that layer, M x H little-endian f32, row-major. The directory must exist; a file that cannot be written is reported on stderr and the run goes on. | which **layer** two runs part at (per-layer cosine against a reference trace) |
| `NPU_INFER_DUMP_OPS=1` | one stderr line per projection and per weighted RMSNorm: `[op] L rows= \|x\| mean absmax ` | which **op** inside the layer |
-`NPU_INFER_DUMP_OPS` is on for any value other than empty or `0`, and is read once per process.
+`NPU_INFER_DUMP_OPS` is on for any value other than empty or `0`, and is read once per process. When it is on, the forward prints
+`[ops-dump] NPU_INFER_DUMP_OPS is on: …` once at model load, before any request (engine#2028). A run that sets the
+variable and never sees that line is running a binary built before the dump existed (#1977): rebuild it, rather than
+looking for a code path that skips the dump.
```
[op] L00 rmsnorm rows=1 |x| mean 0.0143171 absmax 3.01935
diff --git a/npu/forward/include/ops-dump.h b/npu/forward/include/ops-dump.h
index 42b3d0fd6..9fb15babc 100644
--- a/npu/forward/include/ops-dump.h
+++ b/npu/forward/include/ops-dump.h
@@ -36,6 +36,7 @@
// getenv). It can never fail a run: no allocation, no file, stderr only.
#pragma once
+#include
#include
#include
#include
@@ -109,4 +110,21 @@ inline void ops_dump(const char* tag, int layer, int rows, const float* x, int n
if (ops_dump_line(buf, sizeof buf, tag, layer, rows, x, n) > 0) std::fputs(buf, stderr);
}
+// engine#2028: the line a process prints once, at model load, when the dump is armed. A run that sets the variable and
+// does not see this line before its first request is running a binary from before the instrument (#1977): four silent
+// phi-4 runs were spent on exactly that, and blamed on the prompt shape and then on the design path before the binary's
+// age was checked. Separate from the print so a test can match the text.
+inline const char* ops_dump_banner() {
+ return "[ops-dump] NPU_INFER_DUMP_OPS is on: an [op] line follows every projection and weighted RMSNorm, on stderr\n";
+}
+
+// Prints the banner once per process (a serve can load more than one forward), and only when the dump is on. Returns
+// whether this call printed it.
+inline bool ops_dump_announce() {
+ static std::atomic said{false};
+ if (!ops_dump_enabled() || said.exchange(true)) return false;
+ std::fputs(ops_dump_banner(), stderr);
+ return true;
+}
+
} // namespace onebit::npu
diff --git a/npu/forward/src/q4nx_forward.cpp b/npu/forward/src/q4nx_forward.cpp
index 18deb4d7b..b638ad8dd 100644
--- a/npu/forward/src/q4nx_forward.cpp
+++ b/npu/forward/src/q4nx_forward.cpp
@@ -436,6 +436,9 @@ bool Q4nxNpuForward::init_impl(xrt::device* dev, const char* model_path,
// every row measured so far used.
host_gemm_i8_ = (hg[0] == 'i' && hg[1] == '8');
}
+ // engine#2028: say at load, before any request, that NPU_INFER_DUMP_OPS is armed (once per process). No line with
+ // the variable set means the binary predates the dump.
+ onebit::npu::ops_dump_announce();
// The launch layer. A1: everything below that touches a device talks to this, and the settings the
// container declares (weight layout, pb16i image dir, Hadamard fold) are pushed into it as they are
// parsed. A2 turns this into NpuTarget::open(plan, kernels_dir).
diff --git a/tests/ops_dump_test.cpp b/tests/ops_dump_test.cpp
index 71496d7d4..758baa2a4 100644
--- a/tests/ops_dump_test.cpp
+++ b/tests/ops_dump_test.cpp
@@ -18,6 +18,7 @@
//
// ops_dump_test the checks below
// ops_dump_test --enabled prints 1 or 0: what ops_dump_enabled() says in a process started with this environment
+// ops_dump_test --announce calls ops_dump_announce() twice and prints the two results (engine#2028's load line)
#include "ops-dump.h"
#include
@@ -46,12 +47,31 @@ static int child_says(const char* self, const char* env_prefix) {
return c == '1' ? 1 : c == '0' ? 0 : -1;
}
+// Everything a fresh process writes, stderr included, for one environment and one mode.
+static std::string child_output(const char* self, const char* env_prefix, const char* mode) {
+ const std::string cmd = std::string(env_prefix) + " '" + self + "' " + mode + " 2>&1";
+ std::string got;
+ if (FILE* p = popen(cmd.c_str(), "r")) {
+ char line[256];
+ while (std::fgets(line, sizeof line, p)) got += line;
+ pclose(p);
+ }
+ return got;
+}
+
int main(int argc, char** argv) {
using namespace onebit::npu;
if (argc > 1 && !std::strcmp(argv[1], "--enabled")) {
std::printf("%d\n", ops_dump_enabled() ? 1 : 0);
return 0;
}
+ if (argc > 1 && !std::strcmp(argv[1], "--announce")) {
+ const bool first = ops_dump_announce();
+ const bool second = ops_dump_announce();
+ std::fflush(stderr);
+ std::printf("%d%d\n", first ? 1 : 0, second ? 1 : 0);
+ return 0;
+ }
// The rule.
check(!ops_dump_value_on(nullptr), "unset is off");
@@ -66,6 +86,16 @@ int main(int argc, char** argv) {
check(child_says(argv[0], "env NPU_INFER_DUMP_OPS=0") == 0, "process: 0 is off");
check(child_says(argv[0], "env NPU_INFER_DUMP_OPS=1") == 1, "process: 1 is on");
+ // The load line (engine#2028): printed once when the dump is on, never when it is off. Its absence with the variable
+ // set is how a run tells that its binary predates the dump.
+ check(child_output(argv[0], "env NPU_INFER_DUMP_OPS=1", "--announce") == std::string(ops_dump_banner()) + "10\n",
+ "on: the banner once, and the second call is silent; got: " +
+ child_output(argv[0], "env NPU_INFER_DUMP_OPS=1", "--announce"));
+ check(child_output(argv[0], "env -u NPU_INFER_DUMP_OPS", "--announce") == "00\n", "unset: no banner");
+ check(child_output(argv[0], "env NPU_INFER_DUMP_OPS=0", "--announce") == "00\n", "0: no banner");
+ check(std::string(ops_dump_banner()).find("NPU_INFER_DUMP_OPS") != std::string::npos,
+ "the banner names the variable, so a grep for it finds the line");
+
// Read once: whatever this process decided first, a later change of the variable does not move it.
const bool first = ops_dump_enabled();
setenv("NPU_INFER_DUMP_OPS", first ? "0" : "1", 1);