From 678917b64358951ae76056e2e8f28ced468a428a Mon Sep 17 00:00:00 2001 From: pi-inferzo Date: Tue, 10 Nov 2026 14:13:14 -0400 Subject: [PATCH] serve: refuse --npu-opt on a model-generic container, where the per-op forward cannot use it (refs #1740, #1868) engine#1868 refuses --npu-opt when the build links no private route. With an add-on linked there IS one, so the option was accepted -- and then bypassed by the CONTAINER TYPE: a model-generic Q4NX container is routed to the per-op forward before the private route is consulted, so the option was ignored again, one level in. Found by running W6: 'serve -m --npu-opt dx=' asked for the fused route, got per-op, and produced *identical* numbers to the per-op baseline (12,010 vs 12,169 ms for 24 tokens, same reply) -- which reads as 'no gain' rather than 'wrong route'. The engine's own log line says which forward it chose, but nothing said the option had been dropped. Now the per-op branch refuses by name when the caller asked for a private-route option: --npu-opt dx was given, but is a model-generic container, which serves through the per-op forward: the option has no effect here (it applies to a private route's own model directory). Drop it, or serve the model directory that route expects. Verified both directions on the real container with an add-on-enabled build: with the option -> exit 2 and that message; without it -> the per-op forward still starts and prints 'serving it through the per-op forward' as before. --- app/serve.cpp | 15 +++++++++++++++ 1 file changed, 15 insertions(+) diff --git a/app/serve.cpp b/app/serve.cpp index 2ea0d3da3..146ef794e 100644 --- a/app/serve.cpp +++ b/app/serve.cpp @@ -3417,6 +3417,21 @@ int run_serve(int argc, char** argv) { fs::exists(fs::path(o.model) / "config.json") && fs::exists(fs::path(o.model) / "tokenizer.json"); if (model_generic && !kernels.empty() && fs::is_directory(kernels)) { + // engine#1740 (W6): --npu-opt is for a PRIVATE route, and this branch is the per-op forward, which + // cannot consume one. #1868 made the option refuse when no add-on is linked; with an add-on linked the + // option was still accepted here and then bypassed by the CONTAINER TYPE, silently -- a W6 run asked for + // the fused route, got per-op, and produced *identical* numbers, which reads as "no gain" rather than + // "wrong route". Refusing is the only answer that cannot be misread. + if (!o.npu_opts.empty()) { + std::string keys; + for (const auto& kv : o.npu_opts) keys += (keys.empty() ? "" : ", ") + kv.substr(0, kv.find('=')); + std::fprintf(stderr, + "1bit serve: --npu-opt %s was given, but %s is a model-generic container, which serves " + "through the per-op forward: the option has no effect here (it applies to a private " + "route's own model directory). Drop it, or serve the model directory that route " + "expects.\n", keys.c_str(), o.model.c_str()); + return 2; + } std::fprintf(stderr, "1bit serve: %s is a model-generic container: serving it through the per-op forward " "(kernels %s)\n", o.model.c_str(), kernels.c_str());