From 8faab9831cb9ed1dd2de7e10f5b14f00dc31e57a Mon Sep 17 00:00:00 2001
From: bong-water-water-bong
<277547417+bong-water-water-bong@users.noreply.github.com>
Date: Wed, 11 Nov 2026 07:41:01 -0400
Subject: [PATCH] npu_serve_smoke --judge: a directory with no replies says so
(rc 2), instead of three FAILs
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit
The 2026-11-11 10:54Z demo smoke was logged `FAIL` in DEMO-SMOKE.md:
| 2026-11-11T10:54Z | fast | The capital of France is Paris. | 100.5 | ERROR | 0 | **FAIL** |
Triaged from the run's own artefacts, the demo was healthy: fast route line in serve.log, 100.5 tok/s against a 60
floor, first token = timings.prompt_ms 279 ms against a 600 limit, content "The capital of France is Paris."
What failed was the reading. The ad-hoc harness wrote its reply as `npu.json`; `--judge` looks for `reply_0.json`, so
it found no replies and reported
rate FAIL 0 of 0 replies carry no decode timing
first-token FAIL a reply carries no prompt_ms
content FAIL no replies
three lines that read as a broken demo. Renaming the same bytes to reply_0.json gives PASS.
"0 of 0 replies" is not a verdict about a run, it is the wrong directory - so rejudge() now refuses it by name:
serve cannot judge: no replies in
looked for serve.log, reply_0.json …, load.json; the .json files here are: npu.json
if this is another harness's output, name its reply as reply_0.json and re-run --judge
rc 2, no verdict lines. judge() itself still FAILs on an empty reply LIST, which is the different case (a run that
produced no reply); the distinction is the same one #2039 drew between a skip and a failure.
Verified on the real artefacts: the unreadable directory rc 2 with the message above, the renamed one PASS rc 0, and the
self-test gains the arm for a directory holding a `npu.json` but no `reply_*.json` (16 judging cases, 3 stand-in runs,
rc 0).
---
tools/npu_serve_smoke.py | 24 ++++++++++++++++++++++++
1 file changed, 24 insertions(+)
diff --git a/tools/npu_serve_smoke.py b/tools/npu_serve_smoke.py
index 4665b4d1a..4ae057f56 100755
--- a/tools/npu_serve_smoke.py
+++ b/tools/npu_serve_smoke.py
@@ -242,6 +242,17 @@ def rejudge(args):
with open(os.path.join(d, "reply_%d.json" % i)) as f:
replies.append(json.load(f))
i += 1
+ # A directory with no replies is not a demo that failed - it is the wrong directory (or a harness that wrote
+ # its reply somewhere else), and saying so is the difference between "the run failed" and "look again". The
+ # 2026-11-11 10:54Z smoke was logged FAIL on exactly this: the ad-hoc harness wrote `npu.json`, `--judge`
+ # looked for `reply_0.json`, and three FAIL lines read as a broken demo. The same bytes, renamed, give PASS.
+ if not replies:
+ found = sorted(f for f in os.listdir(d) if f.endswith(".json")) if os.path.isdir(d) else []
+ print("serve cannot judge: no replies in %s" % d)
+ print(" looked for serve.log, reply_0.json …, load.json; the .json files here are: %s"
+ % (", ".join(found) if found else "none"))
+ print(" if this is another harness's output, name its reply as reply_0.json and re-run --judge")
+ return 2
load1, ncpu = None, 0 # the load the RUN recorded, if it did: judging again later must not read today's box
try:
with open(os.path.join(d, "load.json")) as f:
@@ -434,6 +445,19 @@ def note_is(name, verdicts, load1, want_hint, want_none=False):
if not degenerate("//////") or degenerate("391") or degenerate("aa") or not degenerate("a a a a"):
fails.append("degenerate(): wrong on a fixture")
+ # engine#2129 follow-up: `--judge` on a directory that holds no reply_*.json must say "cannot judge" (rc 2),
+ # not report three FAILs - a lane triaging a smoke FAIL was sent after a broken demo on 2026-11-11 10:54Z when
+ # the truth was that its harness had written `npu.json`. The judge() path above still FAILs on an empty list
+ # (that is a run that produced no reply); rejudge() refuses a DIRECTORY it cannot read.
+ import tempfile
+ with tempfile.TemporaryDirectory() as td:
+ open(os.path.join(td, "serve.log"), "w").write(fast_log)
+ open(os.path.join(td, "npu.json"), "w").write('{"choices": []}')
+ rc = rejudge(argparse.Namespace(judge=td, floor=60.0, first_token_ms=600.0,
+ fast_route=FAST_ROUTE, slow_route=SLOW_ROUTE))
+ if rc != 2:
+ fails.append("rejudge() on a directory with no reply_*.json returned %r, wanted 2" % rc)
+
self_test_run(fails)
for f in fails:
print("FAIL " + f)