From 8faab9831cb9ed1dd2de7e10f5b14f00dc31e57a Mon Sep 17 00:00:00 2001 From: bong-water-water-bong <277547417+bong-water-water-bong@users.noreply.github.com> Date: Wed, 11 Nov 2026 07:41:01 -0400 Subject: [PATCH] npu_serve_smoke --judge: a directory with no replies says so (rc 2), instead of three FAILs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 2026-11-11 10:54Z demo smoke was logged `FAIL` in DEMO-SMOKE.md: | 2026-11-11T10:54Z | fast | The capital of France is Paris. | 100.5 | ERROR | 0 | **FAIL** | Triaged from the run's own artefacts, the demo was healthy: fast route line in serve.log, 100.5 tok/s against a 60 floor, first token = timings.prompt_ms 279 ms against a 600 limit, content "The capital of France is Paris." What failed was the reading. The ad-hoc harness wrote its reply as `npu.json`; `--judge` looks for `reply_0.json`, so it found no replies and reported rate FAIL 0 of 0 replies carry no decode timing first-token FAIL a reply carries no prompt_ms content FAIL no replies three lines that read as a broken demo. Renaming the same bytes to reply_0.json gives PASS. "0 of 0 replies" is not a verdict about a run, it is the wrong directory - so rejudge() now refuses it by name: serve cannot judge: no replies in looked for serve.log, reply_0.json …, load.json; the .json files here are: npu.json if this is another harness's output, name its reply as reply_0.json and re-run --judge rc 2, no verdict lines. judge() itself still FAILs on an empty reply LIST, which is the different case (a run that produced no reply); the distinction is the same one #2039 drew between a skip and a failure. Verified on the real artefacts: the unreadable directory rc 2 with the message above, the renamed one PASS rc 0, and the self-test gains the arm for a directory holding a `npu.json` but no `reply_*.json` (16 judging cases, 3 stand-in runs, rc 0). --- tools/npu_serve_smoke.py | 24 ++++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/tools/npu_serve_smoke.py b/tools/npu_serve_smoke.py index 4665b4d1a..4ae057f56 100755 --- a/tools/npu_serve_smoke.py +++ b/tools/npu_serve_smoke.py @@ -242,6 +242,17 @@ def rejudge(args): with open(os.path.join(d, "reply_%d.json" % i)) as f: replies.append(json.load(f)) i += 1 + # A directory with no replies is not a demo that failed - it is the wrong directory (or a harness that wrote + # its reply somewhere else), and saying so is the difference between "the run failed" and "look again". The + # 2026-11-11 10:54Z smoke was logged FAIL on exactly this: the ad-hoc harness wrote `npu.json`, `--judge` + # looked for `reply_0.json`, and three FAIL lines read as a broken demo. The same bytes, renamed, give PASS. + if not replies: + found = sorted(f for f in os.listdir(d) if f.endswith(".json")) if os.path.isdir(d) else [] + print("serve cannot judge: no replies in %s" % d) + print(" looked for serve.log, reply_0.json …, load.json; the .json files here are: %s" + % (", ".join(found) if found else "none")) + print(" if this is another harness's output, name its reply as reply_0.json and re-run --judge") + return 2 load1, ncpu = None, 0 # the load the RUN recorded, if it did: judging again later must not read today's box try: with open(os.path.join(d, "load.json")) as f: @@ -434,6 +445,19 @@ def note_is(name, verdicts, load1, want_hint, want_none=False): if not degenerate("//////") or degenerate("391") or degenerate("aa") or not degenerate("a a a a"): fails.append("degenerate(): wrong on a fixture") + # engine#2129 follow-up: `--judge` on a directory that holds no reply_*.json must say "cannot judge" (rc 2), + # not report three FAILs - a lane triaging a smoke FAIL was sent after a broken demo on 2026-11-11 10:54Z when + # the truth was that its harness had written `npu.json`. The judge() path above still FAILs on an empty list + # (that is a run that produced no reply); rejudge() refuses a DIRECTORY it cannot read. + import tempfile + with tempfile.TemporaryDirectory() as td: + open(os.path.join(td, "serve.log"), "w").write(fast_log) + open(os.path.join(td, "npu.json"), "w").write('{"choices": []}') + rc = rejudge(argparse.Namespace(judge=td, floor=60.0, first_token_ms=600.0, + fast_route=FAST_ROUTE, slow_route=SLOW_ROUTE)) + if rc != 2: + fails.append("rejudge() on a directory with no reply_*.json returned %r, wanted 2" % rc) + self_test_run(fails) for f in fails: print("FAIL " + f)