From f9ea34116fd2f9c68d8e848cf597dbb7c187c4f4 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sun, 15 Nov 2026 00:14:39 -0400 Subject: [PATCH] token_rate_floor: --floor 0 is a SKIP, not a pass (#2432 merged before this fix) --floor 0 means 'record, do not judge', and it returned 0 -- so ctest reports PASS for a run that asserted nothing, and the device row's own default (TOKEN_RATE_FLOOR=0) makes that the out-of-the-box behaviour. That is the 'a verdict that answers fine' defect this file exists to catch, in its own default, and it is live on main. #2432 merged at 03:03:44Z; this fix was written at 03:2xZ against its branch and so never landed. Applying it to main directly rather than reopening: the merge was clean, the defect is not. floor <= 0 now returns 77 (CTest's skip, wired via the device row's SKIP_RETURN_CODE) and prints the recorded rate: recorded: 31.3 tokens/s (floor 0) -- recorded 31.3 tokens/s and asserted nothing (floor 0); pass --floor N to judge Sixth arm pins it: 'floor 0 records and SKIPs -- a record-only run is not a pass'. 6 arms, 0 failed. --- tools/ci/token_rate_floor.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/tools/ci/token_rate_floor.py b/tools/ci/token_rate_floor.py index 84fcf3fae..6eba6fab5 100755 --- a/tools/ci/token_rate_floor.py +++ b/tools/ci/token_rate_floor.py @@ -64,7 +64,13 @@ def verdict(rates, floor, subject=""): last = rates[-1] # A floor of 0 means "record, do not judge" -- useful while calibrating on a new box. if floor <= 0: - return 0, {"status": "recorded", "rate": last, "floor": floor, "subject": subject} + # A record-only run asserts NOTHING, so exit 0 would have ctest report PASS for a check that checked + # nothing -- the "a verdict that answers fine" defect this file exists to catch, in its own default + # (the device row's TOKEN_RATE_FLOOR defaults to 0). 77 is CTest's skip; the rate still lands in the + # message so a lane running it by hand gets the number and knows it judged nothing. + return SKIP, {"status": "recorded", "rate": last, "floor": floor, "subject": subject, + "why": f"recorded {last:g} tokens/s and asserted nothing (floor {floor:g}); " + f"pass --floor N to judge"} if last < floor: return 1, {"status": "below_floor", "rate": last, "floor": floor, "subject": subject, "why": f"generated {last:g} tokens/s, floor is {floor:g} -- correct output, " @@ -87,6 +93,9 @@ def self_test(): # engine#2426's lesson applied to rates: a floor without its subject is a recollection, not evidence. ("the verdict carries its subject, so a pasted failure is checkable", "eval: 8.4 tokens per second\n", 20.0, 1), + # engine#2432 review: --floor 0 is "record only", so it must not read as a pass (77 = ctest SKIP). + ("floor 0 records and SKIPs -- a record-only run is not a pass", + "eval: 31.3 tokens per second\n", 0.0, SKIP), ] bad = 0 SUBJECT = "selftest q4_k_m ctx2048"