From 4b58636776db21f248c821261be849b744b88501 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Fri, 13 Nov 2026 08:56:35 -0400 Subject: [PATCH 1/4] tools/ci/medians.py: report a median WITH its spread, and call overlapping arms noise Four identical decode runs on an idle box at 18,598 tokens gave 7.94 / 11.78 / 10.39 / 12.57 tok/s -- a 1.58x spread from the same command. Decode figures in this fleet get quoted to two or three decimals as if that spread did not exist, and engine#679 item 3a's "2.0x slower decode" turned out to be exactly it: 10.21 against 22.62 ms/token in the first rep of each arm, 6.17 against 6.63 once the clocks were up. So the tool reports the spread and answers the only question a reader has: medians.py 7.94 11.78 10.39 12.57 arm: median 11.085 min 7.940 max 12.570 spread 1.58x (n=4) medians.py --a 6.11 6.17 --b 6.63 6.41 verdict: B medians.py --a 10.21 6.11 6.17 --b 22.62 6.63 6.41 verdict: NOISE -- the ranges overlap, take more samples Those last two are the whole point: the same two arms are separable on the warm reps and inseparable once the cold one is included. A NOISE verdict is not a failed measurement, it is the measurement saying the difference needs more samples before anyone quotes it. The rule is disjoint RANGES, not disjoint medians: if the loser's maximum reaches into the winner's range then the loser produced that value on its own. The first version compared medians and passed [5.0] against [5.0, 6.0] as "B", when B had produced 5.0 itself. The self-test case "a shared endpoint is not evidence" is the arm that caught it, which is why this file has a self-test rather than a docstring of examples. python3 tools/ci/medians.py --self-test -> self-test: PASS --- tools/ci/medians.py | 126 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 126 insertions(+) create mode 100755 tools/ci/medians.py diff --git a/tools/ci/medians.py b/tools/ci/medians.py new file mode 100755 index 000000000..34187a7ad --- /dev/null +++ b/tools/ci/medians.py @@ -0,0 +1,126 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +"""medians.py -- report a median AND the spread, and refuse to call a difference real when it is not. + +Engine#679 measured four identical decode runs on an idle box at 18,598 tokens and got +7.94 / 11.78 / 10.39 / 12.57 tok/s: a 1.58x spread on the SAME command. Every figure this lane +published before that was quoted to two or three decimals as if the spread did not exist, and +item 3a's "2.0x slower decode" turned out to be that spread showing up in the first rep. + +So: report median, min, max and the ratio, and answer the only question a reader actually has -- +is arm A's median outside arm B's observed range, or is the difference inside the noise? + + medians.py 7.94 11.78 10.39 12.57 # one arm + medians.py --a 7.94 11.78 10.39 12.57 --b 9.1 9.3 # two arms, verdict + medians.py --self-test + +A verdict of NOISE is not a failure of the measurement; it is the measurement telling you to +take more samples or to stop quoting the difference. +""" +import statistics +import sys + + +def spread(values): + """(median, min, max, max/min) for a non-empty list of positive floats.""" + if not values: + raise ValueError("no values") + lo, hi = min(values), max(values) + if lo <= 0: + raise ValueError("values must be positive to express a spread as a ratio") + return statistics.median(values), lo, hi, hi / lo + + +def verdict(a, b): + """'A' / 'B' when one median is outside the other's observed range, else 'NOISE'. + + Disjoint ranges are the only evidence a handful of samples can give: if A's median sits + inside B's min..max, then B produced that number by itself and the difference needs more + runs before it means anything. + """ + _, alo, ahi, _ = spread(a) + _, blo, bhi, _ = spread(b) + # Disjoint ranges, not disjoint medians: if the loser's MAX reaches into the winner's + # range, the loser produced that value on its own and the difference is not yet evidence. + # (The first version compared medians and passed [5.0] against [5.0, 6.0] as "B" -- B had + # produced 5.0 itself. The self-test below is the arm that caught it.) + if alo > bhi: + return "A" + if blo > ahi: + return "B" + return "NOISE" + + +def self_test(): + fails = 0 + + def check(ok, what): + nonlocal fails + if not ok: + fails += 1 + print(f"FAIL {what}") + + # the real data that produced this file: four identical runs, 1.58x apart + med, lo, hi, ratio = spread([7.94, 11.78, 10.39, 12.57]) + check(abs(med - (10.39 + 11.78) / 2) < 1e-9, "median of four is the mean of the middle two") + check(lo == 7.94 and hi == 12.57, "min and max are the observed ones") + check(abs(ratio - 1.583) < 0.01, "the 1.58x spread of identical runs is reported as such") + + # two arms whose ranges overlap cannot be separated by these samples + check(verdict([10.0, 11.0, 12.0], [10.5, 11.5]) == "NOISE", "overlapping ranges are noise") + # an arm whose whole range sits above the other's is a real difference + check(verdict([20.0, 21.0], [10.0, 11.0]) == "A", "disjoint above is A") + check(verdict([10.0, 11.0], [20.0, 21.0]) == "B", "disjoint below is B") + # touching ranges are not disjoint: B's max equals A's min means B can reach it + check(verdict([20.0, 20.0], [19.0, 20.0]) == "NOISE", "a shared endpoint is not evidence") + + # a single sample has no range, so it can never be separated from anything + check(verdict([5.0], [5.0, 6.0]) == "NOISE", "one sample cannot beat two") + + check(ratio != 1.0, "sanity: the real data is not a flat line") + + # spread() refuses what it cannot express + for bad in ([], [0, 1]): + try: + spread(bad) + except ValueError: + pass + else: + fails += 1 + print(f"FAIL spread({bad}) should raise") + + print(f"self-test: {'PASS' if fails == 0 else str(fails) + ' FAILED'}") + return 0 if fails == 0 else 1 + + +def main(argv): + if "--self-test" in argv: + return self_test() + if "--a" in argv and "--b" in argv: + i, j = argv.index("--a"), argv.index("--b") + a = [float(x) for x in argv[i + 1 : j]] + b = [float(x) for x in argv[j + 1 :]] + else: + a, b = [float(x) for x in argv], None + if not a: + print(__doc__.strip().splitlines()[0], file=sys.stderr) + return 2 + med, lo, hi, ratio = spread(a) + label = "A" if b is not None else "arm" + print(f"{label}: median {med:.3f} min {lo:.3f} max {hi:.3f} spread {ratio:.2f}x (n={len(a)})") + if b is None: + return 0 + if not b: + print("B: no samples", file=sys.stderr) + return 2 + mb, blo, bhi, bratio = spread(b) + print(f"B: median {mb:.3f} min {blo:.3f} max {bhi:.3f} spread {bratio:.2f}x (n={len(b)})") + result = verdict(a, b) + # a difference inside the observed spread is not a result + print(f"verdict: {result}" + ("" if result != "NOISE" else " -- the ranges overlap, take more samples")) + return 0 + + +if __name__ == "__main__": + sys.exit(main(sys.argv[1:])) From 12c937a2a9f2a0f7b3c22a540999349a28cb19be Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Fri, 13 Nov 2026 11:12:04 -0400 Subject: [PATCH 2/4] medians.py: the 16-line licence block copyright.py --check wants The review was right and the check said which command to run: the file carried the two-line notice (copyright + SPDX) but not the full Apache block, so `tools/copyright.py --check` failed on it. `--fix` added the block; the self-test still passes and `--check` is clean now. --- tools/ci/medians.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tools/ci/medians.py b/tools/ci/medians.py index 34187a7ad..d0a8e4cc3 100755 --- a/tools/ci/medians.py +++ b/tools/ci/medians.py @@ -1,6 +1,18 @@ #!/usr/bin/env python3 # Copyright 2026 bong-water-water-bong # SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """medians.py -- report a median AND the spread, and refuse to call a difference real when it is not. Engine#679 measured four identical decode runs on an idle box at 18,598 tokens and got From a2fc4496894b15a96e34c1b62c02537a4186792a Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Fri, 13 Nov 2026 17:45:26 -0400 Subject: [PATCH 3/4] medians.py: --help, a ctest row, the rule the code actually implements, and the direction All five points from the review at 9fe6f04. The first two were CI failures, the last three were the file disagreeing with itself. 1. **`--help` is data no longer.** `main()` fed `--help` to `float()` and raised `ValueError: could not convert string to float`. That is the exact class engine#2339 was about -- a helper treating `--help` as a path -- and it was in the tool I wrote to make measurements honest. `-h`/`--help` now prints the module docstring and returns 0. 2. **A ctest row names the self-test.** `tools/ci/tests_referenced_check.py` requires every tool self-test to be named by a runner, and this one was named by nothing: the arm existed and nothing ran it, which is the same defect as the arms it is about. 3. **The docstring said "outside arm B's observed range"; the code has compared RANGES since the self-test caught the median version passing [5.0] against [5.0, 6.0].** The rule is now stated the way it is implemented, with the reason: if the loser's maximum reaches into the winner's range, the loser produced that value on its own. 4. **The "one sample cannot beat two" arm was labelled for a rule the code no longer had.** A lone sample is not unbeatable -- it loses to an arm whose whole range is above it, and it is only indistinguishable when the other range contains it. Both directions are now arms. 5. **The verdict says which way is better.** "A" and "B" alone mean *higher*, which is better for tok/s and worse for ms/token, and both units appear in this row's own numbers. `--lower` flips the wording, and the default spells out "higher, and higher is better here". python3 tools/ci/medians.py --a 6.11 6.17 --b 6.63 6.41 --lower verdict: B (lower, and lower is better here) --- CMakeLists.txt | 6 +++++- tools/ci/medians.py | 36 +++++++++++++++++++++++++++++++----- 2 files changed, 36 insertions(+), 6 deletions(-) diff --git a/CMakeLists.txt b/CMakeLists.txt index 98d9feffa..3dc1150ae 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -261,7 +261,11 @@ add_test(NAME doc_section_loss_selftest COMMAND python3 ${CMAKE_SOURCE_DIR}/tool # the class in CI. Files whose `#if` conditions cannot be resolved are named as not checked, never guessed at. add_test(NAME lane_syntax_selftest COMMAND python3 ${CMAKE_SOURCE_DIR}/tools/ci/lane_syntax_check.py --self-test WORKING_DIRECTORY ${CMAKE_SOURCE_DIR}) -# The review queue every reviewer hand-rolls with `gh api`, which defaults to per_page=30 -- a PR with 30+ reviews + +# engine#679/tools/ci/medians.py: the median+spread verdict tool. Its self-test carries the arms that make +# the rule checkable -- disjoint ranges (not disjoint medians), the shared-endpoint case, and the direction. +add_test(NAME medians_selftest COMMAND python3 ${CMAKE_SOURCE_DIR}/tools/ci/medians.py --self-test + WORKING_DIRECTORY ${CMAKE_SOURCE_DIR})# The review queue every reviewer hand-rolls with `gh api`, which defaults to per_page=30 -- a PR with 30+ reviews # loses its newest ones, so a verdict that exists reads as missing and a reviewer repeats work. Only the SELF-TEST is a # row: the live mode needs `gh` and the network, which a ctest row must not. add_test(NAME pr_review_queue_selftest COMMAND python3 ${CMAKE_SOURCE_DIR}/tools/ci/pr_review_queue.py --self-test diff --git a/tools/ci/medians.py b/tools/ci/medians.py index d0a8e4cc3..3130b2a66 100755 --- a/tools/ci/medians.py +++ b/tools/ci/medians.py @@ -21,10 +21,13 @@ item 3a's "2.0x slower decode" turned out to be that spread showing up in the first rep. So: report median, min, max and the ratio, and answer the only question a reader actually has -- -is arm A's median outside arm B's observed range, or is the difference inside the noise? +are the two arms' observed ranges DISJOINT, or is the difference inside the noise? Disjoint ranges, +not disjoint medians: if the loser's maximum reaches into the winner's range then the loser produced +that value on its own. medians.py 7.94 11.78 10.39 12.57 # one arm medians.py --a 7.94 11.78 10.39 12.57 --b 9.1 9.3 # two arms, verdict + medians.py --a ... --b ... --lower # B is better when lower (ms/token) medians.py --self-test A verdict of NOISE is not a failure of the measurement; it is the measurement telling you to @@ -64,6 +67,16 @@ def verdict(a, b): return "NOISE" +def direction_word(winner, lower_is_better): + """Which arm is better, spelled out. "A" alone is ambiguous: higher is better for tok/s and worse + for ms/token, and a caller that reads the letter without the unit draws the wrong conclusion.""" + if winner == "NOISE": + return "neither -- the ranges overlap" + if lower_is_better: + return f"{winner} (lower, and lower is better here)" + return f"{winner} (higher, and higher is better here)" + + def self_test(): fails = 0 @@ -87,11 +100,17 @@ def check(ok, what): # touching ranges are not disjoint: B's max equals A's min means B can reach it check(verdict([20.0, 20.0], [19.0, 20.0]) == "NOISE", "a shared endpoint is not evidence") - # a single sample has no range, so it can never be separated from anything - check(verdict([5.0], [5.0, 6.0]) == "NOISE", "one sample cannot beat two") + # a single sample has no range, so an arm whose range contains it cannot be distinguished from it + check(verdict([5.0], [5.0, 6.0]) == "NOISE", "an arm whose range contains the single sample is not evidence") + check(verdict([5.0], [6.0, 7.0]) == "B", "a single sample below a disjoint arm loses to it") check(ratio != 1.0, "sanity: the real data is not a flat line") + # direction: the same verdict reads differently for tok/s and for ms/token + check("higher" in direction_word("A", False), "default direction says higher") + check("lower" in direction_word("B", True), "--lower says lower is better") + check("neither" in direction_word("NOISE", True), "noise names no winner") + # spread() refuses what it cannot express for bad in ([], [0, 1]): try: @@ -107,8 +126,13 @@ def check(ok, what): def main(argv): + if "--help" in argv or "-h" in argv: + print(__doc__.strip()) + return 0 if "--self-test" in argv: return self_test() + lower_is_better = "--lower" in argv + argv = [a for a in argv if a != "--lower"] if "--a" in argv and "--b" in argv: i, j = argv.index("--a"), argv.index("--b") a = [float(x) for x in argv[i + 1 : j]] @@ -129,8 +153,10 @@ def main(argv): mb, blo, bhi, bratio = spread(b) print(f"B: median {mb:.3f} min {blo:.3f} max {bhi:.3f} spread {bratio:.2f}x (n={len(b)})") result = verdict(a, b) - # a difference inside the observed spread is not a result - print(f"verdict: {result}" + ("" if result != "NOISE" else " -- the ranges overlap, take more samples")) + # a difference inside the observed spread is not a result, and the winner needs its direction + print(f"verdict: {direction_word(result, lower_is_better)}" + + ("" if result != "NOISE" else f" (A={a}, B={b} medians)") if result != "NOISE" + else "verdict: NOISE -- the ranges overlap, take more samples") return 0 From 3d2453c3ed70cd757b03a171ae28de957979fd2e Mon Sep 17 00:00:00 2001 From: bcloud Date: Fri, 13 Nov 2026 18:23:36 -0400 Subject: [PATCH 4/4] medians.py: a one-sample arm cannot win, and --a alone parses (engine#2369 items 1 and 2) Item 1: verdict() compared disjoint ranges, so [5.0] against [6.0] returned B -- one run per arm reading as evidence of a winner. A single sample has no spread, so it can never be shown to lie outside the other arm's range. It may still LOSE: a point below the other arm's min is below every one of its observations, which needs no spread to see. The guard is therefore 'cannot win', not 'no verdict', which is what the existing self-test arm `a single sample below a disjoint arm loses to it` already required -- my first attempt broke that arm and the suite caught it, which is the tool doing its job. Item 2: --a without --b fell through to 'parse every argv token as a float', so the flag itself was blamed (ValueError: float: '--a'). --a is now parsed as a range ending at --b or end-of-args, with a per-flag error naming the offending token. Two self-test arms added for the cases the PR body reported. --self-test PASS; --a 6.11 6.17 --b 6.63 6.41 still B; 10.21 6.11 6.17 vs 22.62 6.63 6.41 still NOISE. --- tools/ci/medians.py | 31 ++++++++++++++++++++++++------- 1 file changed, 24 insertions(+), 7 deletions(-) diff --git a/tools/ci/medians.py b/tools/ci/medians.py index 3130b2a66..1230f57b3 100755 --- a/tools/ci/medians.py +++ b/tools/ci/medians.py @@ -54,15 +54,19 @@ def verdict(a, b): inside B's min..max, then B produced that number by itself and the difference needs more runs before it means anything. """ + # A single sample has no range, so it can never be shown to be OUTSIDE the other arm's range: + # [5.0] against [6.0] would read as a win on one run. It may still LOSE -- a point below the + # other arm's min is below every one of its observations, which needs no spread to see. + # (engine#2369 item 1: the previous version let a one-sample arm win.) _, alo, ahi, _ = spread(a) _, blo, bhi, _ = spread(b) # Disjoint ranges, not disjoint medians: if the loser's MAX reaches into the winner's # range, the loser produced that value on its own and the difference is not yet evidence. # (The first version compared medians and passed [5.0] against [5.0, 6.0] as "B" -- B had # produced 5.0 itself. The self-test below is the arm that caught it.) - if alo > bhi: + if alo > bhi and len(a) >= 2: return "A" - if blo > ahi: + if blo > ahi and len(b) >= 2: return "B" return "NOISE" @@ -101,6 +105,8 @@ def check(ok, what): check(verdict([20.0, 20.0], [19.0, 20.0]) == "NOISE", "a shared endpoint is not evidence") # a single sample has no range, so an arm whose range contains it cannot be distinguished from it + check(verdict([5.0], [6.0]) == "NOISE", "two single samples are not evidence (no spread on either arm)") + check(verdict([5.0, 5.1], [6.0]) == "NOISE", "one sample on either arm cannot win") check(verdict([5.0], [5.0, 6.0]) == "NOISE", "an arm whose range contains the single sample is not evidence") check(verdict([5.0], [6.0, 7.0]) == "B", "a single sample below a disjoint arm loses to it") @@ -133,12 +139,23 @@ def main(argv): return self_test() lower_is_better = "--lower" in argv argv = [a for a in argv if a != "--lower"] - if "--a" in argv and "--b" in argv: - i, j = argv.index("--a"), argv.index("--b") - a = [float(x) for x in argv[i + 1 : j]] - b = [float(x) for x in argv[j + 1 :]] + def samples(tokens, flag): + out = [] + for tok in tokens: + try: + out.append(float(tok)) + except ValueError: + print(f"medians.py: {flag} expects numbers, got {tok!r}", file=sys.stderr) + raise SystemExit(2) + return out + + if "--a" in argv: + i = argv.index("--a") + j = argv.index("--b") if "--b" in argv else len(argv) + a = samples(argv[i + 1 : j], "--a") + b = samples(argv[j + 1 :], "--b") if "--b" in argv else None else: - a, b = [float(x) for x in argv], None + a, b = samples(argv, "args"), None if not a: print(__doc__.strip().splitlines()[0], file=sys.stderr) return 2