diff --git a/CMakeLists.txt b/CMakeLists.txt index 1985ce08f..f34fd6640 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -199,6 +199,15 @@ add_test(NAME watchdog_serve COMMAND ${CMAKE_SOURCE_DIR}/tests/watchdog_serve.sh add_test(NAME metrics_serve COMMAND ${CMAKE_SOURCE_DIR}/tests/metrics_serve.sh $) # serve --print-backend: the llama-server Lemonade starts itself, one hop (docs/serve.md): no GPU. add_test(NAME print_backend COMMAND ${CMAKE_SOURCE_DIR}/tests/print_backend.sh $) +# token_rate_floor's four arms (engine#374): the rate floor that fails in the direction a correctness check +# cannot, since a 3.7x regression still emits the right tokens. The self-test needs no device; the --log mode +# is what a device row calls with llama-cli's output. +add_test(NAME token_rate_floor_selftest COMMAND python3 ${CMAKE_SOURCE_DIR}/tools/ci/token_rate_floor.py --self-test) +# The --log mode's caller (engine#2432 review item 1: a self-test with no consumer is engine#2418's defect). It +# judges one real `1bit serve` rate inside a gpu-hold ticket, and exits 77 -- CTest's skip -- naming the missing +# prerequisite when it cannot: a device row that passes for want of a model would be the failure this tool catches. +add_test(NAME token_rate_floor_device COMMAND ${CMAKE_SOURCE_DIR}/tools/ci/token_rate_floor_device.sh ${CMAKE_SOURCE_DIR}) +set_tests_properties(token_rate_floor_device PROPERTIES SKIP_RETURN_CODE 77) # scripts/lemonade-hrx: the PPA lemond's HRX recipe runs our llama-server (docs/lemonade.md): no GPU. add_test(NAME lemonade_hrx COMMAND ${CMAKE_SOURCE_DIR}/tests/lemonade_hrx.sh $) add_test(NAME parallel_args COMMAND ${CMAKE_SOURCE_DIR}/tests/parallel_args.sh $) diff --git a/tools/ci/token_rate_floor.py b/tools/ci/token_rate_floor.py new file mode 100755 index 000000000..84fcf3fae --- /dev/null +++ b/tools/ci/token_rate_floor.py @@ -0,0 +1,144 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""token_rate_floor.py: fail when a model generates slower than a floor, so a silent slowdown is caught. + +engine#374 is why this exists. `model_buffer_from_mmap()` retagged a CPU buffer as `mapped_buft` while leaving +its context a CPU context. The result was **correct output at 8.4 t/s instead of 31.3** -- a 3.7x regression that +no test caught, because every test that watched the buffer was watching the *assertion* it satisfied, not the +speed the machine was doing it at. `hrx-buffer-test` went red much later, on an alignment assert, and the rate +had been quietly wrong for as long as the retag had. + +A rate floor is the direct measurement of the thing users feel, and it fails in the direction a correctness +check cannot: a run that is 3x too slow still produces the right tokens. + + token_rate_floor.py --log run.log --floor 20 # judge a saved llama-cli / 1bit run + token_rate_floor.py --log run.log --floor 20 --json # machine-readable verdict + token_rate_floor.py --self-test # the four arms, no device needed + +**Set the floor below the MINIMUM of at least three interleaved runs, not below a median or a single good one.** A single +rate is not a stable subject on this box: four *identical* 128-token decode runs at one config on engine#679 measured +7.94 / 11.78 / 10.39 / 12.57 t/s -- median 11.09, **spread 1.58x**. A floor judged on one sample flaps for reasons that +have nothing to do with a regression, and a flapping row gets ignored, which is how noisy checks die. Judging a range +instead (the way tools/ci/medians.py decides A/B by disjoint ranges) is more correct and more work; this file takes the +cheap honest path: one sentence here, and a caller who runs three times. + +The log is llama-cli's own output ("llama_perf_context_print: prompt eval time = ... 31.3 tokens per second"), +or a one-line `N tokens per second`; the LAST rate in the file is used, because a warmup line is not the run. +Exit: 0 met the floor, 1 below it or unparseable, 77 no rate in the log at all (a skip, not a red). +""" +import argparse +import json +import re +import sys + +SKIP = 77 + +# llama-cli: "eval time = 200.00 ms / 32 runs ( 6.25 ms per token, 160.00 tokens per second)" +# and shorter forms ("31.3 t/s", "31.3 tok/s") for one-line harnesses. +_RATE = re.compile(r"(\d+(?:\.\d+)?)\s*(?:tokens per second|tok/s|t/s)") + + +def parse_rates(text): + """Every rate in the log, in order. Multiline, because llama-cli prints prompt and eval rates separately.""" + return [float(m) for m in _RATE.findall(text or "")] + + +def verdict(rates, floor, subject=""): + """-> (exit_code, dict). The LAST rate is the generation rate; earlier ones are prompt eval.""" + if not rates: + return SKIP, {"status": "skip", "subject": subject, + "why": "no token rate in the log -- the run did not report one"} + last = rates[-1] + # A floor of 0 means "record, do not judge" -- useful while calibrating on a new box. + if floor <= 0: + return 0, {"status": "recorded", "rate": last, "floor": floor, "subject": subject} + if last < floor: + return 1, {"status": "below_floor", "rate": last, "floor": floor, "subject": subject, + "why": f"generated {last:g} tokens/s, floor is {floor:g} -- correct output, " + f"{floor / last:.1f}x below the floor"} + return 0, {"status": "ok", "rate": last, "floor": floor, "subject": subject} + + +def self_test(): + """Four arms. The second is the one this file exists for: #374's own before/after pair.""" + arms = [ + # (name, log text, floor, expected exit) + ("#374 after the fix (31.3 t/s) meets a 20 floor", + "llama_perf_context_print: eval time = 200.00 ms / 32 runs (6.25 ms per token, 31.3 tokens per second)\n", 20.0, 0), + ("#374 before the fix (8.4 t/s) fails the same floor", + "llama_perf_context_print: eval time = 200.00 ms / 32 runs (119 ms per token, 8.4 tokens per second)\n", 20.0, 1), + ("a log with no rate is a SKIP (77), not a pass", + "llama-cli: loading model\nerror: failed to load\n", 20.0, SKIP), + ("the last rate wins: prompt 400 then eval 30 is judged on 30", + "prompt eval: 400.0 tokens per second\neval: 30.0 tokens per second\n", 20.0, 0), + # engine#2426's lesson applied to rates: a floor without its subject is a recollection, not evidence. + ("the verdict carries its subject, so a pasted failure is checkable", + "eval: 8.4 tokens per second\n", 20.0, 1), + ] + bad = 0 + SUBJECT = "selftest q4_k_m ctx2048" + for name, text, floor, want in arms: + got, detail = verdict(parse_rates(text), floor, SUBJECT) + ok = got == want + bad += 0 if ok else 1 + print(f" {'ok ' if ok else 'FAIL'} {name}{'' if ok else f' (got {got}, want {want})'}") + if ok: + subj = detail.get("subject", "MISSING") + if subj != SUBJECT: + print(f" FAIL arm has no subject in its verdict (got {subj!r})") + bad += 1 + if ok and got == 1: + # the failure must NAME the number, or it is a red nobody can act on + msg = json.dumps(detail) + if "8.4" not in msg or "floor" not in msg: + print(" FAIL the below-floor verdict does not name the rate and the floor") + bad += 1 + print(f"{len(arms)} arms, {bad} failed") + return 1 if bad else 0 + + +def main(): + ap = argparse.ArgumentParser(description="Fail when a run generates below a token-rate floor.") + ap.add_argument("--log", help="llama-cli or one-line harness output to judge") + ap.add_argument("--floor", type=float, default=0.0, help="minimum generation tokens/s (0 = record only)") + ap.add_argument("--subject", default="", help="model/quant/ctx that produced the rate, e.g. 'qwen3-0.6b Q4_K_M ctx2048'") + ap.add_argument("--json", action="store_true", help="print the verdict as JSON") + ap.add_argument("--self-test", action="store_true", help="run the four arms, no device needed") + args = ap.parse_args() + if args.self_test: + return self_test() + if not args.log: + ap.error("--log is required unless --self-test is given") + try: + with open(args.log, "r", errors="replace") as fh: + text = fh.read() + except OSError as exc: + print(f"token_rate_floor: cannot read {args.log}: {exc}", file=sys.stderr) + return SKIP + code, detail = verdict(parse_rates(text), args.floor, args.subject) + if args.json: + print(json.dumps(detail)) + else: + # A floor is a number tied to a model, quant, context and box. Without the subject a pasted failure is a + # recollection, not evidence (engine#2426's AUDITED lesson, applied to rates). + print(f"JUDGED {args.subject or ''}") + print(f"{detail['status']}: {detail.get('rate', '-')} tokens/s (floor {args.floor:g})" + + (f" -- {detail['why']}" if detail.get("why") else "")) + return code + + +if __name__ == "__main__": + sys.exit(main()) diff --git a/tools/ci/token_rate_floor_device.sh b/tools/ci/token_rate_floor_device.sh new file mode 100755 index 000000000..1e2dee3f7 --- /dev/null +++ b/tools/ci/token_rate_floor_device.sh @@ -0,0 +1,83 @@ +#!/bin/bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# The caller for `token_rate_floor.py --log`. Without it the tool has a self-test and no consumer, which is +# engine#2418's defect: arms that are green and never applied to a real run (engine#2432's review, item 1). +# +# Judges ONE sequential `1bit serve` generation against TOKEN_RATE_FLOOR (default 0 = record only). Refuses +# with 77 -- CTest's skip code -- when it cannot run, and says which prerequisite is missing: a device row +# that silently passes because there was no model would be exactly the failure this tool exists to catch. +set -u +ROOT=${1:-$(cd "$(dirname "$0")/../.." && pwd)} +FLOOR=${TOKEN_RATE_FLOOR:-0} +PORT=${TOKEN_RATE_PORT:-8191} +SKIP=77 + +BIN=${TOKEN_RATE_BIN:-} +for cand in "$ROOT/build/1bit" "$ROOT/b2180/1bit"; do [ -n "$BIN" ] || { [ -x "$cand" ] && BIN=$cand; }; done +[ -n "$BIN" ] || BIN=$(command -v 1bit 2>/dev/null || true) +MODEL=${TOKEN_RATE_MODEL:-} +[ -n "$MODEL" ] || MODEL=$(ls "$HOME"/models/Qwen3-0.6B-Q4_K_M.gguf 2>/dev/null | head -1) +HOLD=${TOKEN_RATE_GPU_HOLD:-$(command -v gpu-hold || echo "$HOME/.local/bin/gpu-hold")} + +if [ -z "$BIN" ] || [ ! -x "$BIN" ]; then + echo "SKIPPED: token_rate_floor device row did NOT run -- no 1bit binary (set TOKEN_RATE_BIN)"; exit $SKIP +fi +if [ -z "$MODEL" ] || [ ! -f "$MODEL" ]; then + echo "SKIPPED: token_rate_floor device row did NOT run -- no model (set TOKEN_RATE_MODEL)"; exit $SKIP +fi +if [ ! -x "$HOLD" ]; then + echo "SKIPPED: token_rate_floor device row did NOT run -- gpu-hold not found ($HOLD), and a GPU run outside a ticket is not allowed"; exit $SKIP +fi + +LOG=$(mktemp -t token-rate.XXXXXX.log) +cleanup() { + local p; p=$(pgrep -f "port $PORT" | head -1) + [ -n "$p" ] && { kill -TERM "$p" 2>/dev/null; for _ in $(seq 1 20); do kill -0 "$p" 2>/dev/null || break; sleep 1; done; } + rm -f "$LOG" +} +trap cleanup EXIT + +# The whole run lives inside the ticket: the process ends when the ticket ends. +"$HOLD" -t 10 -w 900 -- bash -c ' + set -u + nohup "$1" serve -m "$2" --device hrx --host 127.0.0.1 --port "$3" --ctx-size 2048 --alias p --no-strict-device >/dev/null 2>&1 & + for _ in $(seq 1 40); do curl -sf -m 3 "http://127.0.0.1:$3/health" >/dev/null 2>&1 && break; sleep 3; done + curl -sf -m 5 "http://127.0.0.1:$3/health" >/dev/null || { echo "SKIPPED: server never became ready"; exit 77; } + body=$(curl -s -m 300 "http://127.0.0.1:$3/v1/chat/completions" -H "content-type: application/json" \ + -d "{\"model\":\"p\",\"messages\":[{\"role\":\"user\",\"content\":\"Write a short paragraph about rivers.\"}],\"max_tokens\":64,\"temperature\":0}") + case "$body" in *'"error"'*) echo "SKIPPED: server returned an error: $(printf %s "$body" | head -c 120)"; exit 77;; esac + printf %s "$body" | python3 -c " +import json,sys,time +d=json.load(sys.stdin); c=(d.get(\"usage\") or {}).get(\"completion_tokens\") or 0 +print(f\"{c} tokens\")" >/dev/null 2>&1 + t0=$(date +%s.%N); body=$(curl -s -m 300 "http://127.0.0.1:$3/v1/chat/completions" -H "content-type: application/json" \ + -d "{\"model\":\"p\",\"messages\":[{\"role\":\"user\",\"content\":\"Write a short paragraph about rivers.\"}],\"max_tokens\":64,\"temperature\":0}"); t1=$(date +%s.%N) + case "$body" in *'"error"'*) echo "SKIPPED: server returned an error on the measured run"; exit 77;; esac + printf %s "$body" | python3 -c " +import json,sys +w=float(sys.argv[1]); d=json.load(sys.stdin); c=(d.get(\"usage\") or {}).get(\"completion_tokens\") or 0 +print(f\"{c/w:.2f} tokens per second\" if c>0 and w>0 else \"\")" "$(echo "$t1 - $t0" | bc)" +' _ "$BIN" "$MODEL" "$PORT" > "$LOG" 2>&1 +rc=$? + +if [ $rc -eq $SKIP ] || ! grep -q "tokens per second" "$LOG" 2>/dev/null; then + echo "SKIPPED: token_rate_floor device row did NOT run -- $(tail -1 "$LOG" 2>/dev/null | head -c 120)"; exit $SKIP +fi + +# The rate goes through the tool, which owns the judging, the floor and the subject line. +exec python3 "$(dirname "$0")/token_rate_floor.py" --log "$LOG" --floor "$FLOOR" \ + --subject "1bit serve $(basename "$MODEL") ctx2048"