Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
9 changes: 9 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,15 @@ add_test(NAME watchdog_serve COMMAND ${CMAKE_SOURCE_DIR}/tests/watchdog_serve.sh
add_test(NAME metrics_serve COMMAND ${CMAKE_SOURCE_DIR}/tests/metrics_serve.sh $<TARGET_FILE:onebit>)
# serve --print-backend: the llama-server Lemonade starts itself, one hop (docs/serve.md): no GPU.
add_test(NAME print_backend COMMAND ${CMAKE_SOURCE_DIR}/tests/print_backend.sh $<TARGET_FILE:onebit>)
# token_rate_floor's four arms (engine#374): the rate floor that fails in the direction a correctness check
# cannot, since a 3.7x regression still emits the right tokens. The self-test needs no device; the --log mode
# is what a device row calls with llama-cli's output.
add_test(NAME token_rate_floor_selftest COMMAND python3 ${CMAKE_SOURCE_DIR}/tools/ci/token_rate_floor.py --self-test)
# The --log mode's caller (engine#2432 review item 1: a self-test with no consumer is engine#2418's defect). It
# judges one real `1bit serve` rate inside a gpu-hold ticket, and exits 77 -- CTest's skip -- naming the missing
# prerequisite when it cannot: a device row that passes for want of a model would be the failure this tool catches.
add_test(NAME token_rate_floor_device COMMAND ${CMAKE_SOURCE_DIR}/tools/ci/token_rate_floor_device.sh ${CMAKE_SOURCE_DIR})
set_tests_properties(token_rate_floor_device PROPERTIES SKIP_RETURN_CODE 77)
# scripts/lemonade-hrx: the PPA lemond's HRX recipe runs our llama-server (docs/lemonade.md): no GPU.
add_test(NAME lemonade_hrx COMMAND ${CMAKE_SOURCE_DIR}/tests/lemonade_hrx.sh $<TARGET_FILE:onebit>)
add_test(NAME parallel_args COMMAND ${CMAKE_SOURCE_DIR}/tests/parallel_args.sh $<TARGET_FILE:onebit>)
Expand Down
144 changes: 144 additions & 0 deletions tools/ci/token_rate_floor.py
Original file line number Diff line number Diff line change
@@ -0,0 +1,144 @@
#!/usr/bin/env python3
# Copyright 2026 bong-water-water-bong
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""token_rate_floor.py: fail when a model generates slower than a floor, so a silent slowdown is caught.

engine#374 is why this exists. `model_buffer_from_mmap()` retagged a CPU buffer as `mapped_buft` while leaving
its context a CPU context. The result was **correct output at 8.4 t/s instead of 31.3** -- a 3.7x regression that
no test caught, because every test that watched the buffer was watching the *assertion* it satisfied, not the
speed the machine was doing it at. `hrx-buffer-test` went red much later, on an alignment assert, and the rate
had been quietly wrong for as long as the retag had.

A rate floor is the direct measurement of the thing users feel, and it fails in the direction a correctness
check cannot: a run that is 3x too slow still produces the right tokens.

token_rate_floor.py --log run.log --floor 20 # judge a saved llama-cli / 1bit run
token_rate_floor.py --log run.log --floor 20 --json # machine-readable verdict
token_rate_floor.py --self-test # the four arms, no device needed

**Set the floor below the MINIMUM of at least three interleaved runs, not below a median or a single good one.** A single
rate is not a stable subject on this box: four *identical* 128-token decode runs at one config on engine#679 measured
7.94 / 11.78 / 10.39 / 12.57 t/s -- median 11.09, **spread 1.58x**. A floor judged on one sample flaps for reasons that
have nothing to do with a regression, and a flapping row gets ignored, which is how noisy checks die. Judging a range
instead (the way tools/ci/medians.py decides A/B by disjoint ranges) is more correct and more work; this file takes the
cheap honest path: one sentence here, and a caller who runs three times.

The log is llama-cli's own output ("llama_perf_context_print: prompt eval time = ... 31.3 tokens per second"),
or a one-line `N tokens per second`; the LAST rate in the file is used, because a warmup line is not the run.
Exit: 0 met the floor, 1 below it or unparseable, 77 no rate in the log at all (a skip, not a red).
"""
import argparse
import json
import re
import sys

SKIP = 77

# llama-cli: "eval time = 200.00 ms / 32 runs ( 6.25 ms per token, 160.00 tokens per second)"
# and shorter forms ("31.3 t/s", "31.3 tok/s") for one-line harnesses.
_RATE = re.compile(r"(\d+(?:\.\d+)?)\s*(?:tokens per second|tok/s|t/s)")


def parse_rates(text):
"""Every rate in the log, in order. Multiline, because llama-cli prints prompt and eval rates separately."""
return [float(m) for m in _RATE.findall(text or "")]


def verdict(rates, floor, subject=""):
"""-> (exit_code, dict). The LAST rate is the generation rate; earlier ones are prompt eval."""
if not rates:
return SKIP, {"status": "skip", "subject": subject,
"why": "no token rate in the log -- the run did not report one"}
last = rates[-1]
# A floor of 0 means "record, do not judge" -- useful while calibrating on a new box.
if floor <= 0:
return 0, {"status": "recorded", "rate": last, "floor": floor, "subject": subject}
if last < floor:
return 1, {"status": "below_floor", "rate": last, "floor": floor, "subject": subject,
"why": f"generated {last:g} tokens/s, floor is {floor:g} -- correct output, "
f"{floor / last:.1f}x below the floor"}
return 0, {"status": "ok", "rate": last, "floor": floor, "subject": subject}


def self_test():
"""Four arms. The second is the one this file exists for: #374's own before/after pair."""
arms = [
# (name, log text, floor, expected exit)
("#374 after the fix (31.3 t/s) meets a 20 floor",
"llama_perf_context_print: eval time = 200.00 ms / 32 runs (6.25 ms per token, 31.3 tokens per second)\n", 20.0, 0),
("#374 before the fix (8.4 t/s) fails the same floor",
"llama_perf_context_print: eval time = 200.00 ms / 32 runs (119 ms per token, 8.4 tokens per second)\n", 20.0, 1),
("a log with no rate is a SKIP (77), not a pass",
"llama-cli: loading model\nerror: failed to load\n", 20.0, SKIP),
("the last rate wins: prompt 400 then eval 30 is judged on 30",
"prompt eval: 400.0 tokens per second\neval: 30.0 tokens per second\n", 20.0, 0),
# engine#2426's lesson applied to rates: a floor without its subject is a recollection, not evidence.
("the verdict carries its subject, so a pasted failure is checkable",
"eval: 8.4 tokens per second\n", 20.0, 1),
]
bad = 0
SUBJECT = "selftest q4_k_m ctx2048"
for name, text, floor, want in arms:
got, detail = verdict(parse_rates(text), floor, SUBJECT)
ok = got == want
bad += 0 if ok else 1
print(f" {'ok ' if ok else 'FAIL'} {name}{'' if ok else f' (got {got}, want {want})'}")
if ok:
subj = detail.get("subject", "MISSING")
if subj != SUBJECT:
print(f" FAIL arm has no subject in its verdict (got {subj!r})")
bad += 1
if ok and got == 1:
# the failure must NAME the number, or it is a red nobody can act on
msg = json.dumps(detail)
if "8.4" not in msg or "floor" not in msg:
print(" FAIL the below-floor verdict does not name the rate and the floor")
bad += 1
print(f"{len(arms)} arms, {bad} failed")
return 1 if bad else 0


def main():
ap = argparse.ArgumentParser(description="Fail when a run generates below a token-rate floor.")
ap.add_argument("--log", help="llama-cli or one-line harness output to judge")
ap.add_argument("--floor", type=float, default=0.0, help="minimum generation tokens/s (0 = record only)")
ap.add_argument("--subject", default="", help="model/quant/ctx that produced the rate, e.g. 'qwen3-0.6b Q4_K_M ctx2048'")
ap.add_argument("--json", action="store_true", help="print the verdict as JSON")
ap.add_argument("--self-test", action="store_true", help="run the four arms, no device needed")
args = ap.parse_args()
if args.self_test:
return self_test()
if not args.log:
ap.error("--log is required unless --self-test is given")
try:
with open(args.log, "r", errors="replace") as fh:
text = fh.read()
except OSError as exc:
print(f"token_rate_floor: cannot read {args.log}: {exc}", file=sys.stderr)
return SKIP
code, detail = verdict(parse_rates(text), args.floor, args.subject)
if args.json:
print(json.dumps(detail))
else:
# A floor is a number tied to a model, quant, context and box. Without the subject a pasted failure is a
# recollection, not evidence (engine#2426's AUDITED <sha> lesson, applied to rates).
print(f"JUDGED {args.subject or '<subject not given>'}")
print(f"{detail['status']}: {detail.get('rate', '-')} tokens/s (floor {args.floor:g})"
+ (f" -- {detail['why']}" if detail.get("why") else ""))
return code


if __name__ == "__main__":
sys.exit(main())
83 changes: 83 additions & 0 deletions tools/ci/token_rate_floor_device.sh
Original file line number Diff line number Diff line change
@@ -0,0 +1,83 @@
#!/bin/bash
# Copyright 2026 bong-water-water-bong
# SPDX-License-Identifier: Apache-2.0
#
# Licensed under the Apache License, Version 2.0 (the "License");
# you may not use this file except in compliance with the License.
# You may obtain a copy of the License at
#
# http://www.apache.org/licenses/LICENSE-2.0
#
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
#
# The caller for `token_rate_floor.py --log`. Without it the tool has a self-test and no consumer, which is
# engine#2418's defect: arms that are green and never applied to a real run (engine#2432's review, item 1).
#
# Judges ONE sequential `1bit serve` generation against TOKEN_RATE_FLOOR (default 0 = record only). Refuses
# with 77 -- CTest's skip code -- when it cannot run, and says which prerequisite is missing: a device row
# that silently passes because there was no model would be exactly the failure this tool exists to catch.
set -u
ROOT=${1:-$(cd "$(dirname "$0")/../.." && pwd)}
FLOOR=${TOKEN_RATE_FLOOR:-0}
PORT=${TOKEN_RATE_PORT:-8191}
SKIP=77

BIN=${TOKEN_RATE_BIN:-}
for cand in "$ROOT/build/1bit" "$ROOT/b2180/1bit"; do [ -n "$BIN" ] || { [ -x "$cand" ] && BIN=$cand; }; done
[ -n "$BIN" ] || BIN=$(command -v 1bit 2>/dev/null || true)
MODEL=${TOKEN_RATE_MODEL:-}
[ -n "$MODEL" ] || MODEL=$(ls "$HOME"/models/Qwen3-0.6B-Q4_K_M.gguf 2>/dev/null | head -1)
HOLD=${TOKEN_RATE_GPU_HOLD:-$(command -v gpu-hold || echo "$HOME/.local/bin/gpu-hold")}

if [ -z "$BIN" ] || [ ! -x "$BIN" ]; then
echo "SKIPPED: token_rate_floor device row did NOT run -- no 1bit binary (set TOKEN_RATE_BIN)"; exit $SKIP
fi
if [ -z "$MODEL" ] || [ ! -f "$MODEL" ]; then
echo "SKIPPED: token_rate_floor device row did NOT run -- no model (set TOKEN_RATE_MODEL)"; exit $SKIP
fi
if [ ! -x "$HOLD" ]; then
echo "SKIPPED: token_rate_floor device row did NOT run -- gpu-hold not found ($HOLD), and a GPU run outside a ticket is not allowed"; exit $SKIP
fi

LOG=$(mktemp -t token-rate.XXXXXX.log)
cleanup() {
local p; p=$(pgrep -f "port $PORT" | head -1)
[ -n "$p" ] && { kill -TERM "$p" 2>/dev/null; for _ in $(seq 1 20); do kill -0 "$p" 2>/dev/null || break; sleep 1; done; }
rm -f "$LOG"
}
trap cleanup EXIT

# The whole run lives inside the ticket: the process ends when the ticket ends.
"$HOLD" -t 10 -w 900 -- bash -c '
set -u
nohup "$1" serve -m "$2" --device hrx --host 127.0.0.1 --port "$3" --ctx-size 2048 --alias p --no-strict-device >/dev/null 2>&1 &
for _ in $(seq 1 40); do curl -sf -m 3 "http://127.0.0.1:$3/health" >/dev/null 2>&1 && break; sleep 3; done
curl -sf -m 5 "http://127.0.0.1:$3/health" >/dev/null || { echo "SKIPPED: server never became ready"; exit 77; }
body=$(curl -s -m 300 "http://127.0.0.1:$3/v1/chat/completions" -H "content-type: application/json" \
-d "{\"model\":\"p\",\"messages\":[{\"role\":\"user\",\"content\":\"Write a short paragraph about rivers.\"}],\"max_tokens\":64,\"temperature\":0}")
case "$body" in *'"error"'*) echo "SKIPPED: server returned an error: $(printf %s "$body" | head -c 120)"; exit 77;; esac
printf %s "$body" | python3 -c "
import json,sys,time
d=json.load(sys.stdin); c=(d.get(\"usage\") or {}).get(\"completion_tokens\") or 0
print(f\"{c} tokens\")" >/dev/null 2>&1
t0=$(date +%s.%N); body=$(curl -s -m 300 "http://127.0.0.1:$3/v1/chat/completions" -H "content-type: application/json" \
-d "{\"model\":\"p\",\"messages\":[{\"role\":\"user\",\"content\":\"Write a short paragraph about rivers.\"}],\"max_tokens\":64,\"temperature\":0}"); t1=$(date +%s.%N)
case "$body" in *'"error"'*) echo "SKIPPED: server returned an error on the measured run"; exit 77;; esac
printf %s "$body" | python3 -c "
import json,sys
w=float(sys.argv[1]); d=json.load(sys.stdin); c=(d.get(\"usage\") or {}).get(\"completion_tokens\") or 0
print(f\"{c/w:.2f} tokens per second\" if c>0 and w>0 else \"\")" "$(echo "$t1 - $t0" | bc)"
' _ "$BIN" "$MODEL" "$PORT" > "$LOG" 2>&1
rc=$?

if [ $rc -eq $SKIP ] || ! grep -q "tokens per second" "$LOG" 2>/dev/null; then
echo "SKIPPED: token_rate_floor device row did NOT run -- $(tail -1 "$LOG" 2>/dev/null | head -c 120)"; exit $SKIP
fi

# The rate goes through the tool, which owns the judging, the floor and the subject line.
exec python3 "$(dirname "$0")/token_rate_floor.py" --log "$LOG" --floor "$FLOOR" \
--subject "1bit serve $(basename "$MODEL") ctx2048"
Loading