From fc349743bc1f3f8bc65522561ebdd8571cdbeaad Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sat, 14 Nov 2026 09:24:38 -0400 Subject: [PATCH 1/4] =?UTF-8?q?tools/ci:=20token=5Frate=5Ffloor=20?= =?UTF-8?q?=E2=80=94=20fail=20when=20a=20run=20generates=20below=20a=20flo?= =?UTF-8?q?or?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit engine#374 is why this exists: model_buffer_from_mmap() retagged a CPU buffer as mapped_buft while leaving its context a CPU context, and the result was correct output at 8.4 t/s instead of 31.3 — a 3.7x regression that no correctness test caught, because every test watching that buffer was watching the assertion it satisfied, not the speed it satisfied it at. hrx-buffer-test went red much later, on an alignment assert. A rate floor measures the thing users feel and fails in the direction a correctness check cannot: a run that is 3x too slow still emits the right tokens. Values named on failure, SKIP (77) when the log carries no rate at all — a missing rate is not a pass. Four arms in --self-test, the second being #374's own before/after pair (31.3 passes a 20 floor, 8.4 fails it). Wired as a ctest row so it cannot become a checker with no caller; the device mode is --log --floor N. --- CMakeLists.txt | 4 ++ tools/ci/token_rate_floor.py | 110 +++++++++++++++++++++++++++++++++++ 2 files changed, 114 insertions(+) create mode 100755 tools/ci/token_rate_floor.py diff --git a/CMakeLists.txt b/CMakeLists.txt index 1985ce08f..bcd2d5da1 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -199,6 +199,10 @@ add_test(NAME watchdog_serve COMMAND ${CMAKE_SOURCE_DIR}/tests/watchdog_serve.sh add_test(NAME metrics_serve COMMAND ${CMAKE_SOURCE_DIR}/tests/metrics_serve.sh $) # serve --print-backend: the llama-server Lemonade starts itself, one hop (docs/serve.md): no GPU. add_test(NAME print_backend COMMAND ${CMAKE_SOURCE_DIR}/tests/print_backend.sh $) +# token_rate_floor's four arms (engine#374): the rate floor that fails in the direction a correctness check +# cannot, since a 3.7x regression still emits the right tokens. The self-test needs no device; the --log mode +# is what a device row calls with llama-cli's output. +add_test(NAME token_rate_floor_selftest COMMAND python3 ${CMAKE_SOURCE_DIR}/tools/ci/token_rate_floor.py --self-test) # scripts/lemonade-hrx: the PPA lemond's HRX recipe runs our llama-server (docs/lemonade.md): no GPU. add_test(NAME lemonade_hrx COMMAND ${CMAKE_SOURCE_DIR}/tests/lemonade_hrx.sh $) add_test(NAME parallel_args COMMAND ${CMAKE_SOURCE_DIR}/tests/parallel_args.sh $) diff --git a/tools/ci/token_rate_floor.py b/tools/ci/token_rate_floor.py new file mode 100755 index 000000000..ddf26649d --- /dev/null +++ b/tools/ci/token_rate_floor.py @@ -0,0 +1,110 @@ +#!/usr/bin/env python3 +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +"""token_rate_floor.py: fail when a model generates slower than a floor, so a silent slowdown is caught. + +engine#374 is why this exists. `model_buffer_from_mmap()` retagged a CPU buffer as `mapped_buft` while leaving +its context a CPU context. The result was **correct output at 8.4 t/s instead of 31.3** -- a 3.7x regression that +no test caught, because every test that watched the buffer was watching the *assertion* it satisfied, not the +speed the machine was doing it at. `hrx-buffer-test` went red much later, on an alignment assert, and the rate +had been quietly wrong for as long as the retag had. + +A rate floor is the direct measurement of the thing users feel, and it fails in the direction a correctness +check cannot: a run that is 3x too slow still produces the right tokens. + + token_rate_floor.py --log run.log --floor 20 # judge a saved llama-cli / 1bit run + token_rate_floor.py --log run.log --floor 20 --json # machine-readable verdict + token_rate_floor.py --self-test # the four arms, no device needed + +The log is llama-cli's own output ("llama_perf_context_print: prompt eval time = ... 31.3 tokens per second"), +or a one-line `N tokens per second`; the LAST rate in the file is used, because a warmup line is not the run. +Exit: 0 met the floor, 1 below it or unparseable, 77 no rate in the log at all (a skip, not a red). +""" +import argparse +import json +import re +import sys + +SKIP = 77 + +# llama-cli: "eval time = 200.00 ms / 32 runs ( 6.25 ms per token, 160.00 tokens per second)" +# and shorter forms ("31.3 t/s", "31.3 tok/s") for one-line harnesses. +_RATE = re.compile(r"(\d+(?:\.\d+)?)\s*(?:tokens per second|tok/s|t/s)") + + +def parse_rates(text): + """Every rate in the log, in order. Multiline, because llama-cli prints prompt and eval rates separately.""" + return [float(m) for m in _RATE.findall(text or "")] + + +def verdict(rates, floor): + """-> (exit_code, dict). The LAST rate is the generation rate; earlier ones are prompt eval.""" + if not rates: + return SKIP, {"status": "skip", "why": "no token rate in the log -- the run did not report one"} + last = rates[-1] + # A floor of 0 means "record, do not judge" -- useful while calibrating on a new box. + if floor <= 0: + return 0, {"status": "recorded", "rate": last, "floor": floor} + if last < floor: + return 1, {"status": "below_floor", "rate": last, "floor": floor, + "why": f"generated {last:g} tokens/s, floor is {floor:g} -- a slowdown, not a failure"} + return 0, {"status": "ok", "rate": last, "floor": floor} + + +def self_test(): + """Four arms. The second is the one this file exists for: #374's own before/after pair.""" + arms = [ + # (name, log text, floor, expected exit) + ("#374 after the fix (31.3 t/s) meets a 20 floor", + "llama_perf_context_print: eval time = 200.00 ms / 32 runs (6.25 ms per token, 31.3 tokens per second)\n", 20.0, 0), + ("#374 before the fix (8.4 t/s) fails the same floor", + "llama_perf_context_print: eval time = 200.00 ms / 32 runs (119 ms per token, 8.4 tokens per second)\n", 20.0, 1), + ("a log with no rate is a SKIP (77), not a pass", + "llama-cli: loading model\nerror: failed to load\n", 20.0, SKIP), + ("the last rate wins: prompt 400 then eval 30 is judged on 30", + "prompt eval: 400.0 tokens per second\neval: 30.0 tokens per second\n", 20.0, 0), + ] + bad = 0 + for name, text, floor, want in arms: + got, detail = verdict(parse_rates(text), floor) + ok = got == want + bad += 0 if ok else 1 + print(f" {'ok ' if ok else 'FAIL'} {name}{'' if ok else f' (got {got}, want {want})'}") + if ok and got == 1: + # the failure must NAME the number, or it is a red nobody can act on + msg = json.dumps(detail) + if "8.4" not in msg or "floor" not in msg: + print(" FAIL the below-floor verdict does not name the rate and the floor") + bad += 1 + print(f"{len(arms)} arms, {bad} failed") + return 1 if bad else 0 + + +def main(): + ap = argparse.ArgumentParser(description="Fail when a run generates below a token-rate floor.") + ap.add_argument("--log", help="llama-cli or one-line harness output to judge") + ap.add_argument("--floor", type=float, default=0.0, help="minimum generation tokens/s (0 = record only)") + ap.add_argument("--json", action="store_true", help="print the verdict as JSON") + ap.add_argument("--self-test", action="store_true", help="run the four arms, no device needed") + args = ap.parse_args() + if args.self_test: + return self_test() + if not args.log: + ap.error("--log is required unless --self-test is given") + try: + with open(args.log, "r", errors="replace") as fh: + text = fh.read() + except OSError as exc: + print(f"token_rate_floor: cannot read {args.log}: {exc}", file=sys.stderr) + return SKIP + code, detail = verdict(parse_rates(text), args.floor) + if args.json: + print(json.dumps(detail)) + else: + print(f"{detail['status']}: {detail.get('rate', '-')} tokens/s (floor {args.floor:g})" + + (f" -- {detail['why']}" if detail.get("why") else "")) + return code + + +if __name__ == "__main__": + sys.exit(main()) From 0e54e2861c98390fd17f50f7d342d74e792573b6 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sat, 14 Nov 2026 11:16:12 -0400 Subject: [PATCH 2/4] token_rate_floor: name the subject, and warn that a floor needs the minimum of three runs Both from the review's asks, both from measurements this box already made: - JUDGED before the verdict. A floor is a number tied to a model, quant, context and box; without the subject a pasted 'below_floor: 8.4' is a recollection, not evidence. Same lesson engine#2426 landed for AUDITED tonight. - The usage now says to set the floor below the MINIMUM of >=3 interleaved runs, and says why with the number: four identical 128-token decode runs on engine#679 measured 7.94 / 11.78 / 10.39 / 12.57 t/s -- median 11.09, spread 1.58x. A floor judged on one sample flaps for reasons unrelated to a regression, and a flapping row gets ignored. Judging a range instead (tools/ci/medians.py's disjoint-ranges A/B) is more correct and more work; not done here, named as the better next step. Fifth arm pins that every verdict carries its subject. --- tools/ci/token_rate_floor.py | 35 ++++++++++++++++++++++++++++------- 1 file changed, 28 insertions(+), 7 deletions(-) diff --git a/tools/ci/token_rate_floor.py b/tools/ci/token_rate_floor.py index ddf26649d..bf3a490fd 100755 --- a/tools/ci/token_rate_floor.py +++ b/tools/ci/token_rate_floor.py @@ -16,6 +16,13 @@ token_rate_floor.py --log run.log --floor 20 --json # machine-readable verdict token_rate_floor.py --self-test # the four arms, no device needed +**Set the floor below the MINIMUM of at least three interleaved runs, not below a median or a single good one.** A single +rate is not a stable subject on this box: four *identical* 128-token decode runs at one config on engine#679 measured +7.94 / 11.78 / 10.39 / 12.57 t/s -- median 11.09, **spread 1.58x**. A floor judged on one sample flaps for reasons that +have nothing to do with a regression, and a flapping row gets ignored, which is how noisy checks die. Judging a range +instead (the way tools/ci/medians.py decides A/B by disjoint ranges) is more correct and more work; this file takes the +cheap honest path: one sentence here, and a caller who runs three times. + The log is llama-cli's own output ("llama_perf_context_print: prompt eval time = ... 31.3 tokens per second"), or a one-line `N tokens per second`; the LAST rate in the file is used, because a warmup line is not the run. Exit: 0 met the floor, 1 below it or unparseable, 77 no rate in the log at all (a skip, not a red). @@ -37,18 +44,19 @@ def parse_rates(text): return [float(m) for m in _RATE.findall(text or "")] -def verdict(rates, floor): +def verdict(rates, floor, subject=""): """-> (exit_code, dict). The LAST rate is the generation rate; earlier ones are prompt eval.""" if not rates: - return SKIP, {"status": "skip", "why": "no token rate in the log -- the run did not report one"} + return SKIP, {"status": "skip", "subject": subject, + "why": "no token rate in the log -- the run did not report one"} last = rates[-1] # A floor of 0 means "record, do not judge" -- useful while calibrating on a new box. if floor <= 0: - return 0, {"status": "recorded", "rate": last, "floor": floor} + return 0, {"status": "recorded", "rate": last, "floor": floor, "subject": subject} if last < floor: - return 1, {"status": "below_floor", "rate": last, "floor": floor, + return 1, {"status": "below_floor", "rate": last, "floor": floor, "subject": subject, "why": f"generated {last:g} tokens/s, floor is {floor:g} -- a slowdown, not a failure"} - return 0, {"status": "ok", "rate": last, "floor": floor} + return 0, {"status": "ok", "rate": last, "floor": floor, "subject": subject} def self_test(): @@ -63,13 +71,22 @@ def self_test(): "llama-cli: loading model\nerror: failed to load\n", 20.0, SKIP), ("the last rate wins: prompt 400 then eval 30 is judged on 30", "prompt eval: 400.0 tokens per second\neval: 30.0 tokens per second\n", 20.0, 0), + # engine#2426's lesson applied to rates: a floor without its subject is a recollection, not evidence. + ("the verdict carries its subject, so a pasted failure is checkable", + "eval: 8.4 tokens per second\n", 20.0, 1), ] bad = 0 + SUBJECT = "selftest q4_k_m ctx2048" for name, text, floor, want in arms: - got, detail = verdict(parse_rates(text), floor) + got, detail = verdict(parse_rates(text), floor, SUBJECT) ok = got == want bad += 0 if ok else 1 print(f" {'ok ' if ok else 'FAIL'} {name}{'' if ok else f' (got {got}, want {want})'}") + if ok: + subj = detail.get("subject", "MISSING") + if subj != SUBJECT: + print(f" FAIL arm has no subject in its verdict (got {subj!r})") + bad += 1 if ok and got == 1: # the failure must NAME the number, or it is a red nobody can act on msg = json.dumps(detail) @@ -84,6 +101,7 @@ def main(): ap = argparse.ArgumentParser(description="Fail when a run generates below a token-rate floor.") ap.add_argument("--log", help="llama-cli or one-line harness output to judge") ap.add_argument("--floor", type=float, default=0.0, help="minimum generation tokens/s (0 = record only)") + ap.add_argument("--subject", default="", help="model/quant/ctx that produced the rate, e.g. 'qwen3-0.6b Q4_K_M ctx2048'") ap.add_argument("--json", action="store_true", help="print the verdict as JSON") ap.add_argument("--self-test", action="store_true", help="run the four arms, no device needed") args = ap.parse_args() @@ -97,10 +115,13 @@ def main(): except OSError as exc: print(f"token_rate_floor: cannot read {args.log}: {exc}", file=sys.stderr) return SKIP - code, detail = verdict(parse_rates(text), args.floor) + code, detail = verdict(parse_rates(text), args.floor, args.subject) if args.json: print(json.dumps(detail)) else: + # A floor is a number tied to a model, quant, context and box. Without the subject a pasted failure is a + # recollection, not evidence (engine#2426's AUDITED lesson, applied to rates). + print(f"JUDGED {args.subject or ''}") print(f"{detail['status']}: {detail.get('rate', '-')} tokens/s (floor {args.floor:g})" + (f" -- {detail['why']}" if detail.get("why") else "")) return code From 1a5785d879eaecd570124aede61f568e8db3e3ae Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sat, 14 Nov 2026 20:57:02 -0400 Subject: [PATCH 3/4] token_rate_floor: complete the licence header (the CI row was red since 15:16Z) The file carried '# Copyright' and '# SPDX-License-Identifier' but not the licence block, so copyright.py reported 'missing copyright notice' and the build failed at the configure/check step. The sequence matters and the first advice on the PR was wrong: --fix on the partial header inserts 14 lines that DUPLICATE those two, producing 'copyright notice duplicated inside the header window' -- one red row traded for another. Delete the two partial lines first, then --fix inserts the 12-line block and --check returns rc=0. Verified locally: copyright.py --check rc=0, --self-test 5 arms 0 failed. --- tools/ci/token_rate_floor.py | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/tools/ci/token_rate_floor.py b/tools/ci/token_rate_floor.py index bf3a490fd..d45cb43b9 100755 --- a/tools/ci/token_rate_floor.py +++ b/tools/ci/token_rate_floor.py @@ -1,6 +1,18 @@ #!/usr/bin/env python3 # Copyright 2026 bong-water-water-bong # SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. """token_rate_floor.py: fail when a model generates slower than a floor, so a silent slowdown is caught. engine#374 is why this exists. `model_buffer_from_mmap()` retagged a CPU buffer as `mapped_buft` while leaving From e0c03bd8a77f77da7842a60eba9ed65a4d2e0436 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sat, 14 Nov 2026 21:38:02 -0400 Subject: [PATCH 4/4] token_rate_floor: give --log a caller, and stop the failing verdict saying 'not a failure' Two asks from the review at 1a5785d, both right: 1. --log had no caller. One ctest row ran --self-test; the mode that would have caught engine#374 was invoked by nothing in the tree -- engine#2418's defect (arms green, never applied) in the tool written to catch that class. tools/ci/token_rate_floor_device.sh judges one real 1bit serve rate inside a gpu-hold ticket and exits 77, naming the missing prerequisite, when it cannot run. 2. below_floor printed '-- a slowdown, not a failure' while exiting 1: the status said failed and the sentence said not. It now prints the multiplier -- 'correct output, 2.4x below the floor' -- which is the number that made engine#374 worth measuring. --- CMakeLists.txt | 5 ++ tools/ci/token_rate_floor.py | 3 +- tools/ci/token_rate_floor_device.sh | 83 +++++++++++++++++++++++++++++ 3 files changed, 90 insertions(+), 1 deletion(-) create mode 100755 tools/ci/token_rate_floor_device.sh diff --git a/CMakeLists.txt b/CMakeLists.txt index bcd2d5da1..f34fd6640 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -203,6 +203,11 @@ add_test(NAME print_backend COMMAND ${CMAKE_SOURCE_DIR}/tests/print_backend.sh $ # cannot, since a 3.7x regression still emits the right tokens. The self-test needs no device; the --log mode # is what a device row calls with llama-cli's output. add_test(NAME token_rate_floor_selftest COMMAND python3 ${CMAKE_SOURCE_DIR}/tools/ci/token_rate_floor.py --self-test) +# The --log mode's caller (engine#2432 review item 1: a self-test with no consumer is engine#2418's defect). It +# judges one real `1bit serve` rate inside a gpu-hold ticket, and exits 77 -- CTest's skip -- naming the missing +# prerequisite when it cannot: a device row that passes for want of a model would be the failure this tool catches. +add_test(NAME token_rate_floor_device COMMAND ${CMAKE_SOURCE_DIR}/tools/ci/token_rate_floor_device.sh ${CMAKE_SOURCE_DIR}) +set_tests_properties(token_rate_floor_device PROPERTIES SKIP_RETURN_CODE 77) # scripts/lemonade-hrx: the PPA lemond's HRX recipe runs our llama-server (docs/lemonade.md): no GPU. add_test(NAME lemonade_hrx COMMAND ${CMAKE_SOURCE_DIR}/tests/lemonade_hrx.sh $) add_test(NAME parallel_args COMMAND ${CMAKE_SOURCE_DIR}/tests/parallel_args.sh $) diff --git a/tools/ci/token_rate_floor.py b/tools/ci/token_rate_floor.py index d45cb43b9..84fcf3fae 100755 --- a/tools/ci/token_rate_floor.py +++ b/tools/ci/token_rate_floor.py @@ -67,7 +67,8 @@ def verdict(rates, floor, subject=""): return 0, {"status": "recorded", "rate": last, "floor": floor, "subject": subject} if last < floor: return 1, {"status": "below_floor", "rate": last, "floor": floor, "subject": subject, - "why": f"generated {last:g} tokens/s, floor is {floor:g} -- a slowdown, not a failure"} + "why": f"generated {last:g} tokens/s, floor is {floor:g} -- correct output, " + f"{floor / last:.1f}x below the floor"} return 0, {"status": "ok", "rate": last, "floor": floor, "subject": subject} diff --git a/tools/ci/token_rate_floor_device.sh b/tools/ci/token_rate_floor_device.sh new file mode 100755 index 000000000..1e2dee3f7 --- /dev/null +++ b/tools/ci/token_rate_floor_device.sh @@ -0,0 +1,83 @@ +#!/bin/bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# The caller for `token_rate_floor.py --log`. Without it the tool has a self-test and no consumer, which is +# engine#2418's defect: arms that are green and never applied to a real run (engine#2432's review, item 1). +# +# Judges ONE sequential `1bit serve` generation against TOKEN_RATE_FLOOR (default 0 = record only). Refuses +# with 77 -- CTest's skip code -- when it cannot run, and says which prerequisite is missing: a device row +# that silently passes because there was no model would be exactly the failure this tool exists to catch. +set -u +ROOT=${1:-$(cd "$(dirname "$0")/../.." && pwd)} +FLOOR=${TOKEN_RATE_FLOOR:-0} +PORT=${TOKEN_RATE_PORT:-8191} +SKIP=77 + +BIN=${TOKEN_RATE_BIN:-} +for cand in "$ROOT/build/1bit" "$ROOT/b2180/1bit"; do [ -n "$BIN" ] || { [ -x "$cand" ] && BIN=$cand; }; done +[ -n "$BIN" ] || BIN=$(command -v 1bit 2>/dev/null || true) +MODEL=${TOKEN_RATE_MODEL:-} +[ -n "$MODEL" ] || MODEL=$(ls "$HOME"/models/Qwen3-0.6B-Q4_K_M.gguf 2>/dev/null | head -1) +HOLD=${TOKEN_RATE_GPU_HOLD:-$(command -v gpu-hold || echo "$HOME/.local/bin/gpu-hold")} + +if [ -z "$BIN" ] || [ ! -x "$BIN" ]; then + echo "SKIPPED: token_rate_floor device row did NOT run -- no 1bit binary (set TOKEN_RATE_BIN)"; exit $SKIP +fi +if [ -z "$MODEL" ] || [ ! -f "$MODEL" ]; then + echo "SKIPPED: token_rate_floor device row did NOT run -- no model (set TOKEN_RATE_MODEL)"; exit $SKIP +fi +if [ ! -x "$HOLD" ]; then + echo "SKIPPED: token_rate_floor device row did NOT run -- gpu-hold not found ($HOLD), and a GPU run outside a ticket is not allowed"; exit $SKIP +fi + +LOG=$(mktemp -t token-rate.XXXXXX.log) +cleanup() { + local p; p=$(pgrep -f "port $PORT" | head -1) + [ -n "$p" ] && { kill -TERM "$p" 2>/dev/null; for _ in $(seq 1 20); do kill -0 "$p" 2>/dev/null || break; sleep 1; done; } + rm -f "$LOG" +} +trap cleanup EXIT + +# The whole run lives inside the ticket: the process ends when the ticket ends. +"$HOLD" -t 10 -w 900 -- bash -c ' + set -u + nohup "$1" serve -m "$2" --device hrx --host 127.0.0.1 --port "$3" --ctx-size 2048 --alias p --no-strict-device >/dev/null 2>&1 & + for _ in $(seq 1 40); do curl -sf -m 3 "http://127.0.0.1:$3/health" >/dev/null 2>&1 && break; sleep 3; done + curl -sf -m 5 "http://127.0.0.1:$3/health" >/dev/null || { echo "SKIPPED: server never became ready"; exit 77; } + body=$(curl -s -m 300 "http://127.0.0.1:$3/v1/chat/completions" -H "content-type: application/json" \ + -d "{\"model\":\"p\",\"messages\":[{\"role\":\"user\",\"content\":\"Write a short paragraph about rivers.\"}],\"max_tokens\":64,\"temperature\":0}") + case "$body" in *'"error"'*) echo "SKIPPED: server returned an error: $(printf %s "$body" | head -c 120)"; exit 77;; esac + printf %s "$body" | python3 -c " +import json,sys,time +d=json.load(sys.stdin); c=(d.get(\"usage\") or {}).get(\"completion_tokens\") or 0 +print(f\"{c} tokens\")" >/dev/null 2>&1 + t0=$(date +%s.%N); body=$(curl -s -m 300 "http://127.0.0.1:$3/v1/chat/completions" -H "content-type: application/json" \ + -d "{\"model\":\"p\",\"messages\":[{\"role\":\"user\",\"content\":\"Write a short paragraph about rivers.\"}],\"max_tokens\":64,\"temperature\":0}"); t1=$(date +%s.%N) + case "$body" in *'"error"'*) echo "SKIPPED: server returned an error on the measured run"; exit 77;; esac + printf %s "$body" | python3 -c " +import json,sys +w=float(sys.argv[1]); d=json.load(sys.stdin); c=(d.get(\"usage\") or {}).get(\"completion_tokens\") or 0 +print(f\"{c/w:.2f} tokens per second\" if c>0 and w>0 else \"\")" "$(echo "$t1 - $t0" | bc)" +' _ "$BIN" "$MODEL" "$PORT" > "$LOG" 2>&1 +rc=$? + +if [ $rc -eq $SKIP ] || ! grep -q "tokens per second" "$LOG" 2>/dev/null; then + echo "SKIPPED: token_rate_floor device row did NOT run -- $(tail -1 "$LOG" 2>/dev/null | head -c 120)"; exit $SKIP +fi + +# The rate goes through the tool, which owns the judging, the floor and the subject line. +exec python3 "$(dirname "$0")/token_rate_floor.py" --log "$LOG" --floor "$FLOOR" \ + --subject "1bit serve $(basename "$MODEL") ctx2048"