diff --git a/CMakeLists.txt b/CMakeLists.txt index d885125d0..d01ac766f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -269,6 +269,12 @@ include(cmake/tests-gpu-hold.cmake) # it feeds the parsers the exact lines `1bit serve` and `llama-server` print, because a read-back that silently returns # empty turns into "the server did not come up" and is blamed on whatever the row under test checks. add_test(NAME serve_port COMMAND bash ${CMAKE_SOURCE_DIR}/tests/serve_port_test.sh) +# engine#1844: the serve_context_limit row's classifier used to accept anything that was not 2xx, so a server that DIED +# mid-request (curl 000) or answered 5xx passed as "a refusal". The classifier and its pass set are pure functions of +# (http code, body), so their cases -- especially the ones that must reject -- run here with no model, no server and no +# device. Deliberately OUTSIDE if(ONEBIT_SERVE_TEST_GGUF): that row is device-gated, and the whole point is that the +# classification is checked in every configure, including the ones that never load a model. +add_test(NAME serve_context_limit_classify COMMAND bash ${CMAKE_SOURCE_DIR}/tests/serve_context_limit_test.sh --self-test) # engine#1594: every target ci.yml's `cmake --build --target …` list names must exist in the configuration ci.yml # creates. #1583 added one name for a target defined inside if(ONEBIT_NPU), which CI does not configure -- and ninja # rejects the whole command line before building anything, so `onebit` stopped being compiled for every PR. Reads the diff --git a/tests/serve_context_limit_test.sh b/tests/serve_context_limit_test.sh index 0722cdecb..e18e4753e 100755 --- a/tests/serve_context_limit_test.sh +++ b/tests/serve_context_limit_test.sh @@ -22,9 +22,15 @@ # per ~/briefs/TNPU-ALL.md §9: a request whose prompt fills the context, then one more turn, and the response must be # **a refusal or a truncated-but-clean answer, never a degenerate repeat of one token**. # -# So the assertion is on the SHAPE of the answer, not on its wording: a 4xx/5xx with an error body passes (that is a -# refusal), a 200 with ordinary text passes (truncated-but-clean), an empty 200 with finish_reason "length" passes, and -# a 200 whose content is one character repeated — `//////` being the reported instance — FAILS. +# So the assertion is on the SHAPE of the answer, not on its wording: a 4xx passes (the server refused), a 200 with +# ordinary text passes (truncated-but-clean), an empty 200 with finish_reason "length" passes, and a 200 whose content +# is one character repeated (the reported instance was `//////`) FAILS. +# +# engine#1844: `000` (no response at all, i.e. the server died mid-request) and `5xx` are NOT refusals -- they are the +# server failing the very property this row exists for, so they have their own verdicts and FAIL. The first version +# accepted anything that was not 2xx, so a crash passed. The classifier and its pass set are exercised without a +# device by `--self-test`, which is also a ctest row: the bug was never the one wrong line, it was that nothing ever +# ran the classifier on the codes it has to reject. # # Device-free by construction: `--device cpu`, per the issue's rule for these four. # @@ -36,6 +42,112 @@ bin=${1:?usage: serve_context_limit_test.sh path/to/1bit [ctx] here=$(cd "$(dirname "$0")/.." && pwd) # shellcheck source=tests/serve_port.sh . "$(dirname "$0")/serve_port.sh" +classify_over_limit() { # -> refusal|refusal-empty|died|server-error|degenerate|clean-empty|clean + python3 - "$1" "$2" <<'PY' +import json, sys, re +code, path = sys.argv[1], sys.argv[2] +try: + body = open(path, "rb").read().decode("utf-8", "replace") +except OSError: + body = "" +if code in ("", "000"): + print("died") # curl's 000: no response at all, so the server went away mid-request + raise SystemExit +if code.startswith("5"): + print("server-error") # it answered, but with its own failure -- not the client's + raise SystemExit +if code.startswith("4"): + print("refusal" if body.strip() else "refusal-empty") + raise SystemExit +if not code.startswith("2"): + print("unclassified-%s" % code) # e.g. a 3xx redirect: neither outcome, so say so rather than pass + raise SystemExit +try: + j = json.loads(body) + choice = (j.get("choices") or [{}])[0] + text = (choice.get("message") or {}).get("content") or "" + finish = choice.get("finish_reason") +except Exception: + text, finish = body.strip(), None +core = text.strip() +if re.fullmatch(r"(.)\1{7,}", core, re.S): + print("degenerate") +elif core == "" and finish in ("length", None): + print("clean-empty") +elif core == "": + print("clean-empty") +else: + print("clean") +PY +} + +# The accepted outcomes, in one place, so the self-test asserts the same set the run uses instead of a copied `case`. +verdict_is_pass() { # -> 0 when that verdict is an acceptable over-limit answer + case "$1" in + refusal|refusal-empty|clean|clean-empty) return 0 ;; + *) return 1 ;; + esac +} + +# engine#1844: the classifier's job is to REJECT codes, so the cases that must fail are the ones worth running. Device- +# free by construction (no model, no server): this is a ctest row of its own, and it fails if either class above is put +# back into the "refusal" bucket. +self_test() { + local tmp fails=0 got + tmp=$(mktemp -d) || { echo "FAIL self_test: mktemp -d"; return 1; } + printf '%s' '{"choices":[{"message":{"content":"////////////////"},"finish_reason":"stop"}]}' > "$tmp/degenerate.json" + printf '%s' '{"choices":[{"message":{"content":"The answer is 4."},"finish_reason":"stop"}]}' > "$tmp/clean.json" + printf '%s' '{"choices":[{"message":{"content":""},"finish_reason":"length"}]}' > "$tmp/empty.json" + printf '%s' '{"error":{"message":"context length exceeded"}}' > "$tmp/err.json" + : > "$tmp/none.body" + + expect() { #