diff --git a/CMakeLists.txt b/CMakeLists.txt index 050a9c8c1..2bf1f0d6f 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -551,21 +551,6 @@ if(ONEBIT_HRX) # hrx skips without a gfx1151 (ONEBIT_HRX_E2E_TARGETS) set_tests_properties(serve_e2e_${_dev} PROPERTIES SKIP_RETURN_CODE 77) endforeach() - # engine#1779 fault 1 (FLM 1.0.12: "a server that stopped accepting connections after clients hung up"): a - # client that disconnects mid-response, repeated, then a fresh client must still be served. The assertion is - # about the LISTENER, so an interrupted request is allowed to get nothing. `cpu` on purpose: the fault is - # server logic, and the issue's rule is that these four are tested without a device where possible. - add_test(NAME serve_client_hangup_cpu - COMMAND ${CMAKE_SOURCE_DIR}/tests/serve_client_hangup_test.sh $ - ${ONEBIT_SERVE_TEST_GGUF} cpu) - set_tests_properties(serve_client_hangup_cpu PROPERTIES SKIP_RETURN_CODE 77) - # engine#1779 fault 2 (FLM 1.0.12: "Qwen3.5 answering `//////` after hitting the context limit"): a prompt that - # fills the context is served, the next turn cannot fit, and the answer must be a refusal or a truncated-but-clean - # reply - never one character repeated. Also `cpu`: the fault is in the request path's clamp, not in a kernel. - add_test(NAME serve_context_limit_cpu - COMMAND ${CMAKE_SOURCE_DIR}/tests/serve_context_limit_test.sh $ - ${ONEBIT_SERVE_TEST_GGUF} cpu) - set_tests_properties(serve_context_limit_cpu PROPERTIES SKIP_RETURN_CODE 77) # Placement must not change the answer (engine #674): the same greedy prompt through the CPU, through the device # with every layer assigned to the CPU, and fully offloaded, compared byte for byte. add_test(NAME hrx_cpu_weight_parity @@ -581,8 +566,40 @@ if(ONEBIT_HRX) COMMAND ${CMAKE_SOURCE_DIR}/tests/hybrid_kvshare_e2e.sh "${ONEBIT_HRX_SERVER}" "${ONEBIT_SERVE_TEST_GGUF}" "${ONEBIT_KVSHARE_PREFILL}") set_tests_properties(hybrid_kvshare_e2e PROPERTIES SKIP_RETURN_CODE 77) endif() + endif() + # engine#1779: these two are registered OUTSIDE both `if(ONEBIT_SERVE_TEST_GGUF)` and `if(ONEBIT_HRX)` on purpose. + # Inside the GGUF guard they did not exist in CI at all (a configure-time cache path CI never sets: the build job + # runs 164 tests and none of the serve_* family is among them, so a gap in the family looked like coverage), and + # inside the HRX guard they would only exist for an HRX build while ci.yml configures without ONEBIT_HRX. They skip + # (77) when the model path is empty, which is a visible skip in the CI table, not an absent test. `onebit` is an + # unconditional target (CMakeLists.txt:97), so $ resolves in any configuration. + # engine#1779 fault 1 (FLM 1.0.12: "a server that stopped accepting connections after clients hung up"): a + # client that disconnects mid-response, repeated, then a fresh client must still be served. The assertion is + # about the LISTENER, so an interrupted request is allowed to get nothing. `cpu` on purpose: the fault is + # server logic, and the issue's rule is that these four are tested without a device where possible. + add_test(NAME serve_client_hangup_cpu + COMMAND ${CMAKE_SOURCE_DIR}/tests/serve_client_hangup_test.sh $ + "${ONEBIT_SERVE_TEST_GGUF}" cpu) + set_tests_properties(serve_client_hangup_cpu PROPERTIES SKIP_RETURN_CODE 77 RESOURCE_LOCK route_ports) + # engine#1779 fault 2 (FLM 1.0.12: "Qwen3.5 answering `//////` after hitting the context limit"): a prompt that + # fills the context is served, the next turn cannot fit, and the answer must be a refusal or a truncated-but-clean + # reply - never one character repeated. Also `cpu`: the fault is in the request path's clamp, not in a kernel. + add_test(NAME serve_context_limit_cpu + COMMAND ${CMAKE_SOURCE_DIR}/tests/serve_context_limit_test.sh $ + "${ONEBIT_SERVE_TEST_GGUF}" cpu) + set_tests_properties(serve_context_limit_cpu PROPERTIES SKIP_RETURN_CODE 77 RESOURCE_LOCK route_ports) +# engine#1779 fault 3 ("wrong vectors from embed-gemma"): tested on the route where OUR code computes the vector - +# `serve.cpp:2264 if (embed_serve::container(model))` serves a container directory in-process - not on the +# `rag_launch()` proxy, where the numbers come from a delegated llama-server. Same registration rule as the two +# above (outside both guards, skip 77 when unset); ONEBIT_EMBED_TEST_CONTAINER names a directory holding model.embed. +set(ONEBIT_EMBED_TEST_CONTAINER "" CACHE PATH "1-bit encoder container (.embed dir) the embed-vector test loads") +add_test(NAME serve_embed_vectors_cpu + COMMAND ${CMAKE_SOURCE_DIR}/tests/serve_embed_vectors_test.sh $ + "${ONEBIT_EMBED_TEST_CONTAINER}" cpu) +set_tests_properties(serve_embed_vectors_cpu PROPERTIES SKIP_RETURN_CODE 77 RESOURCE_LOCK route_ports) + # ── DwarfStar is gone (owner 2026-11-01, removed 2026-11-02) ─────────────── # `--device ds4` ran DwarfStar (third_party/ds4: antirez's CUDA kernels behind a HIP shim plus # hipBLAS), and its ds4-server was the engine's only 1BP reader. It is out of the shipped builds @@ -1315,6 +1332,20 @@ add_executable(npu_design_grid_test tests/npu_design_grid_test.c npu/forward/src npu/forward/src/q4nx_pack_blocked.cpp npu/forward/src/q4nx_head_image.c) target_include_directories(npu_design_grid_test PRIVATE npu/forward/include) target_link_libraries(npu_design_grid_test PRIVATE m) +# engine#1779 fault 4 ("Gemma 4 tool-call parsing"): the answer parser is header-only (app/npu_answer_parser.h), so +# this is a PURE unit test - no server, no model, no device - of the contract that header states: a template's own +# {"name":...,"arguments":{...}} becomes a structured call with the arguments re-dumped as JSON +# text; a call-only reply reports content_null; WITHOUT tools declared the marker stays text in content; and a +# malformed call body is never dropped. The issue asked for name, arguments and a non-tool reply that must not be +# parsed as one; the last two arms are the loss-proofing the header promises at :193 and :216-221. +add_executable(npu_answer_parser_test tests/npu_answer_parser_test.cpp) +target_include_directories(npu_answer_parser_test PRIVATE app) +target_link_libraries(npu_answer_parser_test PRIVATE nlohmann_json::nlohmann_json) +add_test(NAME npu_answer_parser COMMAND npu_answer_parser_test) +# direction B (tools/ci/ci_targets_check.py, engine#1595): a target no ci.yml line names is "never built", and its test +# then reports "Not Run" instead of running. `ci.yml` is a workflow file no lane can patch, so the documented hook is the +# one this tree already uses for onebit_npu_forward_syntax: add_dependencies(onebit ), which the checker follows. +add_dependencies(onebit npu_answer_parser_test) # engine#1781's oracle (merged in #1813) arrived with its test file but no registration, so the comparator behind # NPU_BATCH_CHECK -- "every drafted position's logits, bitwise" instead of one argmax at the last row -- was neither # built nor run while every job stayed green. Header-only, so the test needs no link libraries; host-only, so it needs no diff --git a/app/embed_serve.cpp b/app/embed_serve.cpp index 410841509..3ac605026 100644 --- a/app/embed_serve.cpp +++ b/app/embed_serve.cpp @@ -184,7 +184,11 @@ int serve_only(const std::string& model_dir, const std::string& host, int port, }; srv.Post("/v1/embeddings", handler); srv.Post("/embeddings", handler); - if (!srv.bind_to_port(host, port)) { + // bind_to_port() answers bool; the port comes back from bind_to_any_port(), which is the same bind_internal(host, 0) + // with the chosen port returned. Same idiom as app/serve.cpp:2023 and :2694 - a caller that asked for port 0 has no + // other way to learn which one it got. + const int bound_port = port == 0 ? srv.bind_to_any_port(host) : (srv.bind_to_port(host, port) ? port : -1); + if (bound_port < 0) { std::fprintf(stderr, "1bit serve: cannot listen on %s:%d\n", host.c_str(), port); return 2; } @@ -195,6 +199,11 @@ int serve_only(const std::string& model_dir, const std::string& host, int port, std::thread listener([&] { srv.listen_after_bind(); }); std::fprintf(stderr, "1bit serve: %s on %s embedding (%s), in-process: /v1/embeddings, dim %d, %d-token cap\n", id.c_str(), backend.device().c_str(), model_dir.c_str(), backend.dim(), backend.max_tokens()); + // engine#1779: the readiness line every other serve route prints, and the one `serve_port_from_log()` + // (tests/serve_port.sh) reads back. The in-process embedding route returned before either of the other two + // "ready on" sites (serve.cpp:2050, :2765), so a caller that asked for --port 0 could not discover the port it was + // listening on - which is what engine #1485 fixed for the other routes and left true only for them. + std::fprintf(stderr, "1bit serve: ready on http://%s:%d\n", host.c_str(), bound_port); listener.join(); return 0; } diff --git a/tests/npu_answer_parser_test.cpp b/tests/npu_answer_parser_test.cpp new file mode 100644 index 000000000..f79a067dc --- /dev/null +++ b/tests/npu_answer_parser_test.cpp @@ -0,0 +1,134 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// Copyright 2026 The 1bit-MONSTER Authors. +// SPDX-License-Identifier: Apache-2.0 +// +// npu_answer_parser_test -- engine#1779 fault 4: "Gemma 4 tool-call parsing". +// +// The issue's test: "a template's own tool-call emission parsed into a structured call: name, arguments, and a +// non-tool reply that must not be parsed as one". The parser is header-only (app/npu_answer_parser.h), so this is a +// pure unit test: no server, no model, no device, and it runs on any box. +// +// The arms below are the contract that header states rather than just the happy path: +// * a `{"name":…,"arguments":{…}}` block becomes a ToolCall with the arguments re-dumped as +// JSON text (footer note at :49 says `function.arguments` is a string in OpenAI's shape); +// * a call-only reply sets content_null, an answer with prose does not (:279-281); +// * WITHOUT tools declared the marker is text and stays in content, marker and all (:193 "Without tools the tag is +// just text: keep it in content, marker and all") - that is the strongest form of "a non-tool reply must not be +// parsed as one"; +// * a malformed call body is not dropped: it goes back into content with its markup (:216-221), because what the +// model wrote must never be lost. + +#include +#include + +#include "npu_answer_parser.h" + +namespace { + +int failures = 0; + +void check(bool ok, const std::string& what) { + std::printf("%s %s\n", ok ? "ok " : "FAIL", what.c_str()); + if (!ok) failures++; +} + +bool contains(const std::string& hay, const std::string& needle) { + return hay.find(needle) != std::string::npos; +} + +} // namespace + +int main() { + using namespace onebit::npu; + + // 1. A template's own emission, with reasoning and prose around it. + { + const auto a = parse_answer( + "I should check the weather first.Let me look.{\"name\":\"get_weather\"," + "\"arguments\":{\"city\":\"Paris\"}}", + AnswerFamily::qwen3, true); + check(a.calls.size() == 1, "a tool_call block becomes exactly one call"); + if (a.calls.size() == 1) { + check(a.calls[0].name == "get_weather", "the call keeps its name"); + check(contains(a.calls[0].arguments, "Paris"), "the call keeps its arguments as JSON text"); + check(a.calls[0].arguments.size() > 2 && a.calls[0].arguments.front() == '{', + "arguments is a JSON object string, not a quoted blob"); + check(!a.calls[0].id.empty(), "the call carries an id"); + } + check(a.reasoning == "I should check the weather first.", " goes to reasoning"); + check(a.content == "Let me look.", "prose outside the markup goes to content"); + check(!a.content_null, "a reply with prose does not report null content"); + } + + // 2. Call-only: content: null, not "" (the issue's second rule). + { + const auto a = parse_answer("{\"name\":\"ping\",\"arguments\":{}}", AnswerFamily::qwen3, true); + check(a.calls.size() == 1, "a call-only reply still yields the call"); + check(a.content.empty(), "a call-only reply has empty content"); + check(a.content_null, "a call-only reply reports content_null"); + } + + // 3. A non-tool reply, tools declared: prose must not become a call. + { + const auto a = parse_answer("The weather in Paris is mild and dry today.", AnswerFamily::qwen3, true); + check(a.calls.empty(), "a plain answer produces no calls"); + check(a.content == "The weather in Paris is mild and dry today.", "a plain answer is all content"); + } + + // 4. The marker with NO tools declared is text, marker and all - the strongest "not parsed as one" case. + { + const std::string text = "Use {\"name\":\"x\",\"arguments\":{}} to emit a call."; + const auto a = parse_answer(text, AnswerFamily::qwen3, false); + check(a.calls.empty(), "without tools declared, tool_call markup is not parsed into a call"); + check(contains(a.content, ""), "without tools, the marker stays visible in content"); + check(contains(a.content, "to emit a call."), "without tools, the rest of the text is kept"); + } + + // 5. Malformed call body: never dropped, goes back into content with its markup. + { + const auto a = parse_answer("xnot json at all", AnswerFamily::qwen3, true); + check(a.calls.empty(), "a malformed call body produces no call"); + check(contains(a.content, "") && contains(a.content, "not json at all"), + "a malformed call body is kept in content, markup and all (nothing is lost)"); + } + + // 6. Two calls in one reply, in order. + { + const auto a = parse_answer( + "{\"name\":\"first\",\"arguments\":{\"n\":1}}" + "{\"name\":\"second\",\"arguments\":{\"n\":2}}", + AnswerFamily::qwen3, true); + check(a.calls.size() == 2, "two blocks become two calls"); + if (a.calls.size() == 2) { + check(a.calls[0].name == "first" && a.calls[1].name == "second", "calls keep their order"); + check(a.calls[0].id != a.calls[1].id, "each call gets its own id"); + } + check(a.content_null, "a two-call reply with no prose reports content_null"); + } + + // 7. The family tag set is complete for the one family the header declares, so an unsupported family cannot be + // silently parsed as qwen3. + { + const auto tags = onebit::npu::answer_detail::tags_of(AnswerFamily::qwen3); + check(std::string(tags.call_open) == "" && std::string(tags.call_close) == "", + "the qwen3 family's call tags are the ones this test feeds"); + check(std::string(tags.think_open) == "" && std::string(tags.think_close) == "", + "the qwen3 family's reasoning tags are the ones this test feeds"); + } + + std::printf("%s\n", failures ? "FAILED" : "all cases ok"); + return failures ? 1 : 0; +} diff --git a/tests/serve_client_hangup_test.sh b/tests/serve_client_hangup_test.sh index 1c30887df..ae2671ec8 100755 --- a/tests/serve_client_hangup_test.sh +++ b/tests/serve_client_hangup_test.sh @@ -37,13 +37,14 @@ bin=${1:?usage: serve_client_hangup_test.sh path/to/1bit [round here=$(cd "$(dirname "$0")/.." && pwd) # shellcheck source=tests/serve_port.sh . "$(dirname "$0")/serve_port.sh" # serve_port_wait: read back the port serve bound (engine #1485) -model=${2:?} +model=${2:-} # empty in CI unless ONEBIT_SERVE_TEST_GGUF is set: that is a SKIP, not a failure device=${3:?} rounds=${4:-6} [ "$rounds" -ge 1 ] 2>/dev/null || rounds=6 shift 3 [ $# -gt 0 ] && [ "${1#-}" = "$1" ] 2>/dev/null && shift # the optional rounds argument, when given +[ -n "$model" ] || { echo "skip: no model given (set ONEBIT_SERVE_TEST_GGUF - engine#1779: this test is registered in CI and must show as a skip, not as an absent test)"; exit 77; } [ -x "$bin" ] || { echo "skip: no 1bit at $bin"; exit 77; } [ -f "$model" ] || { echo "skip: no model at $model"; exit 77; } command -v curl > /dev/null || { echo "skip: no curl"; exit 77; } diff --git a/tests/serve_context_limit_test.sh b/tests/serve_context_limit_test.sh index e18e4753e..edf98bae8 100755 --- a/tests/serve_context_limit_test.sh +++ b/tests/serve_context_limit_test.sh @@ -70,7 +70,12 @@ try: except Exception: text, finish = body.strip(), None core = text.strip() -if re.fullmatch(r"(.)\1{7,}", core, re.S): +packed = re.sub(r"\s+", "", core) +if len(packed) >= 6 and len(set(packed)) == 1: + # engine#1791's fix, which main's version did not carry: the reported symptom is "//////" (SIX), and the regex + # above needed eight, so a server answering exactly the reported string classified as 'clean'. One character + # repeated >= 6 (whitespace stripped, so "////// //////" is caught too) is the rule; six rather than four because + # a real completion can legitimately be a short repeated token. print("degenerate") elif core == "" and finish in ("length", None): print("clean-empty") @@ -96,6 +101,9 @@ self_test() { local tmp fails=0 got tmp=$(mktemp -d) || { echo "FAIL self_test: mktemp -d"; return 1; } printf '%s' '{"choices":[{"message":{"content":"////////////////"},"finish_reason":"stop"}]}' > "$tmp/degenerate.json" + # the EXACT reported symptom is six slashes, not sixteen: a detector with an eight-character floor passed this string + # (engine#1791's first version) and only a six-character case proves the fix covers what was filed (engine#1779). + printf '%s' '{"choices":[{"message":{"content":"//////"},"finish_reason":"stop"}]}' > "$tmp/degenerate6.json" printf '%s' '{"choices":[{"message":{"content":"The answer is 4."},"finish_reason":"stop"}]}' > "$tmp/clean.json" printf '%s' '{"choices":[{"message":{"content":""},"finish_reason":"length"}]}' > "$tmp/empty.json" printf '%s' '{"error":{"message":"context length exceeded"}}' > "$tmp/err.json" @@ -130,6 +138,8 @@ self_test() { expect_pass "an empty answer with finish_reason length" clean-empty yes # and the fault itself still has to fail expect "the reported degenerate answer" 200 "$tmp/degenerate.json" degenerate + expect "the reported answer at its real length (six)" 200 "$tmp/degenerate6.json" degenerate + expect_pass "the reported answer at its real length (six)" degenerate no expect_pass "the reported degenerate answer" degenerate no # neither outcome: say so rather than pass expect "a redirect nobody follows" 302 "$tmp/none.body" unclassified-302 @@ -148,7 +158,7 @@ if [ "${1:-}" = "--self-test" ]; then self_test exit $? fi -model=${2:?} +model=${2:-} # empty in CI unless ONEBIT_SERVE_TEST_GGUF is set: that is a SKIP, not a failure device=${3:?} ctx=${4:-512} [ "$ctx" -ge 64 ] 2>/dev/null || ctx=512 @@ -156,6 +166,7 @@ shift 3 if [ $# -gt 0 ] && [ "${1#-}" = "$1" ]; then shift; fi # the optional ctx argument, when given [ -x "$bin" ] || { echo "skip: no 1bit at $bin"; exit 77; } +[ -n "$model" ] || { echo "skip: no model given (set ONEBIT_SERVE_TEST_GGUF)"; exit 77; } [ -f "$model" ] || { echo "skip: no model at $model"; exit 77; } command -v curl > /dev/null || { echo "skip: no curl"; exit 77; } command -v python3 > /dev/null || { echo "skip: no python3"; exit 77; } diff --git a/tests/serve_embed_vectors_test.sh b/tests/serve_embed_vectors_test.sh new file mode 100755 index 000000000..335d8b31f --- /dev/null +++ b/tests/serve_embed_vectors_test.sh @@ -0,0 +1,181 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# Copyright 2026 The 1bit-MONSTER Authors. +# SPDX-License-Identifier: Apache-2.0 +# +# serve_embed_vectors_test -- engine#1779 fault 3: "wrong vectors from embed-gemma". +# +# WHERE THIS MUST BE TESTED, corrected after a peer reading: `app/serve.cpp` has TWO embedding routes. +# serve.cpp:2264 if (embed_serve::container(model)) -> IN-PROCESS, our code computes the vector +# serve.cpp:2275 rag_launch(o, model, role, free_port()) -> a PROXY to a llama-server for a GGUF model +# A vector test on the proxy route would only re-test llama.cpp. This one takes the CONTAINER route - a directory +# holding `model.embed` (`embed_serve.cpp:39-42`) - where our model decides the numbers, served at POST +# /v1/embeddings (`:185`). The issue's test is: the same text embedded twice must be identical, and a known-answer +# pair (a paraphrase closer than an unrelated sentence) must hold. +# +# The metric layer self-checks first, because the two ways this test could lie are both in the measurement: cosine +# that cannot separate anything, or an "identical" compare that passes on unrelated vectors. +# +# usage: tests/serve_embed_vectors_test.sh path/to/1bit [extra serve args] +# exits 0 when the vectors are deterministic and ordered, 1 when not, 77 when it cannot run here. +set -uo pipefail + +bin=${1:?usage: serve_embed_vectors_test.sh path/to/1bit [args]} +here=$(cd "$(dirname "$0")/.." && pwd) +# shellcheck source=tests/serve_port.sh +. "$(dirname "$0")/serve_port.sh" +container=${2:-} # empty in CI unless a container is configured: that is a SKIP, not a failure +device=${3:-cpu} +shift 3 2> /dev/null || shift $# + +if [ -z "${ONEBIT_EMBED_TEST_URL:-}" ]; then + [ -n "$container" ] || { echo "skip: no encoder container given (engine#1779 fault 3 needs one; the in-process container route is app/embed_serve.cpp)"; exit 77; } + [ -x "$bin" ] || { echo "skip: no 1bit at $bin"; exit 77; } + [ -f "$container/model.embed" ] || { echo "skip: no model.embed in $container (a container is a directory holding one, embed_serve.cpp:39-42)"; exit 77; } +fi +command -v curl > /dev/null || { echo "skip: no curl"; exit 77; } +command -v python3 > /dev/null || { echo "skip: no python3"; exit 77; } + +if [ "$device" = hrx ]; then + have=$("$here/scripts/gpu-targets.sh" 2>/dev/null | tr '\n' ' ') + want=${ONEBIT_HRX_E2E_TARGETS:-gfx1151} + found=0 + for t in $want; do case " $have " in *" $t "*) found=1 ;; esac; done + [ $found = 1 ] || { echo "skip: no $want GPU for HRX on this machine (GPUs: ${have:-none})"; exit 77; } +fi + +log=$(mktemp) +srv= +cleanup() { + if [ -n "$srv" ] && kill -0 "$srv" 2> /dev/null; then + kill "$srv" 2> /dev/null + for _ in $(seq 1 50); do kill -0 "$srv" 2> /dev/null || break; sleep 0.1; done + kill -9 "$srv" 2> /dev/null + fi + rm -f "$log" "$log".* +} +trap cleanup EXIT INT TERM + +# ---- the metric layer, self-checked before it is trusted ------------------------------------------------ +vectors=$(python3 - <<'PY' +import json, math, sys + +def cos(a, b): + dot = sum(x * y for x, y in zip(a, b)) + na = math.sqrt(sum(x * x for x in a)); nb = math.sqrt(sum(y * y for y in b)) + return dot / (na * nb) if na and nb else 0.0 + +def identical(a, b): + return len(a) == len(b) and all(x == y for x, y in zip(a, b)) + +# self-check 1: cos must separate a paraphrase-ish pair from an unrelated one on known vectors +a, para, unrel = [1.0, 0.0, 0.0], [0.9, 0.1, 0.0], [0.0, 1.0, 0.0] +if not (cos(a, para) > 0.5 and cos(a, unrel) < 0.1 and cos(a, a) == 1.0): + print("SELFCHECK_FAIL cosine cannot separate a paraphrase from an unrelated sentence"); raise SystemExit(1) +# self-check 2: identical() must not call two different vectors equal +if identical(a, para) or not identical(a, list(a)): + print("SELFCHECK_FAIL identical() is wrong"); raise SystemExit(1) +# self-check 3: a constant-output server must be caught - that is what "two different texts differ" is for +if identical([0.5, 0.5], [0.5, 0.5]) is not True: + print("SELFCHECK_FAIL constant vectors should compare identical, which is why the distinctness arm exists"); raise SystemExit(1) +print("SELFCHECK_OK") +PY +) || { echo "FAIL serve_embed_vectors: the metric layer does not work: $vectors"; exit 1; } +echo "metric self-check: $vectors" + +# ONEBIT_EMBED_TEST_URL points the harness at an already-running /v1/embeddings service instead of spawning serve. +# It exists so this test's own measurement can be exercised against a stub - the discipline the fault-2 detector +# now follows after its eight-vs-six bug: an instrument that has never been driven is not evidence. +if [ -n "${ONEBIT_EMBED_TEST_URL:-}" ]; then + url=${ONEBIT_EMBED_TEST_URL%/} + echo "using ONEBIT_EMBED_TEST_URL=$url (no serve spawned: the harness is being tested, not the model)" +else + "$bin" serve -m "$container" --device "$device" --host 127.0.0.1 --port 0 "$@" > "$log" 2>&1 & + srv=$! + if ! port=$(serve_port_wait_any "$log" "$srv" 24000); then + echo "skip: serve did not report a port in 240 s (see below)"; tail -20 "$log"; exit 77 + fi + url="http://127.0.0.1:$port" + echo "serve on $url, device=$device, container=$(basename "$container")" +fi + +post() { # -> "\t" + python3 -c 'import json,sys; print(json.dumps({"input": sys.argv[1]}))' "$1" > "$log.req.json" + local code + code=$(curl -s -m 120 -o "$log.resp.json" -w '%{http_code}' -H 'Content-Type: application/json' \ + -d @"$log.req.json" "$url/v1/embeddings" 2> /dev/null) + cp "$log.resp.json" "$2" + printf '%s' "$code" +} + +t1="A man is playing a guitar." +t2="A person is playing a musical instrument." # paraphrase of t1 +t3="The stock market fell sharply today." # unrelated to t1 + +c1a=$(post "$t1" "$log.v1a") +c1b=$(post "$t1" "$log.v1b") +c2=$(post "$t2" "$log.v2") +c3=$(post "$t3" "$log.v3") +echo "requests: t1 x2 -> $c1a/$c1b, paraphrase -> $c2, unrelated -> $c3" + +verdict=$(python3 - "$log.v1a" "$log.v1b" "$log.v2" "$log.v3" "$c1a" <<'PY' +import json, math, sys +paths, codes = sys.argv[1:5], sys.argv[5] + +def load(p): + try: + j = json.load(open(p)) + return [float(x) for x in j["data"][0]["embedding"]] + except Exception: + return None + +def cos(a, b): + dot = sum(x * y for x, y in zip(a, b)) + na = math.sqrt(sum(x * x for x in a)); nb = math.sqrt(sum(y * y for y in b)) + return dot / (na * nb) if na and nb else 0.0 + +v = [load(p) for p in paths] +if any(x is None for x in v): + print("no-vectors"); raise SystemExit +if len({len(x) for x in v}) != 1: + print("dim-mismatch"); raise SystemExit +a, b, para, unrel = v +if a != b: + print("nondeterministic"); raise SystemExit +if a == para or a == unrel: + print("constant-output"); raise SystemExit # the arm that catches a server returning one vector for everything +c_para, c_unrel = cos(a, para), cos(a, unrel) +# the known-answer ordering the issue asks for, with a margin so noise cannot flip it +print(f"ordered para={c_para:.4f} unrel={c_unrel:.4f}" if c_para > c_unrel + 0.02 else f"unordered para={c_para:.4f} unrel={c_unrel:.4f}") +PY +) +echo "verdict: $verdict" + +case "$verdict" in + ordered\ *) + echo "PASS serve_embed_vectors: same text -> same vector, and a paraphrase is closer than an unrelated sentence" + exit 0 ;; + nondeterministic) + echo "FAIL serve_embed_vectors: the same text embedded twice gave different vectors"; exit 1 ;; + constant-output) + echo "FAIL serve_embed_vectors: every text returned the same vector (a server that ignores its input)"; exit 1 ;; + unordered\ *) + echo "FAIL serve_embed_vectors: a paraphrase was not closer than an unrelated sentence - $verdict"; exit 1 ;; + no-vectors|dim-mismatch) + echo "FAIL serve_embed_vectors: the response is not a usable embedding ($verdict)"; head -c 200 "$log.v1a"; exit 1 ;; + *) + echo "FAIL serve_embed_vectors: could not classify ($verdict)"; exit 1 ;; +esac