diff --git a/CMakeLists.txt b/CMakeLists.txt index bcf913af0..17dd2c52d 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -533,7 +533,8 @@ file(WRITE ${CMAKE_BINARY_DIR}/generated/route_policy_default.h "// generated from config/route-policy.json by CMakeLists.txt\n#pragma once\n" "static const char kDefaultRoutePolicyJson[] = R\"ONEBITPOLICY(${ONEBIT_ROUTE_POLICY_JSON})ONEBITPOLICY\";\n") set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${CMAKE_SOURCE_DIR}/config/route-policy.json) -add_library(onebit_laya STATIC laya/scorer.cpp laya/safetensors.cpp laya/route.cpp laya/policy.cpp laya/encoder.cpp) +add_library(onebit_laya STATIC laya/scorer.cpp laya/safetensors.cpp laya/route.cpp laya/policy.cpp laya/encoder.cpp + laya/models.cpp) target_include_directories(onebit_laya PUBLIC laya PRIVATE ${CMAKE_BINARY_DIR}/generated) target_link_libraries(onebit_laya PUBLIC onebit_npu nlohmann_json::nlohmann_json) # The engine binary gets `1bit route` and per-request routing through the scorer. @@ -553,6 +554,12 @@ add_executable(laya_gate tests/laya_gate.cpp) add_executable(laya_encoder_test tests/laya_encoder_test.cpp) target_link_libraries(laya_encoder_test PRIVATE onebit_laya) add_test(NAME laya_encoder COMMAND laya_encoder_test) +add_executable(laya_models_test tests/laya_models_test.cpp) +target_link_libraries(laya_models_test PRIVATE onebit_laya) +add_test(NAME laya_models COMMAND laya_models_test ${CMAKE_SOURCE_DIR}/config/laya-models-game.json) +# CI builds `--target onebit` plus a fixed list in .github/workflows/ci.yml; building this check with onebit +# keeps the laya_models ctest from reporting "Not Run" without a workflow change. +add_dependencies(onebit laya_models_test) # Crafted Laya checkpoints the reader and the scorer must refuse (written at run time); runs in CI. add_executable(laya_bounds_test tests/laya_bounds_test.cpp) target_link_libraries(laya_bounds_test PRIVATE onebit_laya nlohmann_json::nlohmann_json) @@ -574,6 +581,8 @@ endif() set(ONEBIT_LAYA_ROUTE_MODEL "" CACHE PATH "Laya checkpoint dir for the per-request routing e2e test") if(ONEBIT_LAYA_ROUTE_MODEL) add_test(NAME laya_route_e2e COMMAND ${CMAKE_SOURCE_DIR}/tests/laya_route_e2e.sh $ ${ONEBIT_LAYA_ROUTE_MODEL}) + # --laya-models: model auto picked per request among the table's rows (docs/laya-models.md) + add_test(NAME laya_models_e2e COMMAND ${CMAKE_SOURCE_DIR}/tests/laya_models_e2e.sh $ ${ONEBIT_LAYA_ROUTE_MODEL}) # `1bit route --serve`, the resident form the fork calls per request (docs/laya-first.md §8). add_test(NAME laya_route_serve COMMAND ${CMAKE_SOURCE_DIR}/tests/laya_route_serve.sh $ ${ONEBIT_LAYA_ROUTE_MODEL}) # One checkpoint resolution for serve --laya and 1bit route (app/laya_checkpoint.h). @@ -1242,7 +1251,7 @@ set(_onebit_route_tests laya_gguf_route mem_admit_route hrx_cwsr_route hrx_runtime_route hybrid_route hybrid_kvshare_route) # Tests that only exist for some configurations (a model was passed, a device is built in). -list(APPEND _onebit_route_tests hybrid_e2e laya_route_e2e pm_laya_route serve_e2e_cpu serve_e2e_hrx +list(APPEND _onebit_route_tests hybrid_e2e laya_route_e2e laya_models_e2e pm_laya_route serve_e2e_cpu serve_e2e_hrx hrx_cpu_weight_parity) foreach(_onebit_route_test IN LISTS _onebit_route_tests) if(TEST ${_onebit_route_test}) diff --git a/app/route_log.h b/app/route_log.h new file mode 100644 index 000000000..333f2457e --- /dev/null +++ b/app/route_log.h @@ -0,0 +1,192 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// app/route_log.h — what `1bit serve --laya-models` decided, request by request (docs/laya-models.md): +// the recent decisions for GET /v1/route/decisions, and the totals for GET /metrics. Every number is +// one serve measured (the router's time, the request's wall time) or one the backend reported in the +// reply (usage and timings); nothing is estimated. +#pragma once + +#include +#include +#include +#include +#include +#include +#include + +#include + +namespace onebit { + +struct RouteDecision { + long id = 0; + double t_unix = 0; // when the request arrived (seconds since the epoch) + std::string cls; // the class used (after the fallback); "pinned" when the request named a row + std::string laya_class; // what Laya answered ("" for pinned) + float confidence = 0; + std::string model, device; // the row that answered + double router_ms = 0; // the Laya decision (0 when pinned or cached) + bool cached = false; + bool stream = false; + int status = 0; // 0 for a stream (the status leaves after the handler) + double request_ms = 0; // serve's wall time for a non-streamed reply + long prompt_tokens = -1, completion_tokens = -1; // the reply's usage, -1 when absent + double prompt_tok_s = -1, decode_tok_s = -1; // the reply's timings, -1 when absent + + nlohmann::json to_json() const { + nlohmann::json j = {{"id", id}, {"t", t_unix}, {"class", cls}, + {"laya_class", laya_class}, {"confidence", confidence}, {"model", model}, + {"device", device}, {"router_ms", router_ms}, {"cached", cached}, + {"stream", stream}}; + if (!stream) { + j["status"] = status; + j["request_ms"] = request_ms; + } + if (prompt_tokens >= 0) j["prompt_tokens"] = prompt_tokens; + if (completion_tokens >= 0) j["completion_tokens"] = completion_tokens; + if (prompt_tok_s >= 0) j["prompt_tok_s"] = prompt_tok_s; + if (decode_tok_s >= 0) j["decode_tok_s"] = decode_tok_s; + return j; + } +}; + +// Reads usage and timings from a non-streamed reply body (llama-server and the NPU routes both send them). +inline void read_reply_stats(const std::string& body, RouteDecision& d) { + if (body.empty() || body.front() != '{') return; + const nlohmann::json out = nlohmann::json::parse(body, nullptr, false); + if (!out.is_object()) return; + if (out.contains("usage") && out["usage"].is_object()) { + const auto& u = out["usage"]; + if (u.contains("prompt_tokens") && u["prompt_tokens"].is_number()) d.prompt_tokens = u["prompt_tokens"].get(); + if (u.contains("completion_tokens") && u["completion_tokens"].is_number()) + d.completion_tokens = u["completion_tokens"].get(); + } + if (out.contains("timings") && out["timings"].is_object()) { + const auto& t = out["timings"]; + if (t.contains("prompt_per_second") && t["prompt_per_second"].is_number()) + d.prompt_tok_s = t["prompt_per_second"].get(); + if (t.contains("predicted_per_second") && t["predicted_per_second"].is_number()) + d.decode_tok_s = t["predicted_per_second"].get(); + } +} + +class RouteLog { +public: + explicit RouteLog(size_t keep = 512) : keep_(keep) {} + + long next_id() { + std::lock_guard l(mu_); + return ++last_id_; + } + + void add(const RouteDecision& d) { + std::lock_guard l(mu_); + recent_.push_back(d); + while (recent_.size() > keep_) recent_.pop_front(); + Totals& t = totals_[{d.cls, d.model, d.device}]; + ++t.decisions; + if (!d.cached && d.cls != "pinned") { + t.router_ms += d.router_ms; + ++t.routed; + } + if (!d.stream) { + t.request_ms += d.request_ms; + ++t.answered; + } + if (d.completion_tokens >= 0) t.completion_tokens += d.completion_tokens; + if (d.prompt_tokens >= 0) t.prompt_tokens += d.prompt_tokens; + if (d.decode_tok_s >= 0) last_decode_[{d.model, d.device}] = d.decode_tok_s; + } + + // the decisions after `since` (an id), oldest first, at most `limit` + nlohmann::json recent(long since, size_t limit) const { + std::lock_guard l(mu_); + nlohmann::json arr = nlohmann::json::array(); + size_t skip = 0; + size_t n = 0; + for (const auto& d : recent_) n += d.id > since; + if (n > limit) skip = n - limit; + for (const auto& d : recent_) { + if (d.id <= since) continue; + if (skip) { --skip; continue; } + arr.push_back(d.to_json()); + } + return {{"object", "list"}, {"last_id", last_id_}, {"data", arr}}; + } + + // Prometheus text format 0.0.4, appended to serve's own metrics + std::string render() const { + std::lock_guard l(mu_); + std::string out; + auto head = [&](const char* name, const char* type, const char* help) { + out += std::string("# HELP ") + name + " " + help + "\n# TYPE " + name + " " + type + "\n"; + }; + auto labels = [](const Key& k) { + return "{class=\"" + esc(std::get<0>(k)) + "\",model=\"" + esc(std::get<1>(k)) + "\",device=\"" + + esc(std::get<2>(k)) + "\"}"; + }; + head("onebit_route_decisions_total", "counter", "requests --laya-models routed, by class, model and device"); + for (const auto& [k, t] : totals_) out += "onebit_route_decisions_total" + labels(k) + " " + std::to_string(t.decisions) + "\n"; + head("onebit_route_router_ms_sum", "counter", "milliseconds spent in Laya decisions (not cached, not pinned)"); + for (const auto& [k, t] : totals_) out += "onebit_route_router_ms_sum" + labels(k) + " " + fmt(t.router_ms) + "\n"; + head("onebit_route_router_count", "counter", "Laya decisions taken (not cached, not pinned)"); + for (const auto& [k, t] : totals_) out += "onebit_route_router_count" + labels(k) + " " + std::to_string(t.routed) + "\n"; + head("onebit_route_request_ms_sum", "counter", "wall milliseconds of non-streamed routed requests"); + for (const auto& [k, t] : totals_) out += "onebit_route_request_ms_sum" + labels(k) + " " + fmt(t.request_ms) + "\n"; + head("onebit_route_request_count", "counter", "non-streamed routed requests answered"); + for (const auto& [k, t] : totals_) out += "onebit_route_request_count" + labels(k) + " " + std::to_string(t.answered) + "\n"; + head("onebit_route_completion_tokens_total", "counter", "completion tokens the backends reported (usage)"); + for (const auto& [k, t] : totals_) + out += "onebit_route_completion_tokens_total" + labels(k) + " " + std::to_string(t.completion_tokens) + "\n"; + head("onebit_route_prompt_tokens_total", "counter", "prompt tokens the backends reported (usage)"); + for (const auto& [k, t] : totals_) + out += "onebit_route_prompt_tokens_total" + labels(k) + " " + std::to_string(t.prompt_tokens) + "\n"; + head("onebit_route_last_decode_tok_s", "gauge", "decode tok/s of the last reply that reported timings, by model and device"); + for (const auto& [k, v] : last_decode_) + out += "onebit_route_last_decode_tok_s{model=\"" + esc(k.first) + "\",device=\"" + esc(k.second) + "\"} " + fmt(v) + "\n"; + return out; + } + +private: + using Key = std::tuple; + struct Totals { + long decisions = 0, routed = 0, answered = 0, completion_tokens = 0, prompt_tokens = 0; + double router_ms = 0, request_ms = 0; + }; + static std::string esc(const std::string& s) { + std::string o; + for (char c : s) { + if (c == '\\' || c == '"') { o += '\\'; o += c; } + else if (c == '\n') o += "\\n"; + else o += c; + } + return o; + } + static std::string fmt(double v) { + char b[64]; + std::snprintf(b, sizeof b, "%.6g", v); + return b; + } + + size_t keep_; + mutable std::mutex mu_; + long last_id_ = 0; + std::deque recent_; + std::map totals_; + std::map, double> last_decode_; +}; + +} // namespace onebit diff --git a/app/serve.cpp b/app/serve.cpp index c7110017e..ac4e2bf4d 100644 --- a/app/serve.cpp +++ b/app/serve.cpp @@ -74,6 +74,7 @@ #include "pm.h" #include "laya_checkpoint.h" #include "recipes.h" +#include "route_log.h" #include "serve_metrics.h" #include "serve_watchdog.h" #include "serve_hybrid.h" @@ -90,6 +91,7 @@ #endif #ifdef ONEBIT_LAYA #include "encoder.h" +#include "models.h" #include "policy.h" #include "route.h" #include "scorer.h" @@ -179,6 +181,7 @@ struct Options { std::string laya_model; // --laya-model DIR: route each request by the Laya scorer (docs/laya.md) bool laya_auto = false; // --laya: the same with the installed checkpoint (laya_checkpoint()) std::string route_policy; // --route-policy FILE: request class -> device (default: built in) + std::string laya_models; // --laya-models FILE: model auto routed per request (docs/laya-models.md) std::string recipes; // --recipes FILE: tuned backend settings (default: built in, docs/recipes.md) bool no_recipes = false; // --no-recipes: the backend gets serve's own flags only // --template-pin FILE: expected chat-template sha256 per model (default: config/template-pins.json, @@ -1493,22 +1496,12 @@ class LayaDaemon { std::mutex mu_; }; -// classify_request (laya/route.cpp) through the daemon: the same size gate for long_doc and the -// same question, asked of the GGUF -onebit::laya::RequestClass classify_request_gguf(LayaDaemon& d, const std::string& state) { +// One choice question asked of the GGUF daemon, as the C++ scorer's score() asks it: the answer's +// class, confidence and probabilities in the question's option order (empty label on failure). +onebit::laya::RequestClass classify_question_gguf(LayaDaemon& d, const onebit::laya::Question& q, const std::string& state) { onebit::laya::RequestClass out; - if (state.size() >= onebit::laya::kLongDocChars) { - out.label = "long_doc"; - out.confidence = 1.0f; - out.probabilities = {{"long_doc", 1.0f}}; - return out; - } - int variant = 0; - if (const char* v = std::getenv("ONEBIT_LAYA_CLASS_VARIANT")) variant = std::atoi(v); - const onebit::laya::Question q = onebit::laya::request_class_question(variant); nlohmann::ordered_json criteria = nlohmann::ordered_json::object(); - for (const auto& [key, desc] : q.criteria) - if (key != "long_doc") criteria[key] = desc; + for (const auto& [key, desc] : q.criteria) criteria[key] = desc; nlohmann::ordered_json req = {{"state", state}, {"questions", {{"cls", {{"type", q.type}, {"instructions", q.instructions}, {"criteria", criteria}}}}}}; json answer; @@ -1522,6 +1515,36 @@ onebit::laya::RequestClass classify_request_gguf(LayaDaemon& d, const std::strin if ((*a)["probabilities"].contains(key)) out.probabilities.emplace_back(key, (*a)["probabilities"][key].get()); return out; } + +// classify_request (laya/route.cpp) through the daemon: the same size gate for long_doc and the +// same question, asked of the GGUF +onebit::laya::RequestClass classify_request_gguf(LayaDaemon& d, const std::string& state) { + if (state.size() >= onebit::laya::kLongDocChars) { + onebit::laya::RequestClass out; + out.label = "long_doc"; + out.confidence = 1.0f; + out.probabilities = {{"long_doc", 1.0f}}; + return out; + } + int variant = 0; + if (const char* v = std::getenv("ONEBIT_LAYA_CLASS_VARIANT")) variant = std::atoi(v); + onebit::laya::Question q = onebit::laya::request_class_question(variant); + q.criteria.erase(std::remove_if(q.criteria.begin(), q.criteria.end(), [](const auto& c) { return c.first == "long_doc"; }), + q.criteria.end()); + return classify_question_gguf(d, q, state); +} + +// classify_with_table (laya/models.cpp) through the daemon: the table's size gate, then its question +onebit::laya::RequestClass classify_table_gguf(LayaDaemon& d, const onebit::laya::ModelTable& t, const std::string& state) { + if (t.size_gated(state)) { + onebit::laya::RequestClass out; + out.label = t.size_gate_class; + out.confidence = 1.0f; + out.probabilities = {{t.size_gate_class, 1.0f}}; + return out; + } + return classify_question_gguf(d, t.laya_question(), state); +} #endif #endif @@ -1560,6 +1583,62 @@ std::string request_state(const httplib::Request& req, bool head_only = false) { return out; } +#ifdef ONEBIT_LAYA +// The Laya scorer serve routes with (docs/laya.md): with ONEBIT_LAYA_DEVICE=npu the C++ scorer with +// its encoder on the NPU (a private add-on, laya/encoder.h); else ggmlc's GGUF scorer as a daemon on +// HRX0 when the binary and the GGUF are there; else the C++ scorer on the CPU. `note` ends the log line. +#if defined(__linux__) +void start_laya_scorer(const Options& o, const std::string& policy_note, std::unique_ptr& scorer, + std::unique_ptr& laya_daemon) { +#else +void start_laya_scorer(const Options& o, const std::string& policy_note, std::unique_ptr& scorer) { +#endif + // ONEBIT_LAYA_DEVICE=npu: the C++ scorer with its encoder on the NPU, from a private add-on + // (laya/encoder.h, docs/laya.md); else the GGUF scorer, on HRX0 unless it names another + const char* dev = std::getenv("ONEBIT_LAYA_DEVICE"); + const bool laya_npu = dev && std::string(dev) == "npu"; + const std::string gguf = laya_npu ? "" : laya_gguf(o.laya_model), bin = laya_npu ? "" : laya_ggml_bin(); +#if defined(__linux__) + if (!gguf.empty() && !bin.empty()) { + // the GGUF scorer, with the HSA runtime HRX needs (docs/hrx.md) + std::vector env; + if (const std::string hsa = hrx_libhsa(o.hrx_libhsa); !hsa.empty()) env.push_back("IREE_HAL_AMDGPU_LIBHSA_PATH=" + hsa); + if (!dev || !*dev || std::string(dev) == "hrx") { + announce_hrx_runtime(o); + announce_cwsr_env(o); + } + for (const auto& e : hrx_cwsr_env(o)) env.push_back(e); // engine#315 (app/hrx_cwsr.h) + laya_daemon = std::make_unique(std::vector{ + bin, "daemon", gguf, "--family", "typed-decisions", "--device", dev && *dev ? dev : "hrx"}, env); + std::fprintf(stderr, "1bit serve: laya: %s through %s%s\n", gguf.c_str(), bin.c_str(), policy_note.c_str()); + } +#endif + if (!gguf.empty() && gguf == o.laya_model && bin.empty()) + throw std::runtime_error("laya: a GGUF needs ggmlc's laya (build with -DONEBIT_LAYA_GGML=ON, or set ONEBIT_LAYA_GGML)"); +#if defined(__linux__) + if (!laya_daemon) +#endif + { + const onebit::laya::EncoderFactory* f = laya_npu ? onebit::laya::find_encoder("npu") : nullptr; + if (laya_npu && !f) + throw std::runtime_error("laya: ONEBIT_LAYA_DEVICE=npu needs a build with the NPU add-on " + "(-DONEBIT_NPU_PRIVATE=, docs/laya.md)"); + const std::string ckpt = laya_checkpoint(o.laya_model); + scorer = std::make_unique(); + if (!scorer->load(ckpt)) throw std::runtime_error("laya: " + scorer->error()); + if (f) { + std::string err; + onebit::laya::EncoderFn fn = (*f)(ckpt, &err); + if (!fn) throw std::runtime_error("laya: the NPU encoder: " + err); + scorer->set_encoder(std::move(fn)); + } + std::fprintf(stderr, "1bit serve: laya: checkpoint %s%s%s\n", ckpt.c_str(), + laya_npu ? " (encoder on the NPU)" : "", policy_note.c_str()); + } +} + +#endif + // --print-backend (docs/serve.md, "One hop from Lemonade"): a route that needs serve in the // request path. Exit 5 tells Lemonade's onebit recipe to run `1bit serve` as before. constexpr int kServeOnly = 5; @@ -1690,6 +1769,274 @@ static void warm_backend(int port, const std::string& device) { } } +#if defined(ONEBIT_LAYA) && !defined(_WIN32) +// The text Laya decides a --laya-models request on: the last user message (each game step is its own +// decision; the system prompt is the same on every one), or a completion's prompt. +std::string last_user_text(const json& body) { + if (body.contains("prompt") && body["prompt"].is_string()) return body["prompt"].get(); + if (!body.contains("messages") || !body["messages"].is_array()) return ""; + for (auto it = body["messages"].rbegin(); it != body["messages"].rend(); ++it) { + const json& m = *it; + if (!m.is_object() || !m.contains("role") || !m["role"].is_string() || m["role"].get() != "user") continue; + if (!m.contains("content")) return ""; + const json& c = m["content"]; + if (c.is_string()) return c.get(); + std::string out; + if (c.is_array()) + for (const auto& part : c) + if (part.is_object() && part.value("type", "") == "text") out += (out.empty() ? "" : "\n") + part.value("text", ""); + return out; + } + return ""; +} + +// --laya-models FILE (docs/laya-models.md): several models behind one port. Each table row is its own +// `1bit serve -m --device ` child (so the NPU and HRX routes are exactly what they are +// alone); a request with any model name that is not a row's alias (`auto` by convention) is classified +// by Laya with the table's own question, and the class's row answers it. The decision is in the +// reply's X-1bit-Route* headers, its `route` object (non-streamed), GET /v1/route/decisions and /metrics. +int serve_models(const Options& o) { + if (!o.model.empty()) throw std::runtime_error("--laya-models names its models in the table; drop -m"); + if (o.pm || !o.role.empty() || !o.embed.empty() || !o.rerank.empty()) + throw std::runtime_error("--laya-models does not combine with --pm, --embedding/--reranking, --embed or --rerank"); + if (o.device != "auto") throw std::runtime_error("--laya-models: each row names its device; drop --device"); + if (o.print_backend) return serve_only("--laya-models routes each request in serve"); + const onebit::laya::ModelTable table = onebit::laya::load_model_table(o.laya_models); + std::unique_ptr scorer; +#if defined(__linux__) + std::unique_ptr laya_daemon; + start_laya_scorer(o, ", models " + o.laya_models, scorer, laya_daemon); +#else + start_laya_scorer(o, ", models " + o.laya_models, scorer); +#endif + const std::vector rows = table.backends(); + std::string self; + { + std::error_code ec; + self = fs::read_symlink("/proc/self/exe", ec).string(); + if (ec || self.empty()) throw std::runtime_error("--laya-models: cannot find this binary (/proc/self/exe)"); + } + struct Backend { + onebit::laya::ModelRow row; + int port = 0; + std::vector argv; + }; + std::vector backends; + for (const auto& r : rows) { + if (!fs::exists(r.model)) throw std::runtime_error("--laya-models: " + r.alias + ": no such model " + r.model); + if (!r.bin.empty() && !fs::exists(r.bin)) throw std::runtime_error("--laya-models: " + r.alias + ": no such binary " + r.bin); + Backend b; + b.row = r; + b.port = free_port(); + b.argv = {r.bin.empty() ? self : r.bin, "serve", "-m", r.model, "--device", r.device, "--port", std::to_string(b.port), + "--host", "127.0.0.1", "--alias", r.alias}; + b.argv.insert(b.argv.end(), r.args.begin(), r.args.end()); + backends.push_back(std::move(b)); + } + std::fprintf(stderr, "1bit serve: laya-models: %zu classes, %zu backends (%s)\n", table.classes.size(), backends.size(), + o.laya_models.c_str()); + for (const auto& [cls, r] : table.rows) + std::fprintf(stderr, "1bit serve: laya-models: %s -> %s on %s\n", cls.c_str(), r.alias.c_str(), r.device.c_str()); + + std::atomic ready{false}; + ServeMetrics metrics("laya-models"); + Lifecycle lifecycle("laya-models"); + RouteLog log; + httplib::Server srv; + install_request_guard(srv, o.host); + std::vector> children; + std::mutex scorer_mu, cache_mu; + std::unordered_map cache; // state -> class + std::deque cache_order; + std::vector> inflight(backends.size()); + + auto health = [&](const httplib::Request&, httplib::Response& res) { + if (!ready) { + res.status = 503; + res.set_content(R"({"status":"loading"})", "application/json"); + return; + } + for (const auto& c : children) + if (!c->alive()) { + res.status = 503; + res.set_content(json{{"status", "error"}, {"error", "backend on port " + std::to_string(c->port()) + " exited"}}.dump(), + "application/json"); + return; + } + res.set_content(R"({"status":"ok"})", "application/json"); + }; + srv.Get("/health", health); + srv.Get("/v1/health", health); + srv.Get("/v1/models", [&](const httplib::Request&, httplib::Response& res) { + json data = json::array({{{"id", "auto"}, {"object", "model"}, {"owned_by", "1bit"}, {"device", "laya"}}}); + for (const auto& b : backends) + data.push_back({{"id", b.row.alias}, {"object", "model"}, {"owned_by", "1bit"}, {"device", b.row.device}}); + res.set_content(json{{"object", "list"}, {"data", data}}.dump(), "application/json"); + }); + srv.Get("/v1/route/table", [&](const httplib::Request&, httplib::Response& res) { + json classes = json::array(); + for (const auto& [cls, desc] : table.classes) { + const auto& r = *table.row_for(cls); + classes.push_back({{"class", cls}, {"description", desc}, {"model", r.alias}, {"device", r.device}}); + } + res.set_content(json{{"question", table.question}, {"default", table.fallback}, + {"min_confidence", table.min_confidence}, {"classes", classes}}.dump(), "application/json"); + }); + srv.Get("/v1/route/decisions", [&](const httplib::Request& q, httplib::Response& res) { + long since = 0; + size_t limit = 100; + try { + if (q.has_param("since")) since = std::stol(q.get_param_value("since")); + if (q.has_param("limit")) limit = size_t(std::max(1L, std::min(512L, std::stol(q.get_param_value("limit"))))); + } catch (const std::exception&) { + res.status = 400; + res.set_content(R"({"error":{"message":"since and limit are integers"}})", "application/json"); + return; + } + res.set_content(log.recent(since, limit).dump(), "application/json"); + }); + srv.Get("/metrics", [&](const httplib::Request&, httplib::Response& r) { + r.set_content(metrics.render(lifecycle) + log.render(), "text/plain; version=0.0.4; charset=utf-8"); + }); + auto post = [&](const httplib::Request& q, httplib::Response& r) { + if (!ready) { + r.status = 503; + r.set_content(R"({"error":{"message":"models are loading"}})", "application/json"); + return; + } + RouteDecision d; + d.id = log.next_id(); + d.t_unix = std::chrono::duration(std::chrono::system_clock::now().time_since_epoch()).count(); + const json body = json::parse(q.body, nullptr, false); + if (!body.is_object()) { + r.status = 400; + r.set_content(R"({"error":{"message":"request body is not JSON"}})", "application/json"); + return; + } + d.stream = body.value("stream", false); + const std::string asked = body.contains("model") && body["model"].is_string() ? body["model"].get() : ""; + const onebit::laya::ModelRow* row = table.by_alias(asked); + if (row) { + d.cls = "pinned"; + } else { + const std::string state = last_user_text(body); + onebit::laya::RequestClass cls; + { + std::lock_guard lock(cache_mu); + if (const auto c = cache.find(state); c != cache.end()) { + cls = c->second; + d.cached = true; + } + } + const auto t0 = std::chrono::steady_clock::now(); + if (!d.cached) { + std::lock_guard lock(scorer_mu); // the scorer is not reentrant +#if defined(__linux__) + if (laya_daemon) cls = classify_table_gguf(*laya_daemon, table, state); + else +#endif + cls = onebit::laya::classify_with_table(*scorer, table, state); + } + d.router_ms = d.cached ? 0.0 : std::chrono::duration(std::chrono::steady_clock::now() - t0).count(); + if (cls.label.empty()) { + r.status = 502; + r.set_content(R"({"error":{"message":"laya routing failed"}})", "application/json"); + return; + } + if (!d.cached) { + std::lock_guard lock(cache_mu); + if (cache.size() >= 4096) { + cache.erase(cache_order.front()); + cache_order.pop_front(); + } + if (cache.emplace(state, cls).second) cache_order.push_back(state); + } + d.laya_class = cls.label; + d.confidence = cls.confidence; + if (table.size_gated(state)) { + // the gate decides by size alone: min_confidence is for Laya's answers + d.cls = cls.label; + row = table.row_for(cls.label); + } else { + row = &table.pick(cls.label, cls.confidence, &d.cls); + } + } + const size_t i = table.backend_index(*row); + d.model = row->alias; + d.device = row->device; + char route_hdr[160], ms_hdr[32]; + std::snprintf(route_hdr, sizeof route_hdr, "%s %.2f %s", d.cls.c_str(), d.confidence, d.device.c_str()); + std::snprintf(ms_hdr, sizeof ms_hdr, "%.1f", d.router_ms); + std::fprintf(stderr, "1bit serve: laya-models: #%ld %s -> %s | %s %.1f ms\n", d.id, route_hdr, d.model.c_str(), + d.cls == "pinned" ? "pinned" : d.cached ? "cached" : "classified", d.router_ms); + auto held = std::make_shared(&inflight[i]); + const auto t_req = std::chrono::steady_clock::now(); + forward(backends[i].port, held, -1, d.model, false, "", o.precise, q, r); + r.set_header("X-1bit-Route", route_hdr); // class, Laya confidence, device (the --laya format) + r.set_header("X-1bit-Route-Model", d.model); + r.set_header("X-1bit-Route-Ms", ms_hdr); + r.set_header("X-1bit-Route-Id", std::to_string(d.id)); + if (!d.stream) { + d.status = r.status; + d.request_ms = std::chrono::duration(std::chrono::steady_clock::now() - t_req).count(); + read_reply_stats(r.body, d); + json out = json::parse(r.body, nullptr, false); + if (out.is_object()) { + json route = {{"id", d.id}, {"class", d.cls}, {"laya_class", d.laya_class}, {"confidence", d.confidence}, + {"model", d.model}, {"device", d.device}, {"router_ms", d.router_ms}, {"cached", d.cached}, + {"request_ms", d.request_ms}}; + out["route"] = route; + r.set_content(out.dump(), "application/json"); + } + } + log.add(d); + }; + srv.Post("/v1/chat/completions", post); + srv.Post("/v1/completions", post); + + if (!srv.bind_to_port(o.host, o.port)) + throw std::runtime_error("cannot listen on " + o.host + ":" + std::to_string(o.port)); + std::thread listener([&] { srv.listen_after_bind(); }); + struct Stop { + httplib::Server& s; + std::thread& t; + ~Stop() { s.stop(); if (t.joinable()) t.join(); } + } stop{srv, listener}; + struct sigaction sa{}; + sa.sa_handler = on_stop_signal; + ::sigaction(SIGTERM, &sa, nullptr); + ::sigaction(SIGINT, &sa, nullptr); + auto line = [&metrics](const std::string& l) { metrics.backend_line(l); }; + for (const auto& b : backends) { + std::fprintf(stderr, "1bit serve: laya-models: starting %s on %s (port %d%s%s)\n", b.row.alias.c_str(), b.row.device.c_str(), + b.port, b.row.bin.empty() ? "" : ", ", b.row.bin.c_str()); + children.push_back(std::make_unique(b.argv, std::vector{}, b.port, "/health", line)); + } + for (size_t i = 0; i < children.size(); ++i) + if (!children[i]->wait_ready(std::chrono::seconds(900), g_stop)) { + if (g_stop) return 0; + std::fprintf(stderr, "1bit serve: laya-models: %s did not become ready\n", backends[i].row.alias.c_str()); + return 1; + } + ready = true; + lifecycle.count_start(); + std::fprintf(stderr, "1bit serve: ready on http://%s:%d (laya-models: model auto is routed per request)\n", o.host.c_str(), o.port); + auto all_alive = [&] { + for (auto& c : children) if (!c->alive()) return false; + return true; + }; + while (all_alive() && !g_stop) std::this_thread::sleep_for(std::chrono::milliseconds(200)); + if (!g_stop) { + for (size_t i = 0; i < children.size(); ++i) + if (!children[i]->alive()) + std::fprintf(stderr, "1bit serve: laya-models: %s exited (%s)\n", backends[i].row.alias.c_str(), + children[i]->exit_text().c_str()); + return 1; + } + return 0; +} +#endif + int serve_child(const Options& given) { Options o = given; if (hadamard_q4_0(o.model)) @@ -1779,50 +2126,13 @@ int serve_child(const Options& given) { if (laya && !o.role.empty()) throw std::runtime_error("--laya does not combine with --embedding/--reranking"); if (laya) policy = onebit::laya::load_route_policy(o.route_policy); - // ONEBIT_LAYA_DEVICE=npu: the C++ scorer with its encoder on the NPU, from a private add-on - // (laya/encoder.h, docs/laya.md); else the GGUF scorer, on HRX0 unless it names another - const char* dev = std::getenv("ONEBIT_LAYA_DEVICE"); - const bool laya_npu = dev && std::string(dev) == "npu"; // the route policy is the device router's; --pm-router laya only uses the classifier const std::string policy_note = laya ? (o.route_policy.empty() ? ", policy built in" : ", policy " + o.route_policy) : ""; - const std::string gguf = laya_npu ? "" : laya_gguf(o.laya_model), bin = laya_npu ? "" : laya_ggml_bin(); -#if defined(__linux__) - if (!gguf.empty() && !bin.empty()) { - // the GGUF scorer, with the HSA runtime HRX needs (docs/hrx.md) - std::vector env; - if (const std::string hsa = hrx_libhsa(o.hrx_libhsa); !hsa.empty()) env.push_back("IREE_HAL_AMDGPU_LIBHSA_PATH=" + hsa); - if (!dev || !*dev || std::string(dev) == "hrx") { - announce_hrx_runtime(o); - announce_cwsr_env(o); - } - for (const auto& e : hrx_cwsr_env(o)) env.push_back(e); // engine#315 (app/hrx_cwsr.h) - laya_daemon = std::make_unique(std::vector{ - bin, "daemon", gguf, "--family", "typed-decisions", "--device", dev && *dev ? dev : "hrx"}, env); - std::fprintf(stderr, "1bit serve: laya: %s through %s%s\n", gguf.c_str(), bin.c_str(), policy_note.c_str()); - } -#endif - if (!gguf.empty() && gguf == o.laya_model && bin.empty()) - throw std::runtime_error("laya: a GGUF needs ggmlc's laya (build with -DONEBIT_LAYA_GGML=ON, or set ONEBIT_LAYA_GGML)"); #if defined(__linux__) - if (!laya_daemon) + start_laya_scorer(o, policy_note, scorer, laya_daemon); +#else + start_laya_scorer(o, policy_note, scorer); #endif - { - const onebit::laya::EncoderFactory* f = laya_npu ? onebit::laya::find_encoder("npu") : nullptr; - if (laya_npu && !f) - throw std::runtime_error("laya: ONEBIT_LAYA_DEVICE=npu needs a build with the NPU add-on " - "(-DONEBIT_NPU_PRIVATE=, docs/laya.md)"); - const std::string ckpt = laya_checkpoint(o.laya_model); - scorer = std::make_unique(); - if (!scorer->load(ckpt)) throw std::runtime_error("laya: " + scorer->error()); - if (f) { - std::string err; - onebit::laya::EncoderFn fn = (*f)(ckpt, &err); - if (!fn) throw std::runtime_error("laya: the NPU encoder: " + err); - scorer->set_encoder(std::move(fn)); - } - std::fprintf(stderr, "1bit serve: laya: checkpoint %s%s%s\n", ckpt.c_str(), - laya_npu ? " (encoder on the NPU)" : "", policy_note.c_str()); - } if (laya) { for (const std::string& dev : gguf_devices()) { try { @@ -2768,7 +3078,8 @@ void usage(FILE* out) { " (default 45000; 0 refuses at once; docs/npu.md)\n" " [--laya | --laya-model DIR] [--route-policy FILE] with --device auto, Laya classifies\n" " each conversation and the route policy picks its device (docs/laya.md)\n" - " [--recipes FILE | --no-recipes] tuned llama-server settings per model and route\n" + " [--laya-models FILE (no -m) --laya] several models behind one port: Laya picks\n" + " the model and its device for each request with model auto (docs/laya-models.md)\n" " [--recipes FILE | --no-recipes] tuned llama-server settings per model and route\n" " (default: config/recipes.json, built in; docs/recipes.md)\n" " [--template-pin FILE | --no-template-pin] [--template-pin-strict] a GGUF's\n" " tokenizer.chat_template is a program that runs on every request: the load logs\n" @@ -2878,6 +3189,7 @@ int run_serve(int argc, char** argv) { else if (a == "--laya-model") o.laya_model = next(); else if (a == "--laya") o.laya_auto = true; else if (a == "--route-policy") o.route_policy = next(); + else if (a == "--laya-models") o.laya_models = next(); else if (a == "--recipes") o.recipes = next(); else if (a == "--no-recipes") o.no_recipes = true; else if (a == "--template-pin") o.template_pin = next(); @@ -2908,6 +3220,14 @@ int run_serve(int argc, char** argv) { else if (a == "-h" || a == "--help") { usage(stdout); return 0; } else throw std::runtime_error("unknown option " + a); } + if (!o.laya_models.empty()) { +#if defined(ONEBIT_LAYA) && !defined(_WIN32) + if (o.laya_model.empty() && !o.laya_auto) throw std::runtime_error("--laya-models needs --laya or --laya-model DIR"); + return serve_models(o); +#else + throw std::runtime_error("--laya-models is not part of this build (Linux, -DONEBIT_LAYA=ON; docs/laya-models.md)"); +#endif + } if (o.model.empty()) { usage(stderr); return 2; } if (o.watchdog.secs < 0 || o.watchdog.prompt_tps < 0) throw std::runtime_error("--watchdog and --watchdog-prompt-tps take a number >= 0 (0 turns it off)"); diff --git a/config/laya-models-game.json b/config/laya-models-game.json new file mode 100644 index 000000000..ff7699dc2 --- /dev/null +++ b/config/laya-models-game.json @@ -0,0 +1,17 @@ +{ + "about": "Model table for the game agent demo (engine #1313, docs/laya-models.md): 1bit serve --laya-models config/laya-models-game.json --laya. Each request with model auto is classified by Laya as reflex or plan; reflex steps go to a small model on the NPU, planning to a bigger model on HRX. Model paths are this box's; copy the file and change them for another.", + "question": { + "instructions": "What does the game agent need to answer this step?", + "classes": { + "reflex": "one quick action from what is in view right now: move, turn, jump, mine or place the block ahead, attack, eat, pick up an item", + "plan": "planning: a crafting tree, a build plan, choosing the next goals, writing a reusable skill, or recovering after a death or a failed step" + } + }, + "default": "plan", + "min_confidence": 0.0, + "classes": { + "reflex": {"model": "~/models/npu/Qwen3-0.6B", "device": "npu", "alias": "Qwen3-0.6B-NPU"}, + "plan": {"model": "~/models/Qwen3-Coder-30B-A3B-Instruct-Q4_K_M.gguf", "device": "hrx", + "alias": "Qwen3-Coder-30B-A3B-HRX", "args": ["--ctx-size", "16384"]} + } +} diff --git a/docs/laya-models.md b/docs/laya-models.md new file mode 100644 index 000000000..f6bfc005c --- /dev/null +++ b/docs/laya-models.md @@ -0,0 +1,140 @@ + +# Several models behind one port: `1bit serve --laya-models` + +`--laya` ([laya.md](laya.md)) picks a *device* for the one model a `serve` runs. `--laya-models FILE` +picks the *model and its device* for every request: a model table names its own classes, the Laya +question that tells them apart, and for each class the model that answers it and where it runs. +A client sends `"model": "auto"` and Laya decides each request. + +The first table is the game agent's (engine #1313): quick reflex steps go to a small model on the +NPU, and planning goes to a bigger model on HRX. + +```sh +1bit serve --laya-models config/laya-models-game.json --laya --port 8000 +curl -s localhost:8000/v1/chat/completions -H 'Content-Type: application/json' \ + -d '{"model":"auto","messages":[{"role":"user","content":"A zombie is 2 blocks to the left. Next action?"}]}' +``` + +## The table + +```json +{ + "question": { + "instructions": "What does the game agent need to answer this step?", + "classes": { + "reflex": "one quick action from what is in view right now: ...", + "plan": "planning: a crafting tree, a build plan, choosing the next goals, ..." + } + }, + "default": "plan", + "min_confidence": 0.0, + "size_gate": {"chars": 1024, "class": "plan"}, + "classes": { + "reflex": {"model": "~/models/npu/Qwen3-0.6B", "device": "npu", "alias": "Qwen3-0.6B-NPU"}, + "plan": {"model": "~/models/Qwen3-Coder-30B-A3B-Instruct-Q4_K_M.gguf", "device": "hrx", + "alias": "Qwen3-Coder-30B-A3B-HRX", "args": ["--ctx-size", "16384"]} + } +} +``` + +| field | meaning | +|---|---| +| `question.instructions`, `question.classes` | one Laya `choice` question; the classes are its options, in the file's order | +| `default` | the class a request takes when Laya's confidence is below `min_confidence` (default: the first class) | +| `size_gate` | optional: a request this long or longer is that class, with no model call and no `min_confidence` | +| `classes..model` | a `.gguf` file or an NPU model directory; `~` and paths relative to the table's directory resolve | +| `classes..device` | `npu`, `hrx`, `cpu` or `hybrid`, passed to the row's `1bit serve --device` | +| `classes..alias` | the name replies carry (default: the file name); a request naming it skips Laya | +| `classes..args` | extra `1bit serve` arguments for that row | +| `classes..bin` | optional: the `1bit` binary that serves that row (default: this one), e.g. a release build that carries the sealed NPU kernels while this one is a source build | + +Every class needs a row, and every row a class; an unknown device, a missing model file, or one alias for +two different backends is refused by name at start. Two classes may name the same model, device and args: +they share one backend. + +## What a request does + +1. Each row runs as its own `1bit serve -m --device ` child on a loopback port, started + when serve starts. The NPU row is exactly the NPU route, the HRX row exactly the HRX route. `/health` + answers 200 once every row is ready. +2. A request whose `model` is a row's alias goes to that row (`pinned`). Any other name (`auto` by + convention) is decided by Laya: the table's size gate, then the table's question about the **last user + message** (each step of an agent is its own decision; the system prompt is the same on every one). + A state asked before is answered from a cache (4,096 states), at 0 ms. +3. The class's row answers. Below `min_confidence` the `default` class's row answers. + +The scorer is the one `--laya` uses: ggmlc's GGUF daemon on HRX0 when the build has it, else the C++ +scorer (`ONEBIT_LAYA_DEVICE=npu` with the NPU add-on puts its encoder on the NPU). + +## What a reply says + +| where | what | +|---|---| +| `X-1bit-Route` | ` `, the `--laya` format (`pinned 0.00 npu` for a pinned request) | +| `X-1bit-Route-Model` | the alias of the row that answered | +| `X-1bit-Route-Ms` | the Laya decision in milliseconds (0.0 when pinned or cached) | +| `X-1bit-Route-Id` | the decision's id in `/v1/route/decisions` | +| body `route` (non-streamed) | `id`, `class`, `laya_class`, `confidence`, `model`, `device`, `router_ms`, `cached`, `request_ms` | +| body `model` | the row's alias | + +`GET /v1/route/decisions?since=ID&limit=N` returns the last decisions (up to 512 kept) after `since`, +oldest first: the fields above plus the reply's `prompt_tokens`, `completion_tokens`, `prompt_tok_s` and +`decode_tok_s` from its `usage` and `timings` (llama-server and the NPU routes both send them; a streamed +reply has none of them, and no status). `GET /v1/route/table` returns the table's question and each class's +model and device. `GET /v1/models` lists `auto` and every alias. `GET /metrics` adds, by class, model and +device: `onebit_route_decisions_total`, `onebit_route_router_ms_sum` / `_count` (Laya decisions only), +`onebit_route_request_ms_sum` / `_count`, `onebit_route_completion_tokens_total`, +`onebit_route_prompt_tokens_total`, and `onebit_route_last_decode_tok_s` by model and device. These are the +numbers a demo HUD reads; every one was measured by serve or reported by the backend. + +## Limits + +- `--laya-models` takes no `-m`, `--device`, `--pm`, `--embed`/`--rerank` or a role; per-row options go in + `args`. Linux only. +- Laya's zero-shot answer to a new question is a starting point, not a measured router: the base + checkpoints are near chance on decision types they were not tuned for (Laya's own README). The game + table's accuracy is measured below, and a fine-tune on the game's own logged decisions is the plan for + raising it (engine #1313). + +## Tests + +`tests/laya_models_test.cpp` (ctest `laya_models`, no checkpoint): parsing, the question, the pick and +its fallback, aliases, shared backends, the size gate, the refusals, and the shipped game table. +`tests/laya_models_e2e.sh` (ctest `laya_models_e2e`, with `ONEBIT_LAYA_ROUTE_MODEL`): two rows on fake +llama-servers behind real `1bit serve` children; the reply comes from the row the headers name, pinned and +cached requests, the decisions log, the table, `/v1/models` and `/metrics`, and the refusals. + +## Measured + +Strix Halo, 2026-11-07, `config/laya-models-game.json` with this box's paths: Qwen3-0.6B (Q4NX) on the NPU and +Qwen3-Coder-30B-A3B Q4_K_M on HRX, the C++ scorer on the `typed-decisions/` checkpoint (CPU), and 20 +hand-labelled game steps (12 reflex, 8 plan) sent as `model: auto`, `max_tokens 48`, temperature 0. +The NPU row ran from a release build (`bin`), because the source build had no NPU add-on. + +| | result | +|---|---| +| ready, both rows (the 30B from the page cache) | 16 s | +| class right, zero-shot | **19/20**: reflex 11/12, plan 8/8. The one miss, "place a wall block here", went to plan at confidence 0.01 | +| confidence | low throughout: 0.01-0.57, median 0.13. `min_confidence` stays 0 until the fine-tune | +| Laya decision (`router_ms`) | p50 **302 ms** (259-358): the CPU scorer on a shared box. ggmlc's GGUF scorer on HRX (25 ms p50 in [laya.md](laya.md)) is the fast path | +| decode, from the replies' `timings` | NPU p50 85.2 tok/s, HRX p50 86.8 tok/s | +| headers, `route` object, `/v1/route/decisions` and `/metrics` | agree on all 20 (11 decisions npu, 9 hrx) | + +Qwen3-0.6B thinks by default, so a 48-token budget can end inside the thinking: every NPU reply above had an empty +`content` with all 48 tokens in `reasoning_content`. A request with `"chat_template_kwargs":{"enable_thinking":false}` +gets the answer in `content` (checked on the same row). Routing does not depend on it. diff --git a/docs/laya.md b/docs/laya.md index da5df2164..91c8d98ff 100644 --- a/docs/laya.md +++ b/docs/laya.md @@ -21,6 +21,8 @@ or long document (95.5% right), and a measured policy picks the device. The test labelled requests. This is step 4 of the port (docs/PORTING.md). [Laya first](laya-first.md) is what comes next: Laya deciding the first request on a cold Lemonade, and which of our models answers it. +[`--laya-models`](laya-models.md) puts several models behind one `serve`: Laya picks the model and +its device for each request with `"model": "auto"`, from a table with its own classes and question. [Laya](https://github.com/NandhaKishorM/laya) (Apache-2.0) is a non-autoregressive decision model: a ModernBERT-style encoder plus an RLCD decision head. It answers typed questions (`choice`, `score`, `noul`) about a diff --git a/docs/serve.md b/docs/serve.md index 9ec19fab9..3f0db0fb4 100644 --- a/docs/serve.md +++ b/docs/serve.md @@ -31,6 +31,7 @@ so it also works on its own with any OpenAI client. [--hrx-cwsr-placement=auto|on|off] [--mtp HEAD.gguf | --dflash DRAFT.gguf | --no-mtp] [--mtp-max N] [--mtp-p-min P] [--mmproj MMPROJ.gguf] [--laya | --laya-model DIR] [--route-policy FILE] + [--laya-models FILE --laya] # no -m: model auto per request (docs/laya-models.md) [--pm [--pm-experts FILE] [--lemonade-url URL]] [--moe-slots N|auto] [--moe-subst R] [--moe-prefetch N] [--moe-predict off|router|block] # refused in this build, below @@ -294,7 +295,7 @@ two processes (lemond, llama-server) instead of two hops and three. |---|---|---| | 0 | the plan is on stdout | starts it | | 3 | the model does not fit ([above](#will-it-fit)) | fails the load with serve's line in its log | -| 5 | the route needs serve in the request path: an NPU model directory, `--device npu`, `hybrid`, `--pm`, `--laya`, `--embed` or `--rerank` | runs `1bit serve` as before | +| 5 | the route needs serve in the request path: an NPU model directory, `--device npu`, `hybrid`, `--pm`, `--laya`, `--laya-models`, `--embed` or `--rerank` | runs `1bit serve` as before | Any other exit (an older engine without the flag) also falls back to `1bit serve`, and `LEMONADE_ONEBIT_ONE_HOP=0` in lemond's environment turns one hop off. `env` lists only variables @@ -526,7 +527,9 @@ it is the engine's only GPU route. With `--laya` (or `--laya-model DIR`), Laya classifies each conversation and the route policy picks the device: the built-in policy sends every class to HRX, long documents included, since HRX prefills Qwen3.8-27B at 335 tok/s pp512 (docs/laya.md). The first turn of -a conversation pays about 0.5 s for the decision. +a conversation pays about 0.5 s for the decision. With `--laya-models FILE` (and no `-m`), one +`serve` runs several models, each row of the table as its own child, and Laya picks the model and its +device for every request with `"model": "auto"` ([laya-models.md](laya-models.md)). HRX needs TheRock's HSA runtime: the distro `libhsa` rejects gfx1151's PM4-emulation probe, and then HRX registers no device. `serve` sets diff --git a/laya/models.cpp b/laya/models.cpp new file mode 100644 index 000000000..817bd34eb --- /dev/null +++ b/laya/models.cpp @@ -0,0 +1,232 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +#include "models.h" + +#include +#include +#include +#include +#include +#include + +#include + +#include "route.h" + +namespace onebit::laya { + +namespace { + +std::string resolve_path(const std::string& p, const std::string& base_dir) { + namespace fs = std::filesystem; + if (p.empty()) return p; + if (p[0] == '~' && (p.size() == 1 || p[1] == '/')) { + const char* home = std::getenv("HOME"); + return std::string(home ? home : "") + p.substr(1); + } + if (fs::path(p).is_absolute() || base_dir.empty()) return p; + return (fs::path(base_dir) / p).lexically_normal().string(); +} + +bool known_device(const std::string& d) { + return d == "npu" || d == "hrx" || d == "cpu" || d == "hybrid"; +} + +} // namespace + +bool ModelTable::parse(const std::string& json_text, const std::string& base_dir, std::string& err) { + // ordered: the question's options keep the file's order (Laya's argmax index order) + nlohmann::ordered_json j; + try { + j = nlohmann::ordered_json::parse(json_text); + } catch (const std::exception& e) { + err = std::string("model table: ") + e.what(); + return false; + } + if (!j.is_object()) { err = "model table: not a JSON object"; return false; } + const auto& q = j.contains("question") ? j["question"] : nlohmann::ordered_json(); + if (!q.is_object() || !q.contains("instructions") || !q["instructions"].is_string() || !q.contains("classes") || + !q["classes"].is_object()) { + err = "model table: needs \"question\": {\"instructions\": ..., \"classes\": {class: description}}"; + return false; + } + question = q["instructions"].get(); + classes.clear(); + for (const auto& [cls, desc] : q["classes"].items()) { + if (!desc.is_string() || desc.get().empty()) { + err = "model table: class \"" + cls + "\" needs a description"; + return false; + } + classes.emplace_back(cls, desc.get()); + } + if (classes.size() < 2) { err = "model table: the question needs at least two classes"; return false; } + auto is_class = [&](const std::string& c) { + return std::any_of(classes.begin(), classes.end(), [&](const auto& kv) { return kv.first == c; }); + }; + if (!j.contains("classes") || !j["classes"].is_object()) { err = "model table: needs a \"classes\" object"; return false; } + rows.clear(); + for (const auto& [cls, r] : j["classes"].items()) { + if (!is_class(cls)) { err = "model table: row \"" + cls + "\" is not one of the question's classes"; return false; } + if (!r.is_object() || !r.contains("model") || !r["model"].is_string() || !r.contains("device") || + !r["device"].is_string()) { + err = "model table: class \"" + cls + "\" needs \"model\" and \"device\""; + return false; + } + ModelRow row; + row.model = resolve_path(r["model"].get(), base_dir); + row.device = r["device"].get(); + if (!known_device(row.device)) { + err = "model table: class \"" + cls + "\": unknown device \"" + row.device + "\" (npu, hrx, cpu, hybrid)"; + return false; + } + row.alias = r.value("alias", std::filesystem::path(row.model).filename().string()); + if (r.contains("args")) { + if (!r["args"].is_array()) { err = "model table: class \"" + cls + "\": \"args\" is a list of strings"; return false; } + for (const auto& a : r["args"]) { + if (!a.is_string()) { err = "model table: class \"" + cls + "\": \"args\" is a list of strings"; return false; } + row.args.push_back(a.get()); + } + } + if (r.contains("bin")) { + if (!r["bin"].is_string() || r["bin"].get().empty()) { + err = "model table: class \"" + cls + "\": \"bin\" is the path of a 1bit binary"; + return false; + } + row.bin = resolve_path(r["bin"].get(), base_dir); + } + rows.emplace_back(cls, std::move(row)); + } + for (const auto& [cls, desc] : classes) { + (void)desc; + if (std::none_of(rows.begin(), rows.end(), [&](const auto& kv) { return kv.first == cls; })) { + err = "model table: class \"" + cls + "\" has no row"; + return false; + } + } + // two rows with one alias would make a pinned request ambiguous + for (size_t a = 0; a < rows.size(); ++a) + for (size_t b = a + 1; b < rows.size(); ++b) + if (rows[a].second.alias == rows[b].second.alias && + (rows[a].second.model != rows[b].second.model || rows[a].second.device != rows[b].second.device || + rows[a].second.args != rows[b].second.args || rows[a].second.bin != rows[b].second.bin)) { + err = "model table: classes \"" + rows[a].first + "\" and \"" + rows[b].first + + "\" name different backends with one alias \"" + rows[a].second.alias + "\""; + return false; + } + fallback = j.value("default", classes.front().first); + if (!is_class(fallback)) { err = "model table: \"default\" \"" + fallback + "\" is not a class"; return false; } + min_confidence = j.value("min_confidence", 0.0f); + size_gate_chars = 0; + size_gate_class.clear(); + if (j.contains("size_gate")) { + const auto& g = j["size_gate"]; + if (!g.is_object() || !g.contains("chars") || !g["chars"].is_number_unsigned() || !g.contains("class") || + !g["class"].is_string() || !is_class(g["class"].get())) { + err = "model table: \"size_gate\" is {\"chars\": N, \"class\": }"; + return false; + } + size_gate_chars = g["chars"].get(); + size_gate_class = g["class"].get(); + } + return true; +} + +Question ModelTable::laya_question() const { + Question q; + q.type = "choice"; + q.instructions = question; + q.criteria = classes; + return q; +} + +const ModelRow& ModelTable::pick(const std::string& cls, float confidence, std::string* used) const { + const std::string c = (confidence >= min_confidence && !cls.empty()) ? cls : fallback; + for (const auto& [k, row] : rows) + if (k == c) { + if (used) *used = c; + return row; + } + for (const auto& [k, row] : rows) + if (k == fallback) { + if (used) *used = fallback; + return row; + } + throw std::logic_error("model table: no row for the fallback class"); // parse() rules this out +} + +const ModelRow* ModelTable::row_for(const std::string& cls) const { + for (const auto& [k, row] : rows) + if (k == cls) return &row; + return nullptr; +} + +const ModelRow* ModelTable::by_alias(const std::string& name) const { + for (const auto& [k, row] : rows) { + (void)k; + if (row.alias == name) return &row; + } + return nullptr; +} + +static bool same_backend(const ModelRow& a, const ModelRow& b) { + return a.model == b.model && a.device == b.device && a.args == b.args && a.bin == b.bin; +} + +std::vector ModelTable::backends() const { + std::vector out; + for (const auto& [k, row] : rows) { + (void)k; + if (std::none_of(out.begin(), out.end(), [&](const ModelRow& o) { return same_backend(o, row); })) out.push_back(row); + } + return out; +} + +size_t ModelTable::backend_index(const ModelRow& row) const { + const auto b = backends(); + for (size_t i = 0; i < b.size(); ++i) + if (same_backend(b[i], row)) return i; + throw std::logic_error("model table: row is not a backend"); +} + +ModelTable load_model_table(const std::string& path) { + std::ifstream in(path); + if (!in) throw std::runtime_error("model table: cannot read " + path); + std::stringstream ss; + ss << in.rdbuf(); + ModelTable t; + std::string err; + const std::string base = std::filesystem::absolute(path).parent_path().string(); + if (!t.parse(ss.str(), base, err)) throw std::runtime_error(err + " in " + path); + return t; +} + +RequestClass classify_with_table(Scorer& scorer, const ModelTable& table, const std::string& state) { + RequestClass out; + if (table.size_gated(state)) { + out.label = table.size_gate_class; + out.confidence = 1.0f; + out.probabilities = {{table.size_gate_class, 1.0f}}; + return out; + } + std::vector answers; + if (!scorer.score(state, {table.laya_question()}, answers) || answers.empty()) return out; + out.label = answers.front().choice; + out.confidence = answers.front().confidence; + out.probabilities = answers.front().probabilities; + return out; +} + +} // namespace onebit::laya diff --git a/laya/models.h b/laya/models.h new file mode 100644 index 000000000..a6f4fe2c4 --- /dev/null +++ b/laya/models.h @@ -0,0 +1,80 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// laya/models.h — a model table for `1bit serve --laya-models FILE` (docs/laya-models.md). +// Where --laya picks a *device* for one model, a model table picks a *model and its device* +// for every request: the table names its own classes and the Laya question that tells them +// apart, and each class names the model that answers it and the device it runs on. +#pragma once + +#include +#include +#include + +#include "scorer.h" + +namespace onebit::laya { + +struct RequestClass; + +struct ModelRow { + std::string model; // a .gguf file or an NPU model directory (~ and the table's directory resolved) + std::string device; // npu, hrx, cpu or hybrid: passed to the row's `1bit serve --device` + std::string alias; // the name replies carry; a request naming it skips Laya + std::vector args; // extra `1bit serve` arguments for this row + std::string bin; // the `1bit` binary that serves this row ("" = this one), e.g. a release build +}; + +struct ModelTable { + std::string question; // Laya's instructions + std::vector> classes; // class -> description, the question's options in order + std::vector> rows; // class -> row (every class has one) + std::string fallback; // the class a low-confidence request takes + float min_confidence = 0.0f; + size_t size_gate_chars = 0; // 0: off; else a request this long or longer is `size_gate_class` + std::string size_gate_class; + + // Parses the table; `base_dir` resolves relative model paths. False with the reason in `err`. + bool parse(const std::string& json_text, const std::string& base_dir, std::string& err); + + // The question asked of Laya: one choice among the classes. + Question laya_question() const; + + // The row for class `cls` at Laya confidence `confidence`: the class's own row, or the + // fallback class's row when the confidence is below min_confidence. Returns the class + // that was used (cls or fallback) through `used`. + const ModelRow& pick(const std::string& cls, float confidence, std::string* used = nullptr) const; + + // Whether `state` is decided by the size gate (no model call, and no min_confidence). + bool size_gated(const std::string& state) const { return size_gate_chars > 0 && state.size() >= size_gate_chars; } + + // The class's own row (nullptr for an unknown class). + const ModelRow* row_for(const std::string& cls) const; + + // The row whose alias is `name`, or nullptr. + const ModelRow* by_alias(const std::string& name) const; + + // The distinct backends (model, device, args): two classes may share one. + std::vector backends() const; + size_t backend_index(const ModelRow& row) const; +}; + +// Loads a table file (the table's own directory resolves relative model paths). +ModelTable load_model_table(const std::string& path); + +// Classifies `state` with `table`'s question on the C++ scorer, applying the size gate first. +RequestClass classify_with_table(Scorer& scorer, const ModelTable& table, const std::string& state); + +} // namespace onebit::laya diff --git a/tests/fake_backend_route.py b/tests/fake_backend_route.py index 1eb54e12d..7c5252061 100755 --- a/tests/fake_backend_route.py +++ b/tests/fake_backend_route.py @@ -36,6 +36,8 @@ def device(): dev = device() +# the model file's stem, so a test serving several models (tests/laya_models_e2e.sh) sees which answered +model = argv[argv.index("-m") + 1].rsplit("/", 1)[-1].rsplit(".", 1)[0] if "-m" in argv else "fake" port = None for i, a in enumerate(argv): if a in ("--port", "-p") and i + 1 < len(argv): @@ -65,10 +67,13 @@ def do_GET(self): def do_POST(self): n = int(self.headers.get("Content-Length", 0)) req = json.loads(self.rfile.read(n)) if n else {} - text = "device:" + dev + text = "device:" + dev + " model:" + model if not req.get("stream"): msg = {"message": {"role": "assistant", "content": text}} if "chat" in self.path else {"text": text} - self._json(200, {"id": "x", "model": "fake", "choices": [dict(index=0, finish_reason="stop", **msg)]}) + # usage and timings in llama-server's shape (fixed numbers: the test checks they are relayed) + self._json(200, {"id": "x", "model": "fake", "choices": [dict(index=0, finish_reason="stop", **msg)], + "usage": {"prompt_tokens": 12, "completion_tokens": 3, "total_tokens": 15}, + "timings": {"prompt_per_second": 400.0, "predicted_per_second": 50.0}}) return self.send_response(200) self.send_header("Content-Type", "text/event-stream") diff --git a/tests/laya_models_e2e.sh b/tests/laya_models_e2e.sh new file mode 100755 index 000000000..785994f20 --- /dev/null +++ b/tests/laya_models_e2e.sh @@ -0,0 +1,103 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# laya_models_e2e.sh: `1bit serve --laya-models` end to end without a GPU (docs/laya-models.md). +# Two rows, two fake llama-servers (tests/fake_backend_route.py, which names the model file that +# answered), each behind its own `1bit serve` child. The table's size gate sends long requests to +# `reflex` and min_confidence 1.01 sends every other one to the default `plan`, so where a request +# lands is known whatever Laya answers, while Laya is still asked (laya_class, router_ms > 0). +# Checks: the reply comes from the row the header names; X-1bit-Route / -Model / -Ms / -Id; the +# reply's `route` object; a request naming a row's alias skips Laya; a repeated state is cached; +# GET /v1/route/decisions, /v1/route/table, /v1/models and /metrics; refusals by name. +set -uo pipefail + +bin=${1:?usage: laya_models_e2e.sh path/to/1bit path/to/laya-model-dir} +laya_model=${2:?usage: laya_models_e2e.sh path/to/1bit path/to/laya-model-dir} +here=$(cd "$(dirname "$0")" && pwd) +scratch=$(mktemp -d) +touch "$scratch/small.gguf" "$scratch/big.gguf" +fake="$here/fake_backend_route.py" +cat >"$scratch/table.json" <&1) +check "--laya-models without --laya says so" '[[ "$out" == *"needs --laya"* ]]' +out=$("$bin" serve --laya-models "$scratch/table.json" --laya-model "$laya_model" -m "$scratch/big.gguf" --port "$port" 2>&1) +check "--laya-models with -m is refused" '[[ "$out" == *"drop -m"* ]]' +echo '{"question":{"instructions":"x","classes":{"a":"d","b":"e"}},"classes":{"a":{"model":"m","device":"hrx"}}}' >"$scratch/bad.json" +out=$("$bin" serve --laya-models "$scratch/bad.json" --laya-model "$laya_model" --port "$port" 2>&1) +check "a table with a class and no row is refused by name" '[[ "$out" == *"\"b\" has no row"* ]]' + +"$bin" serve --laya-models "$scratch/table.json" --laya-model "$laya_model" --port "$port" >"$scratch/serve.log" 2>&1 & +pid=$! +cleanup() { kill "$pid" 2>/dev/null; wait "$pid" 2>/dev/null; rm -rf "$scratch"; } +trap cleanup EXIT +for _ in $(seq 1 1200); do [ "$(curl -s -o /dev/null -w '%{http_code}' "$api/health")" = 200 ] && break; sleep 0.1; done +check "ready" '[ "$(curl -s -o /dev/null -w "%{http_code}" "$api/health")" = 200 ]' + +ask() { # model, text -> headers in $scratch/h, body in $scratch/b + python3 -c 'import json,sys; print(json.dumps({"model":sys.argv[1],"messages":[{"role":"system","content":"You play a voxel game."},{"role":"user","content":sys.argv[2]}]}))' "$1" "$2" >"$scratch/req.json" + curl -s -D "$scratch/h" -o "$scratch/b" "$api/v1/chat/completions" -H 'Content-Type: application/json' -d @"$scratch/req.json" + tr -d '\r' <"$scratch/h" >"$scratch/h2" +} +hdr() { sed -n "s/^[Xx]-1bit-$1: //Ip" "$scratch/h2"; } +field() { python3 -c 'import json,sys; d=json.load(open(sys.argv[1])); print(eval(sys.argv[2]))' "$scratch/b" "$1"; } + +ask auto "A tree is in front of you. What do you do now?" +check "short step -> plan row (min_confidence fallback): reply from big" 'grep -q "model:big" "$scratch/b"' +check " X-1bit-Route ($(hdr Route))" '[[ "$(hdr Route)" == "plan "*" cpu" ]]' +check " X-1bit-Route-Model ($(hdr Route-Model))" '[ "$(hdr Route-Model)" = big-hrx ]' +check " X-1bit-Route-Ms > 0 ($(hdr Route-Ms))" 'python3 -c "import sys; sys.exit(0 if float(sys.argv[1]) > 0 else 1)" "$(hdr Route-Ms)"' +check " route object: Laya was asked ($(field "d[\"route\"][\"laya_class\"]"))" '[ -n "$(field "d[\"route\"][\"laya_class\"]")" ]' +check " reply model is the row alias" '[ "$(field "d[\"model\"]")" = big-hrx ]' + +long=$(python3 -c 'print("Inventory: " + ", ".join(["oak log"] * 40))') +ask auto "$long" +check "long step -> size gate reflex: reply from small" 'grep -q "model:small" "$scratch/b"' +check " route class reflex, confidence 1" '[ "$(field "d[\"route\"][\"class\"]")" = reflex ] && [ "$(field "d[\"route\"][\"confidence\"]")" = 1.0 ]' + +ask small-npu "A tree is in front of you. What do you do now?" +check "a row alias is pinned: reply from small, no Laya" 'grep -q "model:small" "$scratch/b" && [[ "$(hdr Route)" == "pinned "* ]]' + +ask auto "A tree is in front of you. What do you do now?" +check "a repeated state is cached" '[ "$(field "d[\"route\"][\"cached\"]")" = True ] && [ "$(hdr Route-Ms)" = 0.0 ]' + +dec=$(curl -s "$api/v1/route/decisions?since=0") +check "decisions: 4 recorded" '[ "$(echo "$dec" | python3 -c "import json,sys; print(len(json.load(sys.stdin)[\"data\"]))")" = 4 ]' +check "decisions: usage and timings relayed" 'echo "$dec" | python3 -c "import json,sys; d=json.load(sys.stdin)[\"data\"][0]; sys.exit(0 if d[\"completion_tokens\"]==3 and d[\"decode_tok_s\"]==50.0 and d[\"request_ms\"]>0 else 1)"' +check "decisions: since filters" '[ "$(curl -s "$api/v1/route/decisions?since=3" | python3 -c "import json,sys; print(len(json.load(sys.stdin)[\"data\"]))")" = 1 ]' +tab=$(curl -s "$api/v1/route/table") +check "table: two classes with their rows" 'echo "$tab" | python3 -c "import json,sys; t=json.load(sys.stdin); sys.exit(0 if [c[\"model\"] for c in t[\"classes\"]]==[\"small-npu\",\"big-hrx\"] else 1)"' +check "/v1/models lists auto and both aliases" '[ "$(curl -s "$api/v1/models" | python3 -c "import json,sys; print(\" \".join(m[\"id\"] for m in json.load(sys.stdin)[\"data\"]))")" = "auto small-npu big-hrx" ]' +met=$(curl -s "$api/metrics") +check "metrics: decisions by class/model/device" '[[ "$met" == *"onebit_route_decisions_total{class=\"plan\",model=\"big-hrx\",device=\"cpu\"} 2"* ]]' +check "metrics: router ms counted for Laya decisions only" '[[ "$met" == *"onebit_route_router_count{class=\"plan\",model=\"big-hrx\",device=\"cpu\"} 1"* ]]' +check "metrics: last decode tok/s" '[[ "$met" == *"onebit_route_last_decode_tok_s{model=\"small-npu\",device=\"cpu\"} 50"* ]]' +check "serve logged every decision" '[ "$(grep -c "laya-models: #[0-9]" "$scratch/serve.log")" = 4 ]' + +if [ $fail -ne 0 ]; then echo "--- serve log"; cat "$scratch/serve.log"; echo FAIL; exit 1; fi +echo PASS diff --git a/tests/laya_models_test.cpp b/tests/laya_models_test.cpp new file mode 100644 index 000000000..67de59f50 --- /dev/null +++ b/tests/laya_models_test.cpp @@ -0,0 +1,106 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +// The model table of `1bit serve --laya-models` (laya/models.h): parsing, the Laya question it +// builds, the pick with its fallback, aliases and shared backends, and the refusals by name. +// Needs no checkpoint: argv[1] is the shipped config/laya-models-game.json, checked as well. +#include "models.h" + +#include +#include + +namespace { +int failures = 0; +void check(const std::string& what, bool ok) { + std::printf("%s %s\n", ok ? "PASS" : "FAIL", what.c_str()); + if (!ok) failures++; +} + +const char* kTable = R"({ + "question": {"instructions": "Which?", "classes": {"reflex": "quick", "plan": "slow", "chat": "talk"}}, + "default": "plan", "min_confidence": 0.2, + "size_gate": {"chars": 100, "class": "plan"}, + "classes": { + "reflex": {"model": "small.gguf", "device": "npu", "alias": "small"}, + "plan": {"model": "/abs/big.gguf", "device": "hrx", "args": ["--ctx-size", "4096"], "bin": "rel/1bit"}, + "chat": {"model": "/abs/big.gguf", "device": "hrx", "args": ["--ctx-size", "4096"], "bin": "rel/1bit"} + }})"; + +bool refuses(const std::string& text, const std::string& needle) { + onebit::laya::ModelTable t; + std::string err; + const bool ok = t.parse(text, "/base", err); + return !ok && err.find(needle) != std::string::npos; +} +} // namespace + +int main(int argc, char** argv) { + using namespace onebit::laya; + ModelTable t; + std::string err; + check("parses (" + err + ")", t.parse(kTable, "/base", err)); + check("question keeps the file's class order", t.classes.size() == 3 && t.classes[0].first == "reflex" && + t.classes[1].first == "plan" && t.classes[2].first == "chat"); + const Question q = t.laya_question(); + check("question is one choice with every class", q.type == "choice" && q.instructions == "Which?" && q.criteria.size() == 3); + check("relative model resolves against the table dir", t.pick("reflex", 0.9f).model == "/base/small.gguf"); + check("alias defaults to the file name", t.pick("plan", 0.9f).alias == "big.gguf"); + std::string used; + check("confident reflex -> npu", t.pick("reflex", 0.5f, &used).device == "npu" && used == "reflex"); + check("unsure reflex -> fallback plan", t.pick("reflex", 0.1f, &used).device == "hrx" && used == "plan"); + check("empty class -> fallback", t.pick("", 0.9f, &used).device == "hrx" && used == "plan"); + check("by_alias finds a row", t.by_alias("small") && t.by_alias("small")->device == "npu"); + check("by_alias misses an unknown name", t.by_alias("auto") == nullptr); + check("two classes share one backend", t.backends().size() == 2 && + t.backend_index(t.pick("plan", 1.f)) == t.backend_index(t.pick("chat", 1.f))); + check("row_for is the class's own row", t.row_for("reflex") && t.row_for("reflex")->device == "npu" && !t.row_for("x")); + check("size_gated by length", t.size_gated(std::string(100, 'a')) && !t.size_gated(std::string(99, 'a'))); + check("bin resolves against the table dir", t.pick("plan", 1.f).bin == "/base/rel/1bit" && t.pick("reflex", 1.f).bin.empty()); + check("size gate read", t.size_gate_chars == 100 && t.size_gate_class == "plan"); + + check("refuses one class", refuses(R"({"question":{"instructions":"x","classes":{"a":"d"}},"classes":{"a":{"model":"m","device":"hrx"}}})", + "at least two")); + check("refuses a class without a row", + refuses(R"({"question":{"instructions":"x","classes":{"a":"d","b":"e"}},"classes":{"a":{"model":"m","device":"hrx"}}})", + "\"b\" has no row")); + check("refuses an unknown device", + refuses(R"({"question":{"instructions":"x","classes":{"a":"d","b":"e"}},"classes":{"a":{"model":"m","device":"vulkan"},"b":{"model":"m","device":"hrx"}}})", + "unknown device")); + check("refuses a row for no class", + refuses(R"({"question":{"instructions":"x","classes":{"a":"d","b":"e"}},"classes":{"a":{"model":"m","device":"hrx"},"b":{"model":"m","device":"hrx"},"c":{"model":"m","device":"hrx"}}})", + "not one of the question's classes")); + check("refuses an unknown default", + refuses(R"({"question":{"instructions":"x","classes":{"a":"d","b":"e"}},"default":"z","classes":{"a":{"model":"m","device":"hrx"},"b":{"model":"m","device":"hrx"}}})", + "\"default\"")); + check("refuses an empty bin", + refuses(R"({"question":{"instructions":"x","classes":{"a":"d","b":"e"}},"classes":{"a":{"model":"m","device":"hrx","bin":""},"b":{"model":"m","device":"hrx"}}})", + "\"bin\"")); + check("refuses one alias for two backends", + refuses(R"({"question":{"instructions":"x","classes":{"a":"d","b":"e"}},"classes":{"a":{"model":"m","device":"hrx","alias":"s"},"b":{"model":"n","device":"npu","alias":"s"}}})", + "one alias")); + + if (argc > 1) { + try { + const ModelTable g = load_model_table(argv[1]); + check("shipped game table: reflex on npu, plan on hrx", + g.pick("reflex", 1.f).device == "npu" && g.pick("plan", 1.f).device == "hrx"); + check("shipped game table: default plan", g.fallback == "plan"); + } catch (const std::exception& e) { + check(std::string("shipped game table loads: ") + e.what(), false); + } + } + std::printf("%s\n", failures ? "FAIL" : "PASS"); + return failures ? 1 : 0; +} diff --git a/tests/laya_route_e2e.sh b/tests/laya_route_e2e.sh index 3e9e55a0e..1d178b1a3 100755 --- a/tests/laya_route_e2e.sh +++ b/tests/laya_route_e2e.sh @@ -82,10 +82,10 @@ min_distinct=1 check "the policy routed to $distinct distinct devices" '[ "$distinct" -ge "$min_distinct" ]' # a second turn of the first conversation: same class, no new decision -decisions_before=$(grep -c "1bit serve: laya: [a-z_]* [0-9.]* " "$scratch/serve.log") +decisions_before=$(grep -c "1bit serve: laya: .* | classified " "$scratch/serve.log") body=$(python3 -c 'import json,sys; print(json.dumps({"messages":[{"role":"user","content":sys.argv[1]},{"role":"assistant","content":"fn reverse() {}"},{"role":"user","content":"thanks, now add a test"}]}))' "${states[0]}") curl -s -o "$scratch/reply2.json" "$api/v1/chat/completions" -H 'Content-Type: application/json' -d "$body" -decisions_after=$(grep -c "1bit serve: laya: [a-z_]* [0-9.]* " "$scratch/serve.log") +decisions_after=$(grep -c "1bit serve: laya: .* | classified " "$scratch/serve.log") first=$(echo "$seen" | awk '{print $1}') check "second turn reuses the decision (decisions $decisions_before -> $decisions_after)" '[ "$decisions_after" = "$decisions_before" ]' check " and goes to the same device ($first)" 'grep -q "device:$first" "$scratch/reply2.json"'