From 966ea9ade6c7285c43f91de344a7e059527b2e06 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sat, 7 Nov 2026 01:41:56 -0400 Subject: [PATCH] =?UTF-8?q?route:=20--laya-models=20--eval=20and=20--dump-?= =?UTF-8?q?encoder=20=E2=80=94=20a=20model=20table's=20routing=20measured,?= =?UTF-8?q?=20the=20head's=20input=20written=20(Refs=20#1313)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `1bit route --laya-models FILE --eval CASES.jsonl [--json] [--per-case F]` runs the table's question over labelled states by the path serve takes (size gate, then Laya) and reports accuracy, per-class recall and precision, the confusion matrix, confidence, ECE, log loss and decision time (laya/eval.cpp, tested without a checkpoint). `--dump-encoder CASES.jsonl --out DIR [--rotate]` writes the decision head's input per state: the encoder output after the final norm (h.f32), marker positions, qtype and the engine's own logits and probabilities (cases.jsonl), so a head-only fine-tune needs neither the encoder nor a second tokenizer, and can check that its head reproduces the engine first. Scorer: RawOutput::keep_encoder (opt-in; the default path is unchanged). Co-Authored-By: Claude Opus 5.5 --- CMakeLists.txt | 11 +- app/main.cpp | 27 ++++- app/route_eval.cpp | 223 +++++++++++++++++++++++++++++++++++++++ app/route_eval.h | 39 +++++++ docs/laya-models.md | 44 ++++++++ laya/eval.cpp | 145 +++++++++++++++++++++++++ laya/eval.h | 57 ++++++++++ laya/scorer.cpp | 11 ++ laya/scorer.h | 8 ++ tests/laya_eval_e2e.sh | 95 +++++++++++++++++ tests/laya_eval_test.cpp | 83 +++++++++++++++ 11 files changed, 740 insertions(+), 3 deletions(-) create mode 100644 app/route_eval.cpp create mode 100644 app/route_eval.h create mode 100644 laya/eval.cpp create mode 100644 laya/eval.h create mode 100755 tests/laya_eval_e2e.sh create mode 100644 tests/laya_eval_test.cpp diff --git a/CMakeLists.txt b/CMakeLists.txt index 17dd2c52d..e20058fda 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -94,7 +94,7 @@ file(WRITE ${CMAKE_BINARY_DIR}/generated/laya_first_default.h "// generated from config/laya-first.json by CMakeLists.txt\n#pragma once\n" "static const char kDefaultLayaFirstJson[] = R\"ONEBITLAYAFIRST(${ONEBIT_LAYA_FIRST_JSON})ONEBITLAYAFIRST\";\n") set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${CMAKE_SOURCE_DIR}/config/laya-first.json) -add_executable(onebit app/main.cpp app/serve.cpp app/recipes.cpp app/pm.cpp app/route_serve.cpp app/laya_checkpoint.cpp app/laya_first.cpp) +add_executable(onebit app/main.cpp app/serve.cpp app/recipes.cpp app/pm.cpp app/route_serve.cpp app/route_eval.cpp app/laya_checkpoint.cpp app/laya_first.cpp) # config/recipes.json (tuned llama-server settings per model and route, docs/recipes.md) is # compiled in as serve's default recipes; --recipes replaces them at run time. file(READ ${CMAKE_SOURCE_DIR}/config/recipes.json ONEBIT_RECIPES_JSON) @@ -534,7 +534,7 @@ file(WRITE ${CMAKE_BINARY_DIR}/generated/route_policy_default.h "static const char kDefaultRoutePolicyJson[] = R\"ONEBITPOLICY(${ONEBIT_ROUTE_POLICY_JSON})ONEBITPOLICY\";\n") set_property(DIRECTORY APPEND PROPERTY CMAKE_CONFIGURE_DEPENDS ${CMAKE_SOURCE_DIR}/config/route-policy.json) add_library(onebit_laya STATIC laya/scorer.cpp laya/safetensors.cpp laya/route.cpp laya/policy.cpp laya/encoder.cpp - laya/models.cpp) + laya/models.cpp laya/eval.cpp) target_include_directories(onebit_laya PUBLIC laya PRIVATE ${CMAKE_BINARY_DIR}/generated) target_link_libraries(onebit_laya PUBLIC onebit_npu nlohmann_json::nlohmann_json) # The engine binary gets `1bit route` and per-request routing through the scorer. @@ -560,6 +560,11 @@ add_test(NAME laya_models COMMAND laya_models_test ${CMAKE_SOURCE_DIR}/config/la # CI builds `--target onebit` plus a fixed list in .github/workflows/ci.yml; building this check with onebit # keeps the laya_models ctest from reporting "Not Run" without a workflow change. add_dependencies(onebit laya_models_test) +# The routing eval's arithmetic (laya/eval.cpp): accuracy, recall, precision, confusion, ECE, log loss. +add_executable(laya_eval_test tests/laya_eval_test.cpp) +target_link_libraries(laya_eval_test PRIVATE onebit_laya nlohmann_json::nlohmann_json) +add_test(NAME laya_eval COMMAND laya_eval_test) +add_dependencies(onebit laya_eval_test) # Crafted Laya checkpoints the reader and the scorer must refuse (written at run time); runs in CI. add_executable(laya_bounds_test tests/laya_bounds_test.cpp) target_link_libraries(laya_bounds_test PRIVATE onebit_laya nlohmann_json::nlohmann_json) @@ -583,6 +588,8 @@ if(ONEBIT_LAYA_ROUTE_MODEL) add_test(NAME laya_route_e2e COMMAND ${CMAKE_SOURCE_DIR}/tests/laya_route_e2e.sh $ ${ONEBIT_LAYA_ROUTE_MODEL}) # --laya-models: model auto picked per request among the table's rows (docs/laya-models.md) add_test(NAME laya_models_e2e COMMAND ${CMAKE_SOURCE_DIR}/tests/laya_models_e2e.sh $ ${ONEBIT_LAYA_ROUTE_MODEL}) + # `1bit route --laya-models --eval / --dump-encoder`: the table's routing measured, the head's input written + add_test(NAME laya_eval_e2e COMMAND ${CMAKE_SOURCE_DIR}/tests/laya_eval_e2e.sh $ ${ONEBIT_LAYA_ROUTE_MODEL}) # `1bit route --serve`, the resident form the fork calls per request (docs/laya-first.md ยง8). add_test(NAME laya_route_serve COMMAND ${CMAKE_SOURCE_DIR}/tests/laya_route_serve.sh $ ${ONEBIT_LAYA_ROUTE_MODEL}) # One checkpoint resolution for serve --laya and 1bit route (app/laya_checkpoint.h). diff --git a/app/main.cpp b/app/main.cpp index 33b557d94..4cefcafaf 100644 --- a/app/main.cpp +++ b/app/main.cpp @@ -57,6 +57,7 @@ int onebit_ort_run_main(int argc, char** argv); #include "policy.h" #include "route.h" #include "route_serve.h" +#include "route_eval.h" #include "scorer.h" #endif #include "serve.h" @@ -175,6 +176,7 @@ int run_npu(int argc, char** argv) { int run_route(int argc, char** argv) { std::string laya_model, state, policy_path, devices_str = "npu,hrx,cpu"; bool classify = false, serve = false, json_out = false; + onebit::RouteEvalOptions ev; for (int i = 0; i < argc; ++i) { const std::string a = argv[i]; auto next = [&]() -> std::string { @@ -188,6 +190,12 @@ int run_route(int argc, char** argv) { else if (a == "--classify") classify = true; else if (a == "--json") json_out = true; else if (a == "--serve") serve = true; + else if (a == "--laya-models") ev.table = next(); + else if (a == "--eval") ev.eval_cases = next(); + else if (a == "--dump-encoder") ev.dump_cases = next(); + else if (a == "--out") ev.out_dir = next(); + else if (a == "--per-case") ev.per_case = next(); + else if (a == "--rotate") ev.rotate = true; else if (a == "--help" || a == "-h") { std::printf("usage: 1bit route [--laya-model ] --state [--devices npu,hrx,cpu]\n" " [--route-policy FILE] [--classify]\n" @@ -198,10 +206,27 @@ int run_route(int argc, char** argv) { " [--route-policy FILE]\n" " Resident: loads the scorer once, then reads one {\"state\": \"\"} per line\n" " on stdin and writes one {\"class\",\"confidence\",\"device\",\"decision_ms\"}\n" - " per line on stdout, after a first {\"ready\":true,\"load_ms\":N} line; EOF exits\n"); + " per line on stdout, after a first {\"ready\":true,\"load_ms\":N} line; EOF exits\n" + "usage: 1bit route --laya-models FILE [--laya-model ] --eval CASES.jsonl [--json]\n" + " [--per-case OUT.jsonl]\n" + " The table's question over labelled states ({\"state\",\"label\"} per line), the path\n" + " serve --laya-models takes: accuracy, per-class recall and precision, confusion,\n" + " confidence, ECE, log loss and decision time\n" + "usage: 1bit route --laya-models FILE [--laya-model ] --dump-encoder CASES.jsonl\n" + " --out DIR [--rotate]\n" + " Writes the decision head's input for each case (DIR/h.f32, DIR/cases.jsonl,\n" + " DIR/dump.json) for a head-only fine-tune; --rotate adds every option order\n"); return 0; } else throw std::runtime_error("unknown option " + a); } + if (!ev.table.empty()) { + if (serve || !state.empty() || classify) + throw std::runtime_error("--laya-models takes --eval or --dump-encoder, not --state, --classify or --serve"); + ev.laya_model = laya_model; + ev.json = json_out; + return onebit::run_route_eval(ev); + } + if (!ev.eval_cases.empty() || !ev.dump_cases.empty()) throw std::runtime_error("--eval and --dump-encoder need --laya-models FILE"); if (!serve && state.empty()) throw std::runtime_error("--state is required"); std::vector devices; diff --git a/app/route_eval.cpp b/app/route_eval.cpp new file mode 100644 index 000000000..320b81191 --- /dev/null +++ b/app/route_eval.cpp @@ -0,0 +1,223 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +#include "route_eval.h" + +#include +#include +#include +#include +#include +#include +#include + +#include + +#include "eval.h" +#include "laya_checkpoint.h" +#include "models.h" +#include "route.h" +#include "scorer.h" + +namespace onebit { +namespace { + +struct Case { + std::string state; + std::string label; // empty: unlabelled (dump only) + std::string src; +}; + +// One JSON object per line: {"state": TEXT, "label": CLASS, "src": optional id}. The harness's +// decisions.jsonl lines carry the same "state" and "label" keys, so they are cases as they are. +std::vector read_cases(const std::string& path, bool need_label) { + std::ifstream in(path); + if (!in) throw std::runtime_error("cannot read " + path); + std::vector out; + std::string line; + int n = 0; + while (std::getline(in, line)) { + ++n; + if (line.find_first_not_of(" \t\r") == std::string::npos) continue; + nlohmann::json j; + try { + j = nlohmann::json::parse(line); + } catch (const std::exception& e) { + throw std::runtime_error(path + ":" + std::to_string(n) + ": " + e.what()); + } + if (!j.is_object() || !j.contains("state") || !j["state"].is_string()) + throw std::runtime_error(path + ":" + std::to_string(n) + ": needs \"state\": TEXT"); + Case c; + c.state = j["state"].get(); + if (j.contains("label") && j["label"].is_string()) c.label = j["label"].get(); + if (need_label && c.label.empty()) { + // an unlabelled decision (no outcome yet) is not a test case + continue; + } + c.src = j.value("src", path + ":" + std::to_string(n)); + out.push_back(std::move(c)); + } + return out; +} + +double ms_since(std::chrono::steady_clock::time_point t) { + return std::chrono::duration(std::chrono::steady_clock::now() - t).count(); +} + +int index_of(const std::vector>& v, const std::string& k) { + for (size_t i = 0; i < v.size(); ++i) + if (v[i].first == k) return (int)i; + return -1; +} + +int run_eval(laya::Scorer& scorer, const laya::ModelTable& table, const RouteEvalOptions& o) { + const std::vector cases = read_cases(o.eval_cases, true); + if (cases.empty()) throw std::runtime_error(o.eval_cases + ": no labelled cases"); + std::vector classes; + for (const auto& [k, d] : table.classes) { + (void)d; + classes.push_back(k); + } + std::ofstream per; + if (!o.per_case.empty()) { + per.open(o.per_case); + if (!per) throw std::runtime_error("cannot write " + o.per_case); + } + std::vector got; + for (const Case& c : cases) { + if (index_of(table.classes, c.label) < 0) + throw std::runtime_error(c.src + ": label \"" + c.label + "\" is not one of the table's classes"); + const auto t = std::chrono::steady_clock::now(); + // the path serve takes: the size gate, then the table's question + const laya::RequestClass rc = laya::classify_with_table(scorer, table, c.state); + laya::EvalCase e; + e.ms = ms_since(t); + if (rc.label.empty()) throw std::runtime_error(c.src + ": " + scorer.error()); + e.label = c.label; + e.got = rc.label; + e.confidence = rc.confidence; + e.probabilities = rc.probabilities; + e.gated = table.size_gated(c.state); + if (per.is_open()) { + nlohmann::ordered_json j{{"src", c.src}, {"label", e.label}, {"chosen", e.got}, + {"confidence", e.confidence}, {"gated", e.gated}, {"decision_ms", e.ms}}; + nlohmann::ordered_json p = nlohmann::ordered_json::object(); + for (const auto& [k, v] : e.probabilities) p[k] = v; + j["probabilities"] = p; + per << j.dump() << '\n'; + } + got.push_back(std::move(e)); + } + const laya::EvalReport r = laya::evaluate(got, classes); + if (o.json) std::printf("%s\n", laya::report_json(r).c_str()); + else std::printf("%s", laya::report_text(r).c_str()); + return 0; +} + +// DIR/h.f32: every case's encoder rows (float32, little endian, `d` per row) back to back. +// DIR/cases.jsonl: one line per (case, rotation): src, label index (-1 unlabelled), the options in the +// order the sequence carried them, qtype, row offset and length, marker positions, and the engine's own +// logits and probabilities, so a trainer can check that its head reproduces the engine before training. +// DIR/dump.json: d, rows, cases, skipped (size-gated) and the table's question. +int run_dump(laya::Scorer& scorer, const laya::ModelTable& table, const RouteEvalOptions& o) { + namespace fs = std::filesystem; + if (o.out_dir.empty()) throw std::runtime_error("--dump-encoder needs --out DIR"); + const std::vector cases = read_cases(o.dump_cases, false); + fs::create_directories(o.out_dir); + std::ofstream h(o.out_dir + "/h.f32", std::ios::binary); + std::ofstream meta(o.out_dir + "/cases.jsonl"); + if (!h || !meta) throw std::runtime_error("cannot write in " + o.out_dir); + const laya::Question base = table.laya_question(); + const size_t k = base.criteria.size(); + long long rows = 0; + int written = 0, skipped = 0, d = 0; + for (const Case& c : cases) { + if (table.size_gated(c.state)) { // serve never asks Laya about these, so the head never sees them + ++skipped; + continue; + } + const int y = c.label.empty() ? -1 : index_of(base.criteria, c.label); + if (!c.label.empty() && y < 0) + throw std::runtime_error(c.src + ": label \"" + c.label + "\" is not one of the table's classes"); + for (size_t rot = 0; rot < (o.rotate ? k : 1); ++rot) { + laya::Question q = base; + for (size_t i = 0; i < k; ++i) q.criteria[i] = base.criteria[(i + rot) % k]; + laya::RawOutput raw; + raw.keep_encoder = true; + std::vector answers; + if (!scorer.score(c.state, {q}, answers, &raw) || answers.empty()) + throw std::runtime_error(c.src + ": " + scorer.error()); + d = raw.d; + const std::vector& enc = raw.encoder.front(); + const long long len = (long long)(enc.size() / (size_t)d); + h.write(reinterpret_cast(enc.data()), (std::streamsize)(enc.size() * sizeof(float))); + std::vector order; + for (const auto& [key, desc] : q.criteria) { + (void)desc; + order.push_back(key); + } + nlohmann::ordered_json probs = nlohmann::ordered_json::array(); + for (const auto& key : order) { + float p = 0.0f; + for (const auto& [lk, lp] : answers.front().probabilities) + if (lk == key) p = lp; + probs.push_back(p); + } + std::vector logits(raw.logits.front().begin(), raw.logits.front().begin() + (long)k); + nlohmann::ordered_json j{{"src", c.src}, + {"rotation", rot}, + {"label", y < 0 ? -1 : (int)((y + k - rot) % k)}, + {"options", order}, + {"qtype", raw.qtype.front()}, + {"offset", rows}, + {"len", len}, + {"markers", raw.markers.front()}, + {"logits", logits}, + {"probabilities", probs}}; + meta << j.dump() << '\n'; + rows += len; + ++written; + } + } + nlohmann::ordered_json info{{"d", d}, + {"rows", rows}, + {"cases", written}, + {"skipped_size_gated", skipped}, + {"rotate", o.rotate}, + {"question", table.question}, + {"laya_model", o.laya_model}}; + std::ofstream(o.out_dir + "/dump.json") << info.dump(1) << '\n'; + std::fprintf(stderr, "1bit route: %d sequences, %lld rows of %d floats in %s (%d size-gated cases skipped)\n", + written, rows, d, o.out_dir.c_str(), skipped); + return 0; +} + +} // namespace + +int run_route_eval(const RouteEvalOptions& in) { + RouteEvalOptions o = in; + if (o.eval_cases.empty() == o.dump_cases.empty()) + throw std::runtime_error("--laya-models takes exactly one of --eval CASES and --dump-encoder CASES"); + const laya::ModelTable table = laya::load_model_table(o.table); + o.laya_model = laya_checkpoint(o.laya_model); + laya::Scorer scorer; + const auto t = std::chrono::steady_clock::now(); + if (!scorer.load(o.laya_model)) throw std::runtime_error(scorer.error()); + std::fprintf(stderr, "1bit route: scorer %s loaded in %.0f ms\n", o.laya_model.c_str(), ms_since(t)); + return o.eval_cases.empty() ? run_dump(scorer, table, o) : run_eval(scorer, table, o); +} + +} // namespace onebit diff --git a/app/route_eval.h b/app/route_eval.h new file mode 100644 index 000000000..0a74da08a --- /dev/null +++ b/app/route_eval.h @@ -0,0 +1,39 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +// `1bit route --laya-models FILE --eval CASES` and `--dump-encoder CASES --out DIR`: a model table's +// routing measured over labelled states, and the decision head's input written out for a head-only +// fine-tune (docs/laya-models.md, "Measuring and fine-tuning the router"). +#pragma once + +#include + +namespace onebit { + +struct RouteEvalOptions { + std::string laya_model; // --laya-model (resolved by laya_checkpoint) + std::string table; // --laya-models + std::string eval_cases; // --eval + std::string dump_cases; // --dump-encoder + std::string out_dir; // --out (dump) + std::string per_case; // --per-case FILE (eval): one JSON line per case + bool json = false; // --json (eval) + bool rotate = false; // --rotate (dump): every rotation of the options, labels moved with them +}; + +int run_route_eval(const RouteEvalOptions& o); + +} // namespace onebit diff --git a/docs/laya-models.md b/docs/laya-models.md index f6bfc005c..833505d91 100644 --- a/docs/laya-models.md +++ b/docs/laya-models.md @@ -111,10 +111,54 @@ numbers a demo HUD reads; every one was measured by serve or reported by the bac table's accuracy is measured below, and a fine-tune on the game's own logged decisions is the plan for raising it (engine #1313). +## Measuring and fine-tuning the router + +```sh +# accuracy of the table's question over labelled states, by the path serve takes (size gate, then Laya) +1bit route --laya-models table.json --laya-model DIR --eval cases.jsonl [--json] [--per-case per.jsonl] +# the decision head's input for each state, for a head-only fine-tune +1bit route --laya-models table.json --laya-model DIR --dump-encoder cases.jsonl --out dump/ [--rotate] +``` + +A case is one JSON line, `{"state": TEXT, "label": CLASS, "src": optional id}`, where `state` is exactly what +serve classified (the last user message). A line without a `label` is skipped by `--eval`, so a decision log +whose newest lines have no outcome yet can be passed as it is. + +`--eval` prints accuracy, per-class support, recall and precision, the confusion matrix (rows true, columns +chosen), serve's mean confidence, the mean top probability, ECE (10 equal bins over the top probability), log +loss and the decision time p50 and p90. Size-gated cases are counted (`gated`), since serve would gate them too. + +`--dump-encoder` writes three files to `--out`: + +| file | content | +|---|---| +| `h.f32` | every sequence's encoder output after the final norm, float32 little-endian rows of `d`, back to back | +| `cases.jsonl` | per sequence: `src`, `rotation`, `label` (an index into `options`, -1 unlabelled), `options` (the order the sequence carried them), `qtype`, `offset` and `len` (rows in `h.f32`), `markers`, and the engine's own `logits` and `probabilities` | +| `dump.json` | `d`, `rows`, `cases`, `skipped_size_gated`, `rotate`, the question and the checkpoint | + +The head (`type_emb`, the two head layers and `scorer`) maps these rows to the option logits, so a trainer +needs neither the encoder nor a second tokenizer. The engine's logits in each line let it check first that its +head reproduces the engine. Size-gated cases are skipped because serve never asks Laya about them. `--rotate` +writes every rotation of the options with the label moved along, so a fine-tune cannot learn "the first option +wins". + +Measured on this box (`typed-decisions/`, the 20 game steps of the measurement below, the CPU scorer under +`thermal-run`, 4 cores at nice 19): + +| | result | +|---|---| +| `--eval` | 19/20, the same single miss as serve's run below; reflex recall 0.917, plan recall 1.000; mean confidence 0.154, ECE 0.248, log loss 0.373; decision p50 1159 ms on 4 cores at nice 19 | +| `--dump-encoder --rotate` | 40 sequences, 4,404 rows (18 MB) | +| a torch copy of the head over the dump | reproduces the engine's probabilities within 2.4e-7 (logits 1.3e-6) on all 40 | +| option order | mean \|p(reflex) in one order - the other\| 0.065 (max 0.171); 1 of 20 decisions flips with the order | + ## Tests `tests/laya_models_test.cpp` (ctest `laya_models`, no checkpoint): parsing, the question, the pick and its fallback, aliases, shared backends, the size gate, the refusals, and the shipped game table. +`tests/laya_eval_test.cpp` (ctest `laya_eval`, no checkpoint): the eval's arithmetic on hand-computed cases. +`tests/laya_eval_e2e.sh` (ctest `laya_eval_e2e`, with `ONEBIT_LAYA_ROUTE_MODEL`): `--eval` and `--dump-encoder` +on the real scorer, the report's counts, the dump's files against each other, the refusals. `tests/laya_models_e2e.sh` (ctest `laya_models_e2e`, with `ONEBIT_LAYA_ROUTE_MODEL`): two rows on fake llama-servers behind real `1bit serve` children; the reply comes from the row the headers name, pinned and cached requests, the decisions log, the table, `/v1/models` and `/metrics`, and the refusals. diff --git a/laya/eval.cpp b/laya/eval.cpp new file mode 100644 index 000000000..0af29a5af --- /dev/null +++ b/laya/eval.cpp @@ -0,0 +1,145 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +#include "eval.h" + +#include +#include +#include +#include + +#include + +namespace onebit::laya { + +namespace { + +int index_of(const std::vector& v, const std::string& s) { + const auto it = std::find(v.begin(), v.end(), s); + return it == v.end() ? -1 : (int)(it - v.begin()); +} + +double quantile(std::vector v, double q) { + if (v.empty()) return 0.0; + std::sort(v.begin(), v.end()); + const double pos = q * (double)(v.size() - 1); + const size_t lo = (size_t)std::floor(pos), hi = (size_t)std::ceil(pos); + return v[lo] + (v[hi] - v[lo]) * (pos - (double)lo); +} + +} // namespace + +EvalReport evaluate(const std::vector& cases, const std::vector& classes) { + EvalReport r; + r.classes = classes; + const size_t k = classes.size(); + r.support.assign(k, 0); + r.recall.assign(k, 0.0); + r.precision.assign(k, 0.0); + r.confusion.assign(k, std::vector(k, 0)); + std::vector ms; + double bin_conf[10] = {0}, bin_acc[10] = {0}; + int bin_n[10] = {0}; + for (const EvalCase& c : cases) { + const int t = index_of(classes, c.label); + if (t < 0) throw std::runtime_error("eval: label \"" + c.label + "\" is not one of the table's classes"); + const int g = index_of(classes, c.got); + r.n++; + r.support[t]++; + if (c.gated) r.gated++; + if (g >= 0) r.confusion[t][g]++; + const bool ok = g == t; + if (ok) r.correct++; + r.mean_confidence += c.confidence; + float top = 0.0f, pt = 0.0f; + for (const auto& [cls, p] : c.probabilities) { + top = std::max(top, p); + if (cls == c.label) pt = p; + } + r.mean_top_p += top; + r.log_loss += -std::log(std::max(1e-9, (double)pt)); + const int b = std::min(9, (int)(top * 10.0f)); + bin_conf[b] += top; + bin_acc[b] += ok ? 1.0 : 0.0; + bin_n[b]++; + ms.push_back(c.ms); + } + if (r.n == 0) return r; + r.accuracy = (double)r.correct / r.n; + r.mean_confidence /= r.n; + r.mean_top_p /= r.n; + r.log_loss /= r.n; + for (int b = 0; b < 10; ++b) + if (bin_n[b]) r.ece += std::fabs(bin_acc[b] / bin_n[b] - bin_conf[b] / bin_n[b]) * bin_n[b] / r.n; + for (size_t c = 0; c < k; ++c) { + int chosen = 0; + for (size_t t = 0; t < k; ++t) chosen += r.confusion[t][c]; + r.recall[c] = r.support[c] ? (double)r.confusion[c][c] / r.support[c] : 0.0; + r.precision[c] = chosen ? (double)r.confusion[c][c] / chosen : 0.0; + } + r.ms_p50 = quantile(ms, 0.5); + r.ms_p90 = quantile(ms, 0.9); + return r; +} + +std::string report_json(const EvalReport& r) { + nlohmann::ordered_json j; + j["n"] = r.n; + j["correct"] = r.correct; + j["accuracy"] = r.accuracy; + j["gated"] = r.gated; + nlohmann::ordered_json per = nlohmann::ordered_json::object(); + for (size_t c = 0; c < r.classes.size(); ++c) + per[r.classes[c]] = {{"support", r.support[c]}, {"recall", r.recall[c]}, {"precision", r.precision[c]}}; + j["classes"] = per; + j["confusion"] = {{"rows_true_cols_chosen", r.classes}, {"matrix", r.confusion}}; + j["mean_confidence"] = r.mean_confidence; + j["mean_top_p"] = r.mean_top_p; + j["ece"] = r.ece; + j["log_loss"] = r.log_loss; + j["decision_ms_p50"] = r.ms_p50; + j["decision_ms_p90"] = r.ms_p90; + return j.dump(); +} + +std::string report_text(const EvalReport& r) { + std::string out; + char buf[256]; + std::snprintf(buf, sizeof buf, "accuracy %d/%d = %.3f (%d by the size gate)\n", r.correct, r.n, r.accuracy, r.gated); + out += buf; + for (size_t c = 0; c < r.classes.size(); ++c) { + std::snprintf(buf, sizeof buf, " %-16s support %4d recall %.3f precision %.3f\n", r.classes[c].c_str(), + r.support[c], r.recall[c], r.precision[c]); + out += buf; + } + out += "confusion (rows true, columns chosen):\n"; + for (size_t t = 0; t < r.classes.size(); ++t) { + std::snprintf(buf, sizeof buf, " %-16s", r.classes[t].c_str()); + out += buf; + for (const int v : r.confusion[t]) { + std::snprintf(buf, sizeof buf, " %5d", v); + out += buf; + } + out += "\n"; + } + std::snprintf(buf, sizeof buf, + "confidence mean %.3f, top p mean %.3f, ECE %.3f, log loss %.3f, decision p50 %.0f ms, p90 %.0f ms\n", + r.mean_confidence, r.mean_top_p, r.ece, r.log_loss, r.ms_p50, r.ms_p90); + out += buf; + return out; +} + +} // namespace onebit::laya diff --git a/laya/eval.h b/laya/eval.h new file mode 100644 index 000000000..07ade9876 --- /dev/null +++ b/laya/eval.h @@ -0,0 +1,57 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +// Routing accuracy of a model table over labelled states (`1bit route --laya-models FILE --eval CASES`): +// accuracy, per-class recall and precision, the confusion matrix, Laya's confidence, calibration (ECE over +// the top probability) and log loss. Pure arithmetic over the decisions, so it is tested without a checkpoint. +#pragma once + +#include +#include +#include + +namespace onebit::laya { + +struct EvalCase { + std::string label; // the true class + std::string got; // the class the table's question chose + float confidence = 0.0f; // serve's confidence, 1 - H(p)/log k + std::vector> probabilities; // class -> p + double ms = 0.0; // the decision's time + bool gated = false; // decided by the size gate, not by Laya +}; + +struct EvalReport { + std::vector classes; + int n = 0, correct = 0, gated = 0; + double accuracy = 0.0; + std::vector support; // per true class + std::vector recall, precision; // per class + std::vector> confusion; // [true][chosen] + double mean_confidence = 0.0; // serve's confidence + double mean_top_p = 0.0; + double ece = 0.0; // 10 equal-width bins over the top probability + double log_loss = 0.0; // mean -log p(true class), p floored at 1e-9 + double ms_p50 = 0.0, ms_p90 = 0.0; +}; + +// classes: the table's classes in its order. A case whose label is not a class is refused (throws). +EvalReport evaluate(const std::vector& cases, const std::vector& classes); + +std::string report_json(const EvalReport& r); +std::string report_text(const EvalReport& r); + +} // namespace onebit::laya diff --git a/laya/scorer.cpp b/laya/scorer.cpp index d2f3a8d9c..91a6a253e 100644 --- a/laya/scorer.cpp +++ b/laya/scorer.cpp @@ -560,6 +560,17 @@ bool Scorer::score(const std::string& state, const std::vector& questi layernorm(h.data(), final_norm_.data(), nullptr, N * L, D, tmp.data()); h.swap(tmp); } + if (raw && raw->keep_encoder) { + raw->d = D; + raw->encoder.assign(N, {}); + raw->markers.assign(N, {}); + raw->qtype.assign(qtype.begin(), qtype.end()); + for (int b = 0; b < N; ++b) { + const float* row = h.data() + (size_t)b * L * D; + raw->encoder[b].assign(row, row + seqs[b].size() * D); + raw->markers[b] = markers[b]; + } + } // ---- decision head ---- for (int b = 0; b < N; ++b) { diff --git a/laya/scorer.h b/laya/scorer.h index d64753646..e786255f6 100644 --- a/laya/scorer.h +++ b/laya/scorer.h @@ -61,6 +61,14 @@ struct Answer { struct RawOutput { std::vector> logits; // per question, per marker (masked = -1e4) std::vector> act_logits; // per question, 2 entries + // Set keep_encoder to also get the decision head's input: per question, the encoder output after the + // final norm (its own length x d floats, no padding), the option marker positions and the question type. + // `1bit route --dump-encoder` writes these for a head-only fine-tune. + bool keep_encoder = false; + int d = 0; + std::vector> encoder; + std::vector> markers; + std::vector qtype; }; class Scorer { diff --git a/tests/laya_eval_e2e.sh b/tests/laya_eval_e2e.sh new file mode 100755 index 000000000..0a7e732a7 --- /dev/null +++ b/tests/laya_eval_e2e.sh @@ -0,0 +1,95 @@ +#!/usr/bin/env bash +# Copyright 2026 bong-water-water-bong +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +# laya_eval_e2e.sh: `1bit route --laya-models FILE --eval` and `--dump-encoder` on the real scorer +# (docs/laya-models.md, "Measuring and fine-tuning the router"). Four labelled states, one of them past the +# table's size gate. Checks: the JSON report's counts, the per-case file, the text report; the dump's +# files agree with each other (rows x d x 4 bytes, one line per sequence, the gated case skipped, labels +# moved with --rotate); refusals by name. +set -uo pipefail + +bin=${1:?usage: laya_eval_e2e.sh path/to/1bit path/to/laya-model-dir} +laya_model=${2:?usage: laya_eval_e2e.sh path/to/1bit path/to/laya-model-dir} +scratch=$(mktemp -d) +trap 'rm -rf "$scratch"' EXIT +fail=0 +check() { if eval "$2"; then echo "PASS $1"; else echo "FAIL $1"; fail=1; fi; } + +cat > "$scratch/table.json" <<'JSON' +{"question": {"instructions": "What does the game agent need to answer this step?", + "classes": {"reflex": "one quick action from what is in view right now", + "plan": "planning: a crafting tree, a build plan, choosing the next goals"}}, + "default": "plan", "size_gate": {"chars": 300, "class": "plan"}, + "classes": {"reflex": {"model": "small.gguf", "device": "npu"}, "plan": {"model": "big.gguf", "device": "hrx"}}} +JSON +long=$(printf 'Plan the whole base layout. %.0s' $(seq 1 20)) +cat > "$scratch/cases.jsonl" < "$scratch/eval.json" 2> "$scratch/eval.err" +check "eval --json exits 0" "[ $? -eq 0 ]" +check "eval report counts" "python3 -c \" +import json; r = json.load(open('$scratch/eval.json')) +assert r['n'] == 4 and r['gated'] == 1, r +assert sum(map(sum, r['confusion']['matrix'])) == 4 +assert 0 <= r['accuracy'] <= 1 and r['correct'] == round(r['accuracy'] * 4) +assert set(r['classes']) == {'reflex', 'plan'} and r['classes']['reflex']['support'] == 2 +for k in ('ece', 'log_loss', 'mean_confidence', 'decision_ms_p50', 'decision_ms_p90'): assert k in r +\"" +check "per-case file: one line per labelled case, p sums to 1" "python3 -c \" +import json; L = [json.loads(x) for x in open('$scratch/per.jsonl')] +assert [c['src'] for c in L] == ['a', 'b', 'c', 'd'] +assert all(abs(sum(c['probabilities'].values()) - 1) < 1e-4 for c in L) +assert L[3]['gated'] and L[3]['chosen'] == 'plan' and not L[0]['gated'] +\"" +"${R[@]}" --eval "$scratch/cases.jsonl" > "$scratch/eval.txt" 2>&1 +check "text report" "grep -q '^accuracy [0-9]/4' '$scratch/eval.txt' && grep -q 'confusion' '$scratch/eval.txt'" + +"${R[@]}" --dump-encoder "$scratch/cases.jsonl" --out "$scratch/dump" --rotate > "$scratch/dump.out" 2>&1 +check "dump exits 0" "[ $? -eq 0 ]" +check "dump files agree" "python3 -c \" +import json, os; d = '$scratch/dump'; info = json.load(open(d + '/dump.json')) +C = [json.loads(x) for x in open(d + '/cases.jsonl')] +assert info['skipped_size_gated'] == 1 and info['cases'] == len(C) == 8, (info, len(C)) +assert os.path.getsize(d + '/h.f32') == info['rows'] * info['d'] * 4 +assert sum(c['len'] for c in C) == info['rows'] and C[1]['offset'] == C[0]['len'] +assert all(len(c['markers']) == 2 and max(c['markers']) < c['len'] for c in C) +a0, a1 = C[0], C[1] +assert a0['src'] == a1['src'] == 'a' and a0['options'] == ['reflex', 'plan'] and a1['options'] == ['plan', 'reflex'] +assert a0['label'] == 0 and a1['label'] == 1 +assert [c['label'] for c in C if c['src'] == '$scratch/cases.jsonl:5'] == [-1, -1] +\"" + +"${R[@]}" --eval "$scratch/missing.jsonl" > /dev/null 2> "$scratch/e1" +check "refuses a missing case file" "grep -q 'cannot read' '$scratch/e1'" +echo '{"state": "x", "label": "chat"}' > "$scratch/bad.jsonl" +"${R[@]}" --eval "$scratch/bad.jsonl" > /dev/null 2> "$scratch/e2" +check "refuses a label that is not a class" "grep -q 'not one of the table' '$scratch/e2'" +"${R[@]}" --eval "$scratch/cases.jsonl" --dump-encoder "$scratch/cases.jsonl" > /dev/null 2> "$scratch/e3" +check "refuses --eval with --dump-encoder" "grep -q 'exactly one of' '$scratch/e3'" +"${R[@]}" --dump-encoder "$scratch/cases.jsonl" > /dev/null 2> "$scratch/e4" +check "refuses a dump without --out" "grep -q -- '--out DIR' '$scratch/e4'" +"$bin" route --eval "$scratch/cases.jsonl" > /dev/null 2> "$scratch/e5" +check "refuses --eval without --laya-models" "grep -q 'need --laya-models' '$scratch/e5'" + +[ $fail = 0 ] && echo PASS || echo FAIL +exit $fail diff --git a/tests/laya_eval_test.cpp b/tests/laya_eval_test.cpp new file mode 100644 index 000000000..790829591 --- /dev/null +++ b/tests/laya_eval_test.cpp @@ -0,0 +1,83 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + + +// laya/eval.cpp, the arithmetic of `1bit route --laya-models FILE --eval`: hand-computed cases for +// accuracy, recall, precision, the confusion matrix, ECE, log loss, the size-gate count and the refusal. +#include "eval.h" + +#include +#include +#include + +#include + +namespace { +int failures = 0; +void check(const std::string& what, bool ok) { + std::printf("%s %s\n", ok ? "PASS" : "FAIL", what.c_str()); + if (!ok) failures++; +} +bool near(double a, double b) { return std::fabs(a - b) < 1e-6; } + +onebit::laya::EvalCase c(const std::string& label, const std::string& got, float p_reflex, double ms, bool gated = false) { + onebit::laya::EvalCase e; + e.label = label; + e.got = got; + e.probabilities = {{"reflex", p_reflex}, {"plan", 1.0f - p_reflex}}; + e.confidence = 0.5f; + e.ms = ms; + e.gated = gated; + return e; +} +} // namespace + +int main() { + using namespace onebit::laya; + // 4 reflex (3 right), 2 plan (1 right, one by the size gate) + const std::vector cases = { + c("reflex", "reflex", 0.9f, 10), c("reflex", "reflex", 0.8f, 20), c("reflex", "reflex", 0.6f, 30), + c("reflex", "plan", 0.3f, 40), c("plan", "plan", 0.0f, 0, true), c("plan", "reflex", 0.55f, 50), + }; + const EvalReport r = evaluate(cases, {"reflex", "plan"}); + check("n and correct", r.n == 6 && r.correct == 4); + check("accuracy 4/6", near(r.accuracy, 4.0 / 6)); + check("gated counted", r.gated == 1); + check("support", r.support[0] == 4 && r.support[1] == 2); + check("confusion", r.confusion[0][0] == 3 && r.confusion[0][1] == 1 && r.confusion[1][0] == 1 && r.confusion[1][1] == 1); + check("recall", near(r.recall[0], 0.75) && near(r.recall[1], 0.5)); + check("precision", near(r.precision[0], 0.75) && near(r.precision[1], 0.5)); + // -log p(true): 0.9, 0.8, 0.6, 0.3 (reflex), 1.0, 0.45 (plan) + const double ll = -(std::log(0.9) + std::log(0.8) + std::log(0.6) + std::log(0.3) + std::log(1.0) + std::log(0.45)) / 6; + check("log loss", std::fabs(r.log_loss - ll) < 1e-5); + // top p: .9 .8 .6 .7 1.0 .55; bins 9:{.9 ok} 8:{.8 ok} 6:{.6 ok} 7:{.7 wrong} 10->9:{1.0 ok} 5:{.55 wrong} + // bin 9: conf (.9+1)/2=.95, acc 1 -> .05*2/6; bin 8: .2*1/6; bin 6: .4/6; bin 7: .7/6; bin 5: .55/6 + const double ece = (0.05 * 2 + 0.2 + 0.4 + 0.7 + 0.55) / 6; + check("ECE", std::fabs(r.ece - ece) < 1e-5); + check("decision ms p50 / p90", near(r.ms_p50, 25.0) && near(r.ms_p90, 45.0)); + const auto j = nlohmann::json::parse(report_json(r)); + check("json carries the per-class numbers", j["classes"]["plan"]["recall"] == 0.5 && j["n"] == 6); + check("text report names the accuracy", report_text(r).find("accuracy 4/6") != std::string::npos); + bool threw = false; + try { + evaluate({c("chat", "reflex", 0.5f, 1)}, {"reflex", "plan"}); + } catch (const std::exception&) { + threw = true; + } + check("refuses a label that is not a class", threw); + check("empty set is all zeros", evaluate({}, {"reflex", "plan"}).n == 0); + std::printf("%s\n", failures ? "FAIL" : "PASS"); + return failures ? 1 : 0; +}