Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
42 changes: 28 additions & 14 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -279,28 +279,46 @@ add_test(NAME serve_context_limit_classify COMMAND bash ${CMAKE_SOURCE_DIR}/test
# creates. #1583 added one name for a target defined inside if(ONEBIT_NPU), which CI does not configure -- and ninja
# rejects the whole command line before building anything, so `onebit` stopped being compiled for every PR. Reads the
# two files, no build and no submodules; --self-test drives it.
add_test(NAME ci_targets COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/ci/ci_targets_check.py --check --root ${CMAKE_SOURCE_DIR})
add_test(NAME ci_targets_self COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/ci/ci_targets_check.py --self-test)
# engine#1864: `find_package(Python3)` used to be at :761, long after the first `${Python3_EXECUTABLE}` users above, so those
# registrations carried an EMPTY interpreter and ctest executed each script directly -- which passes only while the file keeps
# its exec bit (a mode-only rewrite of tools/ci/hrx_build_scope.py turned that row into BAD_COMMAND). Find the interpreter
# first and register every Python test through one place: it runs the script THROUGH the interpreter, and on a machine
# without one the test is dropped with a status line instead of a .py being handed to the kernel as a program.
find_package(Python3 COMPONENTS Interpreter)
function(add_python_test name)
if(Python3_Interpreter_FOUND)
add_test(NAME ${name} COMMAND ${Python3_EXECUTABLE} ${ARGN})
else()
message(STATUS "ctest: not registering ${name} (no Python3 interpreter)")
endif()
endfunction()
# engine#1864: the ordering rule itself, checked on the file (no configure, no build): every ${Python3_EXECUTABLE} use has
# to follow the find_package above. The nine that did not ran through their exec bit, and a mode-only rewrite of one
# script turned the row into BAD_COMMAND.
add_python_test(python_test_order_check ${CMAKE_SOURCE_DIR}/tests/python_test_order_check.py --check --root ${CMAKE_SOURCE_DIR})
add_python_test(python_test_order_check_self ${CMAKE_SOURCE_DIR}/tests/python_test_order_check.py --self-test)
add_python_test(ci_targets ${CMAKE_SOURCE_DIR}/tools/ci/ci_targets_check.py --check --root ${CMAKE_SOURCE_DIR})
add_python_test(ci_targets_self ${CMAKE_SOURCE_DIR}/tools/ci/ci_targets_check.py --self-test)
# engine#1820: the other half of ci_targets_check -- a standalone shell test that nothing references cannot fail.
# `need` is every tests/*.sh, `have` is every file that mentions its basename somewhere that can run it, and
# tools/ci/unreferenced_tests.txt carries the deliberate exclusions WITH their reasons, so the list can shrink.
add_test(NAME tests_referenced COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/ci/tests_referenced_check.py --check --root ${CMAKE_SOURCE_DIR})
add_test(NAME tests_referenced_self COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/ci/tests_referenced_check.py --self-test)
add_python_test(tests_referenced ${CMAKE_SOURCE_DIR}/tools/ci/tests_referenced_check.py --check --root ${CMAKE_SOURCE_DIR})
add_python_test(tests_referenced_self ${CMAKE_SOURCE_DIR}/tools/ci/tests_referenced_check.py --self-test)
# engine#1777: the release row's own arm, which merged unregistered -- this sweep is what found it (engine#1820).
add_test(NAME release_check_hrx_init_reason COMMAND bash ${CMAKE_SOURCE_DIR}/tests/release_check_hrx_init_reason_test.sh ${CMAKE_SOURCE_DIR}/os/mini/release-check.sh)
# engine#1615: every process holding the GPU/NPU device must belong to a live gpu-hold/npu-hold ticket. Three
# orphans of that shape landed in two days (51 GiB of GTT for 14 h; an NPU server 70 min past a 6-min ticket;
# then two lanes each starting an HRX server outside the token within one half hour). The ancestry test is the
# check; the two permanent services are exempt by name. --self-test drives the decision, no device needed.
add_test(NAME device_holders COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/device_holders.py --self-test)
add_python_test(device_holders ${CMAKE_SOURCE_DIR}/tools/device_holders.py --self-test)
# engine #1668: the bench preflight must pin the load mode, because `-lm none` is +17.5 % prefill at -1.6 % decode for +604 MiB of
# GTT. The script existed with a test that nothing ran; a test nobody runs is a comment, so the loader/flag assertions are registered
# here (the test fakes the box, so it needs no device and no ticket).
add_test(NAME bench_preflight COMMAND bash ${CMAKE_SOURCE_DIR}/tools/tests/test-bench-preflight.sh)
# engine#1702: builds a per-layer MoE expert placement table from a moe_trace, and refuses a prefill-only trace because
# placement is a decode lever (5.4-6.9 % on batched prefill against dedup's 90-94 %). --self-test drives it on synthetic
# traces with a known answer; no model, no device.
add_test(NAME moe_placement COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/moe_placement.py --self-test)
add_python_test(moe_placement ${CMAKE_SOURCE_DIR}/tools/moe_placement.py --self-test)
# engine #1279: the version a package is built with (scripts/pack-version.sh) -- a release tag as
# it stands, a candidate mapped into the ISO week of its own date with a suffix that sorts below that
# release, and the ordering a candidate needs: previous release < candidate < its release. The
Expand Down Expand Up @@ -701,15 +719,12 @@ add_test(NAME route_log COMMAND route_log_test)
# case is an artefact someone would actually push, and the conservative direction -- "full" -- is the
# one every uncertain case must land on, because the build being skipped is the one that catches a
# broken HRX pin. Stdlib only: no git needed for the decision itself.
add_test(NAME hrx_build_scope
COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/ci/hrx_build_scope.py --self-test)
add_test(NAME copyright_check_untracked
COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tests/copyright_check_untracked_test.py)
add_python_test(hrx_build_scope ${CMAKE_SOURCE_DIR}/tools/ci/hrx_build_scope.py --self-test)
add_python_test(copyright_check_untracked ${CMAKE_SOURCE_DIR}/tests/copyright_check_untracked_test.py)
# engine#1517: the prefill flash-attention ratio is measured in adjacent pairs and only quoted when the pairs agree, so a
# loaded box yields VOID instead of a number. The self-test is the estimator's own gate and needs no GPU: it accepts
# agreement, refuses disagreement, and proves a dropped run cannot shift an arm.
add_test(NAME fa_ratio_estimator
COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/fa_ratio.py --self-test)
add_python_test(fa_ratio_estimator ${CMAKE_SOURCE_DIR}/tools/fa_ratio.py --self-test)
add_dependencies(onebit route_log_test)
# CI builds `--target onebit` plus a fixed list in .github/workflows/ci.yml; building this check with onebit
# keeps the laya_models ctest from reporting "Not Run" without a workflow change.
Expand Down Expand Up @@ -757,8 +772,7 @@ if(ONEBIT_LAYA_ROUTE_MODEL)
endif()

# The registry against the reviewed not-an-alias list (docs/arch-gaps.md) and the committed pins:
# no checkouts or network.
find_package(Python3 COMPONENTS Interpreter)
# no checkouts or network. (The interpreter is found once, at :287: engine#1864.)
if(Python3_Interpreter_FOUND)
add_test(NAME registry_gaps COMMAND ${Python3_EXECUTABLE} ${CMAKE_SOURCE_DIR}/tools/registry_build.py --check-gaps)
# pin the guard's firing behavior: the committed registry always passes --check-gaps, so
Expand Down
58 changes: 58 additions & 0 deletions app/forward_serve.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -795,6 +795,64 @@ void handle_systemone(ForwardEngine& e, const httplib::Request& req, httplib::Re

} // namespace

// engine#1862: the sealed-route one-shot id run. `npu-run` was written for the per-role lane kernel set
// (layer_ctx{1,2,17}.elf + lmhead.elf), which no pack on disk ships any more: every pack since the sealed route carries
// SEALED-*.elf / full_i8_* driven by this forward. So an id run on a current pack goes through the SAME forward, and
// the pass it makes is the one the k-row identity check needs -- one launch, M rows, per-row logits.
int run_forward_ids(const std::string& model_dir, const std::string& kernels, const std::vector<int>& ids,
int max_tokens, float penalty, bool host_gemm) {
if (ids.empty()) throw std::runtime_error("--ids: no token ids");
const std::string model = model_dir + "/model.q4nx";
if (!std::filesystem::exists(model)) throw std::runtime_error(model + " does not exist");
// Same device discipline as forward-serve: host-GEMM runs take no lock (engine#1353 -- a host-GEMM run that held it
// for 4 h 52 m stopped two lanes' device work), a run that opens the device takes the exclusive one.
int lock_fd = -1;
if (npu_lock_taken(!host_gemm, std::getenv("NPU_NO_DEVICE_LOCK") != nullptr)) lock_fd = take_npu_device_lock(1);
std::unique_ptr<xrt::device> dev;
if (!host_gemm) dev = std::make_unique<xrt::device>(0);
Q4nxNpuForward fwd;
const bool ok = dev ? fwd.init(*dev, model.c_str(), kernels) : fwd.init_host(model.c_str(), kernels);
if (!ok) throw std::runtime_error("forward init failed: " + std::string(fwd.last_error()));
const int width = fwd.max_batch_rows();
if (int(ids.size()) > width)
std::fprintf(stderr, "1bit npu-run: %zu ids, so this runs in passes of %d rows (the kernel's width)\n", ids.size(),
width);
int done = 0;
while (done < int(ids.size())) {
const int m = std::min(width, int(ids.size()) - done);
std::vector<int> chunk(ids.begin() + done, ids.begin() + done + m);
std::vector<float> all, one;
// begin_verify() is what makes the next pass norm EVERY row and write M x NV logits (the speculative verifier's
// call); plain step_batch() returns only the last row, which cannot answer a per-row question.
fwd.begin_verify(m, done);
if (!fwd.step_batch_all(chunk, done, all))
throw std::runtime_error("forward step failed: " + std::string(fwd.last_error()));
// NV from the block itself (the vocab width is whatever the head returns) rather than from a config accessor:
// the size has to divide by the row count either way, and a block that does not is the thing to report.
if (all.empty() || all.size() % size_t(m) != 0)
throw std::runtime_error("forward returned a logits block of the wrong size (" + std::to_string(all.size()) +
" floats for " + std::to_string(m) + " rows)");
const int nv = int(all.size() / size_t(m));
for (int r = 0; r < m; ++r) {
const float* row = all.data() + size_t(r) * size_t(nv);
int best = 0;
for (int v = 1; v < nv; ++v)
if (row[v] > row[best]) best = v;
std::printf("%d\n", best); // token ids out, in row order: the id the model chooses at that row
}
done += m;
}
// -n only matters for a continuation, which this route does not do: say so rather than ignore the option.
if (max_tokens > 0)
std::fprintf(stderr,
"1bit npu-run: sealed route returns the model's choice at EVERY row of one pass; -n %d is not "
"used (a continuation is `1bit forward-serve`)\n",
max_tokens);
(void)penalty;
if (lock_fd >= 0) ::close(lock_fd);
return 0;
}

int run_forward_serve(int argc, char** argv) {
std::string model, kernels, host = "127.0.0.1", alias, dump_logits;
int port = 8000, streams = 1;
Expand Down
10 changes: 10 additions & 0 deletions app/forward_serve.h
Original file line number Diff line number Diff line change
Expand Up @@ -19,8 +19,18 @@
// OpenAI endpoints as `1bit unified`.
#pragma once

#include <string>
#include <vector>

namespace onebit {

int run_forward_serve(int argc, char** argv);

// engine#1862: `1bit npu-run --ids` on a pack whose kernel directory holds the SEALED set (SEALED-*/full_i8_*), which is
// every pack since that route shipped. Runs the ids through ONE multi-row pass of the same forward `forward-serve`
// drives -- the pass the k-row identity check needs -- and prints each row's argmax. `host_gemm` (NPU_INFER_HOST_GEMM)
// takes no device lock, exactly as forward-serve decides it. Returns the process exit code.
int run_forward_ids(const std::string& model_dir, const std::string& kernels, const std::vector<int>& ids,
int max_tokens, float penalty, bool host_gemm);

} // namespace onebit
20 changes: 13 additions & 7 deletions app/main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -139,7 +139,10 @@ int run_npu(int argc, char** argv) {
std::printf("usage: 1bit npu-run --model <dir> --kernels <dir> --ids \"<id> <id> ...\"\n"
" [-n 24] [--repetition-penalty 1.1] [--no-eos] [--dump-logits <dir>]\n"
" --model directory with model.q4nx and config.json\n"
" --kernels directory with layer_ctx{1,2,17}.elf, lmhead.elf and layer.pdi\n");
" --kernels the pack's kernel directory. Two generations are driven: the per-role lane set\n"
" (layer_ctx{1,2,17}.elf, lmhead.elf, layer.pdi) as a token loop, and the sealed set\n"
" (SEALED-*/full_i8_*) through the in-tree forward as ONE multi-row pass, printing each\n"
" row's choice (engine#1862; -n does not apply to the sealed route)\n");
return 0;
} else throw std::runtime_error("unknown option " + a);
}
Expand All @@ -162,12 +165,15 @@ int run_npu(int argc, char** argv) {
(name.rfind("SEALED-", 0) == 0 || name.rfind("full_i8_", 0) == 0))
++sealed;
}
if (sealed > 0)
throw std::runtime_error(
kernel_dir + " holds the sealed full-ELF kernel set (" + std::to_string(sealed) +
" ELF(s)), which npu-run does not drive: npu-run needs layer_ctx{1,2,17}.elf, lmhead.elf and "
"layer.pdi. The sealed route is driven by `1bit serve --device npu` from the pack's plan.json "
"(engine#1862)");
if (sealed > 0) {
// engine#1862: this is every pack on disk, so refuse-and-redirect was the wrong answer for the CLI that
// exists to hand N ids to ONE pass. The sealed set is driven by the same in-tree forward `forward-serve`
// uses; run the ids through it and print each row's choice.
std::fprintf(stderr, "1bit npu-run: sealed kernel set (%d ELF(s)), one multi-row pass through the "
"in-tree forward\n", sealed);
return onebit::run_forward_ids(model_dir, kernel_dir, prompt, opt.max_tokens,
opt.repetition_penalty, std::getenv("NPU_INFER_HOST_GEMM") != nullptr);
}
throw std::runtime_error("no kernel set in " + kernel_dir +
": neither layer_ctx1.elf (the per-role set npu-run drives) nor a SEALED-*/full_i8_* "
"ELF (the sealed set) is present");
Expand Down
Loading
Loading