Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -431,6 +431,13 @@ add_test(NAME serve_watchdog COMMAND serve_watchdog_test)
# engine#1781: the comparator behind NPU_BATCH_CHECK -- pure arithmetic over two float arrays, so it runs with no model,
# no device and no runtime. It is a header, so the test includes it directly and needs no link libraries.
add_executable(npu_sampling_test tests/npu_sampling_test.cpp)
# The shape rule for a 1.8 external-data initializer (npu/pack_initializer_shape.h, engine#1503's graph-initializer
# reader). Those packs zero every `dims`, so a byte length has to become a shape -- and a shape inferred wrongly imports
# cleanly and serves garbage, so the refusals are the arms worth pinning. Header-only, no parser, no pack, no device.
add_executable(pack_initializer_shape_test tests/pack_initializer_shape_test.cpp)
target_include_directories(pack_initializer_shape_test PRIVATE npu)
add_test(NAME pack_initializer_shape COMMAND pack_initializer_shape_test)
add_dependencies(onebit pack_initializer_shape_test)
target_include_directories(npu_sampling_test PRIVATE app)
target_link_libraries(npu_sampling_test PRIVATE nlohmann_json::nlohmann_json)
# The NPU route's bounded load retry (app/npu_load_retry.h, engine #1010, docs/npu.md "When a load follows an evict"):
Expand Down
107 changes: 107 additions & 0 deletions npu/pack_initializer_shape.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,107 @@
// Copyright 2026 bong-water-water-bong
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// pack_initializer_shape.h -- the SHAPE rule for a 1.8-generation initializer read out of external data
// (engine#1503, the graph-initializer reader). Header-only and dependency-free, so the rule is testable with no
// ONNX parser, no pack and no device -- which is the point: the resolution path must not be written until this rule can
// be shown to refuse the cases it cannot decide.
//
// Why the rule is the hard part and not the reading: those packs zero every `dims` field on the initializers, so all
// that survives is `external_data { offset, length }` against the sibling `.data` file -- a LENGTH, not a shape. A shape
// guessed wrongly imports cleanly and serves garbage, which is the failure mode this thread has recorded three times
// (#1874's unsigned zero points, #1888's wrong quantisation rule, #1993's untied head). So the rule refuses whenever the
// length is consistent with more than one shape, and names what it saw.
#pragma once

#include <cstdint>
#include <string>
#include <vector>

namespace onebit::npu {

// A resolved initializer shape. ndim is 1 for the norms and biases, 2 for a weight; d1 is unused at ndim == 1.
struct InitShape {
bool ok = false;
int ndim = 0;
int64_t d0 = 0;
int64_t d1 = 0;
std::string why; // set on refusal: what was measured, in numbers
};

// Names that are 1-D by convention in this genre. The refusal this reader fixes names exactly these two:
// `model.layers.N.input_layernorm.weight` and `model.layers.N.post_attention_layernorm.weight`.
inline bool is_1d_by_name(const std::string& name) {
if (name.size() >= 5 && name.compare(name.size() - 5, 5, ".bias") == 0) return true;
return name.find("norm") != std::string::npos || name.find("Norm") != std::string::npos;
}

// Resolve one initializer's shape from its byte length and the model's known sizes (hidden, intermediate, vocab, ...).
//
// * a length that is not a whole number of elements is refused (the file and the element type disagree);
// * a 1-D name (norm/bias) takes [elements] ONLY when that length is one of the known sizes -- a norm is exactly one
// of them, and a length that matches none is not a norm;
// * a weight takes [k, elements/k] ONLY when exactly one known size divides it. Zero divisors or two are both a
// refusal: two would mean two plausible weight shapes, which is the guess this rule exists to avoid.
inline InitShape resolve_init_shape(const std::string& name, int64_t length, int elem_bytes,
const std::vector<int64_t>& known) {
InitShape s;
if (length <= 0 || elem_bytes <= 0) {
s.why = "a non-positive length (" + std::to_string(length) + " B) or element size (" +
std::to_string(elem_bytes) + " B)";
return s;
}
if (length % elem_bytes != 0) {
s.why = std::to_string(length) + " B is not a whole number of " + std::to_string(elem_bytes) + "-byte elements";
return s;
}
const int64_t elems = length / elem_bytes;
if (is_1d_by_name(name)) {
for (const int64_t k : known)
if (k > 0 && k == elems) {
s.ok = true;
s.ndim = 1;
s.d0 = elems;
return s;
}
s.why = "a norm/bias of " + std::to_string(elems) + " elements matches none of the model's declared sizes";
return s;
}
int64_t found = 0;
int ndiv = 0;
for (const int64_t k : known)
if (k > 0 && elems % k == 0) {
++ndiv;
found = k;
}
if (ndiv == 0) {
s.why = "a weight of " + std::to_string(elems) + " elements has no declared dimension that divides it";
return s;
}
if (ndiv > 1) {
std::string all;
for (const int64_t k : known)
if (k > 0 && elems % k == 0) all += (all.empty() ? "" : ", ") + std::to_string(k);
s.why = "a weight of " + std::to_string(elems) + " elements is consistent with several declared dimensions (" +
all + "), so the shape cannot be decided from the length";
return s;
}
s.ok = true;
s.ndim = 2;
s.d0 = found;
s.d1 = elems / found;
return s;
}

} // namespace onebit::npu
90 changes: 90 additions & 0 deletions tests/pack_initializer_shape_test.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
// Copyright 2026 bong-water-water-bong
// SPDX-License-Identifier: Apache-2.0
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.
//
// pack_initializer_shape_test -- the shape rule for a 1.8 external-data initializer (engine#1503). No ONNX parser, no
// pack, no device: the rule is a function of (name, byte length, element size, the model's declared sizes), so every arm
// is a hand-computed value. The refusals matter more than the accepts -- a shape inferred wrongly imports cleanly and
// serves garbage, so the cases that must NOT resolve are the ones worth pinning.
#include "pack_initializer_shape.h"

#include <cstdio>
#include <vector>

using onebit::npu::InitShape;
using onebit::npu::resolve_init_shape;

static int failures = 0;
static void check(bool ok, const char* what) {
std::printf("%s %s\n", ok ? "ok " : "FAIL", what);
if (!ok) ++failures;
}

int main() {
// The measured pack: Llama-3.2-1B 1.8 hybrid, hidden 2048, IM 8192, vocab 128256, bf16 elements.
const std::vector<int64_t> known{2048, 8192, 128256};
const int bf16 = 2;

// (i) the refusal this reader exists to fix: a layernorm weight is 2048 bf16 = 4096 B, one of the known sizes.
{
const InitShape s = resolve_init_shape("model.layers.0.input_layernorm.weight", 2048 * bf16, bf16, known);
check(s.ok && s.ndim == 1 && s.d0 == 2048, "(i) a layernorm weight resolves to [2048]");
}
// (ii) a 2-D weight whose dimension is unambiguous: 2048 x 3 elements -- only 2048 divides it among the known
// sizes, so the shape is decided. (2048 x 8192 would NOT be unambiguous: both are declared sizes, which is arm
// (v) below, and my first version of this arm asserted the opposite and was wrong.)
{
const InitShape s = resolve_init_shape("model.layers.0.mlp.down_proj.weight", 2048 * 3LL * bf16, bf16, known);
check(s.ok && s.ndim == 2 && s.d0 == 2048 && s.d1 == 3, "(ii) a weight only one declared size divides resolves");
}
// (iii) a length that is not a whole number of elements is refused, and says so with both numbers.
{
const InitShape s = resolve_init_shape("model.layers.0.mlp.down_proj.weight", 4097, bf16, known);
check(!s.ok && s.why.find("4097") != std::string::npos && s.why.find("2-byte") != std::string::npos,
"(iii) an odd byte length is refused, naming the length and the element size");
}
// (iv) a norm whose length matches none of the declared sizes is refused: it is not a norm of this model.
{
const InitShape s = resolve_init_shape("model.layers.0.input_layernorm.weight", 3000 * bf16, bf16, known);
check(!s.ok && s.why.find("matches none") != std::string::npos, "(iv) a norm of an undeclared size is refused");
}
// (v) the case the rule exists for: a weight consistent with TWO declared dimensions must refuse, not pick one.
// 2048*8192 elements is divisible by both 2048 and 8192, so a weight of that length is ambiguous.
{
const InitShape s = resolve_init_shape("model.layers.0.some.weight", 2048 * 8192LL * bf16, bf16, known);
check(!s.ok && s.why.find("several declared dimensions") != std::string::npos,
"(v) a weight consistent with two dimensions is REFUSED as ambiguous, naming them");
}
// (vi) a weight with no dividing declared dimension is refused.
{
const InitShape s = resolve_init_shape("model.layers.0.some.weight", 1000 * bf16, bf16, known);
check(!s.ok && s.why.find("no declared dimension") != std::string::npos,
"(vi) a weight with no dividing dimension is refused");
}
// (vii) a bias is 1-D by name and takes a known size; a bias of an undeclared size is refused like a norm.
{
const InitShape a = resolve_init_shape("model.layers.0.attn.q_proj.bias", 2048 * bf16, bf16, known);
const InitShape b = resolve_init_shape("model.layers.0.attn.q_proj.bias", 777 * bf16, bf16, known);
check(a.ok && a.ndim == 1 && a.d0 == 2048 && !b.ok, "(vii) a bias is 1-D like a norm, and refuses an undeclared size");
}
// (viii) a non-positive length or element size is refused rather than divided by zero.
{
const InitShape a = resolve_init_shape("x.norm.weight", 0, bf16, known);
const InitShape b = resolve_init_shape("x.norm.weight", 4096, 0, known);
check(!a.ok && !b.ok, "(viii) a zero length or zero element size is refused, not divided by");
}

if (failures == 0) std::printf("pack initializer shape: 8 arms pass\n");
return failures ? 1 : 0;
}
Loading