Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
136 changes: 135 additions & 1 deletion npu/plan/op_registry.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -14,7 +14,9 @@
// limitations under the License.
#include "plan/op_registry.h"

#include <cmath>
#include <cstring>
#include <vector>

namespace onebit::npu::plan {

Expand Down Expand Up @@ -125,11 +127,143 @@ std::string op_coverage() {
std::string missing;
for (int i = 0; i < (int)OpId::count; i++) {
const OpId op = (OpId)i;
if (find_op(op)) continue;
// A reference of EITHER shape counts: `attn_step`'s operands are not a matrix pair, so it is
// registered in the call table (engine#799 rule (b) slice 2) and is no longer a gap.
if (find_op(op) || find_op_call(op)) continue;
if (!missing.empty()) missing += ",";
missing += op_name(op);
}
return missing;
}

// ── op-shaped calls: attn_step (engine#799 rule (b), slice 2) ───────────────────────────────────────
namespace {
// One position's attention: out[d] = sum_i softmax(q . k_i / sqrt(hd))[i] * v_i[d].
constexpr float kAttnQ[] = {1.0f, 0.0f};
constexpr float kAttnK[] = {1.0f, 0.0f, 0.0f, 1.0f}; // two cached positions: (1,0), (0,1)
constexpr float kAttnV[] = {1.0f, 2.0f, 3.0f, 4.0f}; // (1,2), (3,4)
// By hand: scores = (q.k0, q.k1)/sqrt(2) = (0.7071, 0); softmax = (0.66976, 0.33024);
// out = 0.66976*(1,2) + 0.33024*(3,4) = (1.66048, 2.66048).
constexpr float kAttnWantTwo[] = {1.6604769013466862f, 2.6604769013466862f};
// With one cached position the softmax is 1 by definition, so the output is v0 exactly -- a case the
// reference cannot pass by accident, because it pins that the normaliser is the sum and not a bare exp.
constexpr float kAttnWantOne[] = {1.0f, 2.0f};

const OpCallCase kAttnCases[] = {
{"two cached positions, hd 2",
"by hand: scores (1/sqrt2, 0) -> softmax (0.66976, 0.33024) -> 0.66976*(1,2)+0.33024*(3,4)",
{OpId::attn_step, kAttnQ, kAttnK, kAttnV, 2, 2, nullptr},
kAttnWantTwo, 2},
{"one cached position",
"softmax over a single score is 1, so the output is v0 (an implementation that skipped the normaliser "
"would pass the two-position case by luck but not this one)",
{OpId::attn_step, kAttnQ, kAttnK, kAttnV, 1, 2, nullptr},
kAttnWantOne, 2},
};
} // namespace

bool attn_step_ref(const OpCall& call) {
if (!call.q || !call.kcache || !call.vcache || !call.out || call.pos <= 0 || call.hd <= 0) return false;
const double scale = 1.0 / std::sqrt(double(call.hd));
std::vector<double> scores(static_cast<size_t>(call.pos));
double best = -1e300;
for (int i = 0; i < call.pos; ++i) {
double s = 0.0;
for (int d = 0; d < call.hd; ++d) s += double(call.q[d]) * double(call.kcache[i * call.hd + d]);
scores[size_t(i)] = s * scale;
if (scores[size_t(i)] > best) best = scores[size_t(i)];
}
double z = 0.0;
for (auto& s : scores) {
s = std::exp(s - best);
z += s;
}
for (int d = 0; d < call.hd; ++d) {
double acc = 0.0;
for (int i = 0; i < call.pos; ++i) acc += scores[size_t(i)] / z * double(call.vcache[i * call.hd + d]);
call.out[d] = float(acc);
}
return true;
}

namespace {
// ── gdn_step: the delta-rule core ──────────────────────────────────────────────────────────────────
namespace {
constexpr float kGdnK[] = {2.0f};
constexpr float kGdnV[] = {3.0f};
constexpr float kGdnQ[] = {1.0f};
constexpr float kGdnWantNoDecay[] = {3.0f}; // S = 1 + 2*1
constexpr float kGdnWantDecay[] = {4.5f}; // S = 0.5 + 2*2
constexpr float kGdnWantNoWrite[] = {0.5f}; // S = 0.5, the gate writes nothing
// Each case owns its state buffer (the reference advances it in place); the test resets it to the case's
// documented initial S = 1 before running, so the table stays re-runnable.
float kGdnS0[] = {1.0f}, kGdnS1[] = {1.0f}, kGdnS2[] = {1.0f};

const OpCallCase kGdnCases[] = {
{"dk = dv = 1, no decay, gate open",
"by hand: S 1, k 2, v 3, g 0, beta 1 -> err = 3 - 1*2 = 1, S = 1 + 2*1 = 3, out = S*q = 3",
{OpId::gdn_step, kGdnQ, nullptr, nullptr, 0, 0, nullptr, kGdnS0, kGdnK, kGdnV, 1, 1, 1.0f, 0.0f},
kGdnWantNoDecay, 1},
{"dk = dv = 1, half decay, gate open",
"by hand: S 1*exp(ln 0.5) = 0.5, err = 3 - 0.5*2 = 2, S = 0.5 + 2*2 = 4.5, out = 4.5",
{OpId::gdn_step, kGdnQ, nullptr, nullptr, 0, 0, nullptr, kGdnS1, kGdnK, kGdnV, 1, 1, 1.0f, -0.6931471805599453f},
kGdnWantDecay, 1},
{"dk = dv = 1, half decay, gate closed",
"by hand: S = 0.5 and beta = 0 writes nothing, so out = 0.5 -- pins that the gate multiplies the write "
"rather than being added to it",
{OpId::gdn_step, kGdnQ, nullptr, nullptr, 0, 0, nullptr, kGdnS2, kGdnK, kGdnV, 1, 1, 0.0f, -0.6931471805599453f},
kGdnWantNoWrite, 1},
};
} // namespace

bool gdn_delta_ref(const OpCall& call) {
if (!call.state || !call.k || !call.v || !call.q || !call.out || call.dk <= 0 || call.dv <= 0) return false;
const int dk = call.dk, dv = call.dv;
std::vector<double> S(static_cast<size_t>(dk) * dv), err(static_cast<size_t>(dv));
for (size_t i = 0; i < S.size(); ++i) S[i] = double(call.state[i]);
const double decay = std::exp(double(call.g));
for (auto& s : S) s *= decay; // 1. decay the state
for (int j = 0; j < dv; ++j) { // 2. err = v - S^T k (S is [dk][dv])
double acc = 0.0;
for (int i = 0; i < dk; ++i) acc += S[size_t(i) * dv + j] * double(call.k[i]);
err[size_t(j)] = double(call.v[j]) - acc;
}
for (int i = 0; i < dk; ++i) // 3. rank-1 write: S += beta * k (x) err
for (int j = 0; j < dv; ++j) S[size_t(i) * dv + j] += double(call.beta) * double(call.k[i]) * err[size_t(j)];
for (int j = 0; j < dv; ++j) { // 4. readout: out = S^T q
double acc = 0.0;
for (int i = 0; i < dk; ++i) acc += S[size_t(i) * dv + j] * double(call.q[i]);
call.out[j] = float(acc);
}
for (size_t i = 0; i < S.size(); ++i) call.state[i] = float(S[i]);
return true;
}

const OpCallEntry kCallEntries[] = {
{OpId::attn_step, &attn_step_ref, kAttnCases, sizeof(kAttnCases) / sizeof(kAttnCases[0]), nullptr},
{OpId::gdn_step, &gdn_delta_ref, kGdnCases, sizeof(kGdnCases) / sizeof(kGdnCases[0]),
"the delta-rule core (state update + readout); the layer's conv, q/k/v/a/b projections, per-head norms and "
"gates are gemv-shaped and covered by the first table, and their parameters are what load_block() makes "
"resident"},
};
} // namespace

const OpCallEntry* find_op_call(OpId op) {
for (const auto& e : kCallEntries)
if (e.op == op) return &e;
return nullptr;
}
size_t op_call_entry_count() { return sizeof(kCallEntries) / sizeof(kCallEntries[0]); }

std::string op_partial_coverage() {
std::string partial;
for (const auto& e : kCallEntries) {
if (!e.scope || !*e.scope) continue;
if (!partial.empty()) partial += ",";
partial += op_name(e.op);
}
return partial;
}
const OpCallEntry& op_call_entry(size_t i) { return kCallEntries[i]; }

} // namespace onebit::npu::plan
80 changes: 80 additions & 0 deletions npu/plan/op_registry.h
Original file line number Diff line number Diff line change
Expand Up @@ -84,6 +84,86 @@ const OpEntry& op_entry(size_t i);
// the exact string, so the gap is a decision rather than an omission.
std::string op_coverage();

// ── op-shaped calls (engine#799 rule (b), slice 2) ──────────────────────────────────────────────────
// `OpRefFn` above is GEMM-shaped (`a, m, k, n, b, out`), so the two ops whose operands are not a matrix
// pair -- `attn_step` and `gdn_step` -- could not be registered at all, which is why `op_coverage()` named
// them as gaps. `OpCall` is the operand set they need. The GEMM table stays as it is (its entries and
// cases are the shipped contract); this is an additive second shape and `op_coverage()` now means "no
// reference of EITHER shape", which is the honest question a plan asks.
struct OpCall {
OpId op = OpId::gemv;
// attn_step: one query against `pos` cached keys/values of one head.
// q [hd]
// kcache [pos][hd], vcache [pos][hd] (row-major, one row per cached position)
// out [hd] = softmax(q . k_i / sqrt(hd)) over i, applied to v
const float* q = nullptr;
const float* kcache = nullptr;
const float* vcache = nullptr;
int pos = 0, hd = 0;
float* out = nullptr;
// gdn_step: the delta-rule CORE of one layer step -- the stateful, order-sensitive half.
// state [dk][dv] in and out (the recurrence, advanced in place)
// k [dk], v [dv], q [dk] the layer's projected vectors for this position
// beta the write gate, g the log decay (g = 0 means no decay; exp(g) is applied to the state)
// The projections and norms around the core are gemv/elementwise ops covered by the first table, and
// `npu/target/target.h`'s CONTRACT says the layer takes them from what `load_block()` made resident --
// so a reference for the whole layer would have to be a reimplementation of it, which is not what this
// entry claims. Scope is stated here and in every case's `how`, because a partial oracle presented as a
// whole one is exactly the failure this registry exists to prevent.
float* state = nullptr;
const float* k = nullptr;
const float* v = nullptr;
int dk = 0, dv = 0;
float beta = 0.0f, g = 0.0f;
};
using OpCallRefFn = bool (*)(const OpCall&);

// One known-answer case for an op-shaped call: the case owns its operands, so the test can run it without
// knowing the op's shape. `want` is `hd` floats.
struct OpCallCase {
const char* what;
const char* how; // how the expectation was derived, same rule as OpCase::how
OpCall call; // operands; `out` is filled by the reference
const float* want;
int want_len;
};

struct OpCallEntry {
OpId op;
OpCallRefFn ref;
const OpCallCase* cases;
size_t n_cases;
// Empty when the reference covers the WHOLE op as a plan names it. Non-empty says which part it covers,
// because a partial oracle presented as a whole one is the failure this registry exists to prevent:
// `gdn_step` as a plan names it is a whole layer (conv, four projections, per-head norms and gates, then the
// delta rule), and only the delta rule is hand-derivable -- the rest comes from what `load_block()` made
// resident. `op_partial_coverage()` is what keeps that a stated gap instead of an implied coverage.
const char* scope;
};

const OpCallEntry* find_op_call(OpId op);

// The ops whose entry states a PARTIAL scope, comma separated ("" when every entry is whole). The companion
// question to op_coverage(): that one says "nothing registered", this one says "registered, and here is what it
// does not cover" -- so a plan naming such an op is known to be less guarded than one naming a whole entry.
std::string op_partial_coverage();
size_t op_call_entry_count();
const OpCallEntry& op_call_entry(size_t i);

// `attn_step`: one query, one head, causal-agnostic over the cached positions. Hand-derived answers in the
// cases: with hd = 2 and q = (1, 0), the first score is 1/sqrt(2) and the second 0, so the softmax is
// (0.6698, 0.3302) and the output (1.6605, 2.6605); with pos = 1 the softmax is 1 and the output IS v0.
bool attn_step_ref(const OpCall& call);

// `gdn_step`'s delta-rule core: S <- exp(g) * S; err = v - S^T k; S <- S + beta * k (x) err; out = S^T q.
// Hand-derived cases (dk = dv = 1, so each one is one line of arithmetic a reader can redo):
// S 1, k 2, v 3, g 0, beta 1, q 1 -> S stays 1, err = 3 - 1*2 = 1, S = 1 + 2*1 = 3, out = 3
// same but g = ln(0.5) -> S = 0.5, err = 3 - 0.5*2 = 2, S = 0.5 + 2*2 = 4.5, out = 4.5
// same but beta 0 -> S = 0.5 only (no write), out = 0.5
// The g = 0 case and the beta = 0 case together pin the ORDER (decay, then write) and that a zero gate
// really writes nothing; an implementation that added before decaying passes neither.
bool gdn_delta_ref(const OpCall& call);

// The two references this build implements. `gemv_ref` is C[m][n] = A[m][k] x B[k][n]; `lm_head_ref` is
// the same product with m == 1 (the plan layer's lm-head contract), kept separate so its own cases name the
// shape it is used at.
Expand Down
45 changes: 42 additions & 3 deletions tests/npu_plan_op_reference.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -67,10 +67,44 @@ void check_case(const onebit::npu::plan::OpEntry& e, const onebit::npu::plan::Op

} // namespace

// Rule (b) slice 2: the op-shaped table (engine#799). Same discipline as the GEMM table -- walk it, run every
// case, compare against the hand-derived answer -- and the case text says HOW each expectation was derived.
void test_op_call_table() {
using onebit::npu::plan::OpId;
using onebit::npu::plan::op_call_entry;
using onebit::npu::plan::op_call_entry_count;
check(op_call_entry_count() > 0, "the op-shaped registry has at least one entry");
for (size_t i = 0; i < op_call_entry_count(); i++) {
const auto& e = op_call_entry(i);
const std::string label = std::string("op-call ") + onebit::npu::plan::op_name(e.op);
check(e.ref != nullptr, label + " entry has a reference");
check(e.n_cases > 0, label + " entry has known-answer cases");
for (size_t c = 0; c < e.n_cases; c++) {
const auto& kase = e.cases[c];
const std::string what = std::string(label) + " case \"" + (kase.what ? kase.what : "?") + "\"";
check(kase.what && kase.how && *kase.how, what + " states how its answer was derived");
std::vector<float> out(size_t(kase.want_len > 0 ? kase.want_len : kase.call.hd), 0.0f);
auto call = kase.call;
call.out = out.data();
if (call.state && call.dk > 0 && call.dv > 0)
for (int i2 = 0; i2 < call.dk * call.dv; ++i2) call.state[i2] = 1.0f;
check(e.ref(call), what + " reference returns true");
float worst = 0.0f;
for (int d = 0; d < kase.want_len; ++d) worst = std::max(worst, std::fabs(out[size_t(d)] - kase.want[d]));
check(worst <= 1e-5f, what + " matches the hand-derived answer (worst " + std::to_string(worst) + ")");
}
}
// The vacuum check: a malformed call must be refused rather than dereferenced.
auto bad = onebit::npu::plan::OpCall{};
bad.op = OpId::attn_step;
check(!onebit::npu::plan::attn_step_ref(bad), "op-call attn_step refuses a call with no operands");
}

int main() {
using namespace onebit::npu::plan;

// 1. The table is walked, not named.
test_op_call_table();
check(op_entry_count() > 0, "the registry has at least one entry");
for (size_t i = 0; i < op_entry_count(); i++) {
const OpEntry& e = op_entry(i);
Expand Down Expand Up @@ -115,10 +149,15 @@ int main() {
check(find_op(OpId::attn_step) == nullptr, "attn_step has no entry yet");
check(find_op("not_an_op") == nullptr, "an unknown name finds nothing");

// 3. The coverage gap is a decision in this test, not an omission: exactly these two ops are missing.
// 3. Coverage is now two questions, and this test is where both are decisions rather than omissions:
// nothing is MISSING any more (attn_step and gdn_step are both in the op-shaped table), and the one
// PARTIAL entry is named -- gdn_step's reference covers the delta-rule core, not the whole layer, and a
// partial oracle presented as a whole one is the failure this registry exists to prevent.
const std::string missing = op_coverage();
check(missing == "gdn_step,attn_step",
"op_coverage() names exactly the unregistered ops (got \"" + missing + "\")");
check(missing.empty(), "op_coverage() names no unregistered op (got \"" + missing + "\")");
const std::string partial = onebit::npu::plan::op_partial_coverage();
check(partial == "gdn_step",
"op_partial_coverage() names exactly the partially covered op (got \"" + partial + "\")");

// 4. A reference that is handed bad dimensions refuses rather than computing garbage.
float out[4] = {0.0f, 0.0f, 0.0f, 0.0f};
Expand Down
Loading