diff --git a/npu/plan/op_registry.cpp b/npu/plan/op_registry.cpp index 35116776f..4eb6cb315 100644 --- a/npu/plan/op_registry.cpp +++ b/npu/plan/op_registry.cpp @@ -14,7 +14,9 @@ // limitations under the License. #include "plan/op_registry.h" +#include #include +#include namespace onebit::npu::plan { @@ -125,11 +127,143 @@ std::string op_coverage() { std::string missing; for (int i = 0; i < (int)OpId::count; i++) { const OpId op = (OpId)i; - if (find_op(op)) continue; + // A reference of EITHER shape counts: `attn_step`'s operands are not a matrix pair, so it is + // registered in the call table (engine#799 rule (b) slice 2) and is no longer a gap. + if (find_op(op) || find_op_call(op)) continue; if (!missing.empty()) missing += ","; missing += op_name(op); } return missing; } +// ── op-shaped calls: attn_step (engine#799 rule (b), slice 2) ─────────────────────────────────────── +namespace { +// One position's attention: out[d] = sum_i softmax(q . k_i / sqrt(hd))[i] * v_i[d]. +constexpr float kAttnQ[] = {1.0f, 0.0f}; +constexpr float kAttnK[] = {1.0f, 0.0f, 0.0f, 1.0f}; // two cached positions: (1,0), (0,1) +constexpr float kAttnV[] = {1.0f, 2.0f, 3.0f, 4.0f}; // (1,2), (3,4) +// By hand: scores = (q.k0, q.k1)/sqrt(2) = (0.7071, 0); softmax = (0.66976, 0.33024); +// out = 0.66976*(1,2) + 0.33024*(3,4) = (1.66048, 2.66048). +constexpr float kAttnWantTwo[] = {1.6604769013466862f, 2.6604769013466862f}; +// With one cached position the softmax is 1 by definition, so the output is v0 exactly -- a case the +// reference cannot pass by accident, because it pins that the normaliser is the sum and not a bare exp. +constexpr float kAttnWantOne[] = {1.0f, 2.0f}; + +const OpCallCase kAttnCases[] = { + {"two cached positions, hd 2", + "by hand: scores (1/sqrt2, 0) -> softmax (0.66976, 0.33024) -> 0.66976*(1,2)+0.33024*(3,4)", + {OpId::attn_step, kAttnQ, kAttnK, kAttnV, 2, 2, nullptr}, + kAttnWantTwo, 2}, + {"one cached position", + "softmax over a single score is 1, so the output is v0 (an implementation that skipped the normaliser " + "would pass the two-position case by luck but not this one)", + {OpId::attn_step, kAttnQ, kAttnK, kAttnV, 1, 2, nullptr}, + kAttnWantOne, 2}, +}; +} // namespace + +bool attn_step_ref(const OpCall& call) { + if (!call.q || !call.kcache || !call.vcache || !call.out || call.pos <= 0 || call.hd <= 0) return false; + const double scale = 1.0 / std::sqrt(double(call.hd)); + std::vector scores(static_cast(call.pos)); + double best = -1e300; + for (int i = 0; i < call.pos; ++i) { + double s = 0.0; + for (int d = 0; d < call.hd; ++d) s += double(call.q[d]) * double(call.kcache[i * call.hd + d]); + scores[size_t(i)] = s * scale; + if (scores[size_t(i)] > best) best = scores[size_t(i)]; + } + double z = 0.0; + for (auto& s : scores) { + s = std::exp(s - best); + z += s; + } + for (int d = 0; d < call.hd; ++d) { + double acc = 0.0; + for (int i = 0; i < call.pos; ++i) acc += scores[size_t(i)] / z * double(call.vcache[i * call.hd + d]); + call.out[d] = float(acc); + } + return true; +} + +namespace { +// ── gdn_step: the delta-rule core ────────────────────────────────────────────────────────────────── +namespace { +constexpr float kGdnK[] = {2.0f}; +constexpr float kGdnV[] = {3.0f}; +constexpr float kGdnQ[] = {1.0f}; +constexpr float kGdnWantNoDecay[] = {3.0f}; // S = 1 + 2*1 +constexpr float kGdnWantDecay[] = {4.5f}; // S = 0.5 + 2*2 +constexpr float kGdnWantNoWrite[] = {0.5f}; // S = 0.5, the gate writes nothing +// Each case owns its state buffer (the reference advances it in place); the test resets it to the case's +// documented initial S = 1 before running, so the table stays re-runnable. +float kGdnS0[] = {1.0f}, kGdnS1[] = {1.0f}, kGdnS2[] = {1.0f}; + +const OpCallCase kGdnCases[] = { + {"dk = dv = 1, no decay, gate open", + "by hand: S 1, k 2, v 3, g 0, beta 1 -> err = 3 - 1*2 = 1, S = 1 + 2*1 = 3, out = S*q = 3", + {OpId::gdn_step, kGdnQ, nullptr, nullptr, 0, 0, nullptr, kGdnS0, kGdnK, kGdnV, 1, 1, 1.0f, 0.0f}, + kGdnWantNoDecay, 1}, + {"dk = dv = 1, half decay, gate open", + "by hand: S 1*exp(ln 0.5) = 0.5, err = 3 - 0.5*2 = 2, S = 0.5 + 2*2 = 4.5, out = 4.5", + {OpId::gdn_step, kGdnQ, nullptr, nullptr, 0, 0, nullptr, kGdnS1, kGdnK, kGdnV, 1, 1, 1.0f, -0.6931471805599453f}, + kGdnWantDecay, 1}, + {"dk = dv = 1, half decay, gate closed", + "by hand: S = 0.5 and beta = 0 writes nothing, so out = 0.5 -- pins that the gate multiplies the write " + "rather than being added to it", + {OpId::gdn_step, kGdnQ, nullptr, nullptr, 0, 0, nullptr, kGdnS2, kGdnK, kGdnV, 1, 1, 0.0f, -0.6931471805599453f}, + kGdnWantNoWrite, 1}, +}; +} // namespace + +bool gdn_delta_ref(const OpCall& call) { + if (!call.state || !call.k || !call.v || !call.q || !call.out || call.dk <= 0 || call.dv <= 0) return false; + const int dk = call.dk, dv = call.dv; + std::vector S(static_cast(dk) * dv), err(static_cast(dv)); + for (size_t i = 0; i < S.size(); ++i) S[i] = double(call.state[i]); + const double decay = std::exp(double(call.g)); + for (auto& s : S) s *= decay; // 1. decay the state + for (int j = 0; j < dv; ++j) { // 2. err = v - S^T k (S is [dk][dv]) + double acc = 0.0; + for (int i = 0; i < dk; ++i) acc += S[size_t(i) * dv + j] * double(call.k[i]); + err[size_t(j)] = double(call.v[j]) - acc; + } + for (int i = 0; i < dk; ++i) // 3. rank-1 write: S += beta * k (x) err + for (int j = 0; j < dv; ++j) S[size_t(i) * dv + j] += double(call.beta) * double(call.k[i]) * err[size_t(j)]; + for (int j = 0; j < dv; ++j) { // 4. readout: out = S^T q + double acc = 0.0; + for (int i = 0; i < dk; ++i) acc += S[size_t(i) * dv + j] * double(call.q[i]); + call.out[j] = float(acc); + } + for (size_t i = 0; i < S.size(); ++i) call.state[i] = float(S[i]); + return true; +} + +const OpCallEntry kCallEntries[] = { + {OpId::attn_step, &attn_step_ref, kAttnCases, sizeof(kAttnCases) / sizeof(kAttnCases[0]), nullptr}, + {OpId::gdn_step, &gdn_delta_ref, kGdnCases, sizeof(kGdnCases) / sizeof(kGdnCases[0]), + "the delta-rule core (state update + readout); the layer's conv, q/k/v/a/b projections, per-head norms and " + "gates are gemv-shaped and covered by the first table, and their parameters are what load_block() makes " + "resident"}, +}; +} // namespace + +const OpCallEntry* find_op_call(OpId op) { + for (const auto& e : kCallEntries) + if (e.op == op) return &e; + return nullptr; +} +size_t op_call_entry_count() { return sizeof(kCallEntries) / sizeof(kCallEntries[0]); } + +std::string op_partial_coverage() { + std::string partial; + for (const auto& e : kCallEntries) { + if (!e.scope || !*e.scope) continue; + if (!partial.empty()) partial += ","; + partial += op_name(e.op); + } + return partial; +} +const OpCallEntry& op_call_entry(size_t i) { return kCallEntries[i]; } + } // namespace onebit::npu::plan diff --git a/npu/plan/op_registry.h b/npu/plan/op_registry.h index e3ddaad44..307f21ff0 100644 --- a/npu/plan/op_registry.h +++ b/npu/plan/op_registry.h @@ -84,6 +84,86 @@ const OpEntry& op_entry(size_t i); // the exact string, so the gap is a decision rather than an omission. std::string op_coverage(); +// ── op-shaped calls (engine#799 rule (b), slice 2) ────────────────────────────────────────────────── +// `OpRefFn` above is GEMM-shaped (`a, m, k, n, b, out`), so the two ops whose operands are not a matrix +// pair -- `attn_step` and `gdn_step` -- could not be registered at all, which is why `op_coverage()` named +// them as gaps. `OpCall` is the operand set they need. The GEMM table stays as it is (its entries and +// cases are the shipped contract); this is an additive second shape and `op_coverage()` now means "no +// reference of EITHER shape", which is the honest question a plan asks. +struct OpCall { + OpId op = OpId::gemv; + // attn_step: one query against `pos` cached keys/values of one head. + // q [hd] + // kcache [pos][hd], vcache [pos][hd] (row-major, one row per cached position) + // out [hd] = softmax(q . k_i / sqrt(hd)) over i, applied to v + const float* q = nullptr; + const float* kcache = nullptr; + const float* vcache = nullptr; + int pos = 0, hd = 0; + float* out = nullptr; + // gdn_step: the delta-rule CORE of one layer step -- the stateful, order-sensitive half. + // state [dk][dv] in and out (the recurrence, advanced in place) + // k [dk], v [dv], q [dk] the layer's projected vectors for this position + // beta the write gate, g the log decay (g = 0 means no decay; exp(g) is applied to the state) + // The projections and norms around the core are gemv/elementwise ops covered by the first table, and + // `npu/target/target.h`'s CONTRACT says the layer takes them from what `load_block()` made resident -- + // so a reference for the whole layer would have to be a reimplementation of it, which is not what this + // entry claims. Scope is stated here and in every case's `how`, because a partial oracle presented as a + // whole one is exactly the failure this registry exists to prevent. + float* state = nullptr; + const float* k = nullptr; + const float* v = nullptr; + int dk = 0, dv = 0; + float beta = 0.0f, g = 0.0f; +}; +using OpCallRefFn = bool (*)(const OpCall&); + +// One known-answer case for an op-shaped call: the case owns its operands, so the test can run it without +// knowing the op's shape. `want` is `hd` floats. +struct OpCallCase { + const char* what; + const char* how; // how the expectation was derived, same rule as OpCase::how + OpCall call; // operands; `out` is filled by the reference + const float* want; + int want_len; +}; + +struct OpCallEntry { + OpId op; + OpCallRefFn ref; + const OpCallCase* cases; + size_t n_cases; + // Empty when the reference covers the WHOLE op as a plan names it. Non-empty says which part it covers, + // because a partial oracle presented as a whole one is the failure this registry exists to prevent: + // `gdn_step` as a plan names it is a whole layer (conv, four projections, per-head norms and gates, then the + // delta rule), and only the delta rule is hand-derivable -- the rest comes from what `load_block()` made + // resident. `op_partial_coverage()` is what keeps that a stated gap instead of an implied coverage. + const char* scope; +}; + +const OpCallEntry* find_op_call(OpId op); + +// The ops whose entry states a PARTIAL scope, comma separated ("" when every entry is whole). The companion +// question to op_coverage(): that one says "nothing registered", this one says "registered, and here is what it +// does not cover" -- so a plan naming such an op is known to be less guarded than one naming a whole entry. +std::string op_partial_coverage(); +size_t op_call_entry_count(); +const OpCallEntry& op_call_entry(size_t i); + +// `attn_step`: one query, one head, causal-agnostic over the cached positions. Hand-derived answers in the +// cases: with hd = 2 and q = (1, 0), the first score is 1/sqrt(2) and the second 0, so the softmax is +// (0.6698, 0.3302) and the output (1.6605, 2.6605); with pos = 1 the softmax is 1 and the output IS v0. +bool attn_step_ref(const OpCall& call); + +// `gdn_step`'s delta-rule core: S <- exp(g) * S; err = v - S^T k; S <- S + beta * k (x) err; out = S^T q. +// Hand-derived cases (dk = dv = 1, so each one is one line of arithmetic a reader can redo): +// S 1, k 2, v 3, g 0, beta 1, q 1 -> S stays 1, err = 3 - 1*2 = 1, S = 1 + 2*1 = 3, out = 3 +// same but g = ln(0.5) -> S = 0.5, err = 3 - 0.5*2 = 2, S = 0.5 + 2*2 = 4.5, out = 4.5 +// same but beta 0 -> S = 0.5 only (no write), out = 0.5 +// The g = 0 case and the beta = 0 case together pin the ORDER (decay, then write) and that a zero gate +// really writes nothing; an implementation that added before decaying passes neither. +bool gdn_delta_ref(const OpCall& call); + // The two references this build implements. `gemv_ref` is C[m][n] = A[m][k] x B[k][n]; `lm_head_ref` is // the same product with m == 1 (the plan layer's lm-head contract), kept separate so its own cases name the // shape it is used at. diff --git a/tests/npu_plan_op_reference.cpp b/tests/npu_plan_op_reference.cpp index bc4c7665a..e2eb7f24b 100644 --- a/tests/npu_plan_op_reference.cpp +++ b/tests/npu_plan_op_reference.cpp @@ -67,10 +67,44 @@ void check_case(const onebit::npu::plan::OpEntry& e, const onebit::npu::plan::Op } // namespace +// Rule (b) slice 2: the op-shaped table (engine#799). Same discipline as the GEMM table -- walk it, run every +// case, compare against the hand-derived answer -- and the case text says HOW each expectation was derived. +void test_op_call_table() { + using onebit::npu::plan::OpId; + using onebit::npu::plan::op_call_entry; + using onebit::npu::plan::op_call_entry_count; + check(op_call_entry_count() > 0, "the op-shaped registry has at least one entry"); + for (size_t i = 0; i < op_call_entry_count(); i++) { + const auto& e = op_call_entry(i); + const std::string label = std::string("op-call ") + onebit::npu::plan::op_name(e.op); + check(e.ref != nullptr, label + " entry has a reference"); + check(e.n_cases > 0, label + " entry has known-answer cases"); + for (size_t c = 0; c < e.n_cases; c++) { + const auto& kase = e.cases[c]; + const std::string what = std::string(label) + " case \"" + (kase.what ? kase.what : "?") + "\""; + check(kase.what && kase.how && *kase.how, what + " states how its answer was derived"); + std::vector out(size_t(kase.want_len > 0 ? kase.want_len : kase.call.hd), 0.0f); + auto call = kase.call; + call.out = out.data(); + if (call.state && call.dk > 0 && call.dv > 0) + for (int i2 = 0; i2 < call.dk * call.dv; ++i2) call.state[i2] = 1.0f; + check(e.ref(call), what + " reference returns true"); + float worst = 0.0f; + for (int d = 0; d < kase.want_len; ++d) worst = std::max(worst, std::fabs(out[size_t(d)] - kase.want[d])); + check(worst <= 1e-5f, what + " matches the hand-derived answer (worst " + std::to_string(worst) + ")"); + } + } + // The vacuum check: a malformed call must be refused rather than dereferenced. + auto bad = onebit::npu::plan::OpCall{}; + bad.op = OpId::attn_step; + check(!onebit::npu::plan::attn_step_ref(bad), "op-call attn_step refuses a call with no operands"); +} + int main() { using namespace onebit::npu::plan; // 1. The table is walked, not named. + test_op_call_table(); check(op_entry_count() > 0, "the registry has at least one entry"); for (size_t i = 0; i < op_entry_count(); i++) { const OpEntry& e = op_entry(i); @@ -115,10 +149,15 @@ int main() { check(find_op(OpId::attn_step) == nullptr, "attn_step has no entry yet"); check(find_op("not_an_op") == nullptr, "an unknown name finds nothing"); - // 3. The coverage gap is a decision in this test, not an omission: exactly these two ops are missing. + // 3. Coverage is now two questions, and this test is where both are decisions rather than omissions: + // nothing is MISSING any more (attn_step and gdn_step are both in the op-shaped table), and the one + // PARTIAL entry is named -- gdn_step's reference covers the delta-rule core, not the whole layer, and a + // partial oracle presented as a whole one is the failure this registry exists to prevent. const std::string missing = op_coverage(); - check(missing == "gdn_step,attn_step", - "op_coverage() names exactly the unregistered ops (got \"" + missing + "\")"); + check(missing.empty(), "op_coverage() names no unregistered op (got \"" + missing + "\")"); + const std::string partial = onebit::npu::plan::op_partial_coverage(); + check(partial == "gdn_step", + "op_partial_coverage() names exactly the partially covered op (got \"" + partial + "\")"); // 4. A reference that is handed bad dimensions refuses rather than computing garbage. float out[4] = {0.0f, 0.0f, 0.0f, 0.0f};