diff --git a/CMakeLists.txt b/CMakeLists.txt index 381553e54..12c4803fb 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -444,6 +444,13 @@ if(TARGET OpenMP::OpenMP_CXX) target_link_libraries(mtp_draft_test PRIVATE OpenMP::OpenMP_CXX) endif() add_test(NAME mtp_draft COMMAND mtp_draft_test) +# engine#1781 gate 2: the distribution-following acceptance rule (accept with min(1, p/q), resample the residual at the +# first rejection), with the temperature-0 identity and a deliberately wrong drafter whose output must still follow p. +# Header-only and device-free; pulled into CI through npu_sampling_test for the same reason as mtp_draft_test below. +add_executable(spec_accept_test tests/spec_accept_test.cpp) +target_include_directories(spec_accept_test PRIVATE npu) +add_test(NAME spec_accept COMMAND spec_accept_test) +add_dependencies(npu_sampling_test spec_accept_test) # ci.yml's build step lists every ctest target explicitly, and this branch cannot edit that file (the push # token has no `workflow` scope), so the binary is pulled in as a dependency of a listed test target. When a # workflow-scoped push is available, add `mtp_draft_test` to ci.yml's build list and delete this line. diff --git a/docs/npu.md b/docs/npu.md index aa30b1795..ab1a76b58 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -2163,6 +2163,19 @@ is worth more than the whole bar. Those depths keep their PASS/FAIL (the bar is `(thin: N live chain(s), one chain = X pp > the 2 pp bar)` and are counted out of the summary line, because a bare FAIL on 2 chains reads as a drafter regression when it is arithmetic. +**Which drafts a sampler may keep** is a separate rule from how they are committed, and it lives in +`npu/mtp/spec_accept.h` (engine#1781, gate 2). The loop above accepts a draft when it equals the base model's argmax, +which is exact for greedy decode only. With a sampler the contract is that the output follows the **target's** +distribution, and the rule that guarantees it is: accept a drafted token `x` (drawn from the drafter's distribution `q`) +with probability `min(1, p(x)/q(x))`, and at the first rejection emit a token from `normalise(max(0, p - q))` and stop +the chain. `p` and `q` are the final sampling distributions (after temperature, top-k, top-p), and an argmax drafter +states a one-hot `q`. At temperature 0 both rows are one-hot and the rule reduces to the greedy contract, so that +contract is its degenerate case. `tests/spec_accept_test.cpp` (ctest `spec_accept`) pins both: the temperature-0 +identity, and a deliberately wrong drafter (`p = [.45, .35, .20]`, `q = [.10, .80, .10]`) whose emitted tokens are +within 0.0016 of `p` over 200,000 draws at an acceptance rate of 0.55, where the greedy rule on the same rows misses +`p` by 0.55. The header is not called by a serve yet: wiring a speculative loop into `forward-serve` is the rest of +#1781, and that loop must print its acceptance rate beside its tok/s. + **The one thing that must be right**: the drafter attends over its own states, so its cache has to be the base model's *committed stream, row by row*. Feeding it only the current chain is silent — drafts get worse, the text stays correct — and it cost this project 69 % → 14 % first-draft acceptance until the cursor was made a checked diff --git a/npu/mtp/spec_accept.h b/npu/mtp/spec_accept.h new file mode 100644 index 000000000..1eeeadc34 --- /dev/null +++ b/npu/mtp/spec_accept.h @@ -0,0 +1,122 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// spec_accept.h -- the acceptance rule of a speculative decode whose output follows the TARGET's distribution +// (engine#1781, gate 2). +// +// draft.h's loop accepts a drafted token when it equals the base model's argmax. That is exact for greedy decode and +// for nothing else: with a sampler, "accept iff the draft is the target's most likely token, otherwise emit that +// token" always emits the argmax, which is a different model's output (0.55 away from p on p = [.45,.35,.20] with +// q = [.10,.80,.10] -- the arm in tests/spec_accept_test.cpp). The rule that preserves the distribution is the +// standard one, for a drafted token x drawn from the drafter's distribution q at a position where the target's +// distribution is p: +// +// accept x with probability min(1, p(x) / q(x)); +// on the first rejection, emit a token drawn from normalise(max(0, p - q)) and stop the chain. +// +// The emitted token is then distributed exactly as p, whatever q is; q only decides how often a draft survives +// (the acceptance probability is sum_i min(p_i, q_i)). At temperature 0 both rows are one-hot, the ratio is 0 or 1, +// and the rule IS "accept iff the drafter's token is the target's argmax, else emit the argmax" -- so draft.h's +// greedy contract is this rule's degenerate case and not a second rule. +// +// What the caller owes: +// * p is the target's FINAL sampling distribution at that position -- after temperature, top-k, top-p and any +// penalty -- and q is the drafter's, in the same sense. A deterministic drafter (an argmax head) has a one-hot q. +// * x was drawn from q. If q(x) is 0 the premise is false; the step then rejects and resamples, so the output stays +// inside p's support, but nothing is promised about a drafter that lies about its own distribution. +// * the uniforms are independent draws in [0, 1). The functions take them as arguments and own no generator, so a +// run is reproducible from the caller's seed and a test can drive every branch. +// +// Pure and header-only: no model, no device, no allocation. It decides WHICH rows are kept; begin_verify() / +// commit_verify_rows() decide how the kept rows are committed. +#pragma once + +#include + +namespace onebit { +namespace npu { +namespace mtp { + +struct SpecStep { + bool accepted; // the drafted token survives + int token; // the token to emit at this position: the draft when accepted, the residual sample otherwise +}; + +// A token from normalise(max(0, p - q)), by inverse CDF on u in [0, 1). When the residual has no mass (p == q up to +// rounding, where a rejection has probability 0 and can only come from float error) it samples p itself. Never +// returns an index whose weight is zero, and returns -1 only for V <= 0 or a row with no positive mass at all. +inline int spec_residual_sample(const float* p, const float* q, int V, double u) { + if (!p || V <= 0) return -1; + double mass = 0.0; + if (q) + for (int i = 0; i < V; ++i) { + const double r = double(p[i]) - double(q[i]); + if (r > 0.0) mass += r; + } + const bool use_p = !(mass > 0.0); + if (use_p) { + mass = 0.0; + for (int i = 0; i < V; ++i) + if (p[i] > 0.0f) mass += double(p[i]); + if (!(mass > 0.0)) return -1; + } + if (u < 0.0) u = 0.0; + const double target = u * mass; + double acc = 0.0; + int last = -1; + for (int i = 0; i < V; ++i) { + const double w = use_p ? (p[i] > 0.0f ? double(p[i]) : 0.0) : (double(p[i]) - double(q[i])); + if (!(w > 0.0)) continue; + acc += w; + last = i; + if (target < acc) return i; + } + return last; // u so close to 1 that rounding left `target` at or past the total: the last weighted index +} + +// One drafted position. `x` is the token the drafter drew from q. +inline SpecStep spec_accept_step(const float* p, const float* q, int V, int x, double u_accept, double u_resample) { + if (p && q && x >= 0 && x < V && q[x] > 0.0f) { + // u < min(1, p/q) written without the division: u * q < p. With p >= q it holds for every u in [0, 1). + if (u_accept * double(q[x]) < double(p[x])) return {true, x}; + } + return {false, spec_residual_sample(p, q, V, u_resample)}; +} + +struct SpecResult { + int accepted; // how many leading drafts survive: 0..k + int next_token; // the residual sample at the first rejection; -1 when all k survive (the caller then samples + // position k from the target's own row there, which is the pass's k+1-th row) +}; + +// The rule along a chain of k drafted positions. p_rows and q_rows are k rows of V probabilities, row d being the +// distributions at drafted position d; `uniform()` returns the next independent draw in [0, 1). Exactly two draws are +// consumed per position examined, accepted or not, so the number of draws does not leak the outcome into the stream. +template +inline SpecResult spec_accept_prefix(const float* p_rows, const float* q_rows, int V, const int* drafts, int k, + Uniform&& uniform) { + for (int d = 0; d < k; ++d) { + const double ua = uniform(); + const double ur = uniform(); + const SpecStep s = spec_accept_step(p_rows + std::size_t(d) * std::size_t(V), + q_rows + std::size_t(d) * std::size_t(V), V, drafts[d], ua, ur); + if (!s.accepted) return {d, s.token}; + } + return {k, -1}; +} + +} // namespace mtp +} // namespace npu +} // namespace onebit diff --git a/tests/spec_accept_test.cpp b/tests/spec_accept_test.cpp new file mode 100644 index 000000000..c2644af7e --- /dev/null +++ b/tests/spec_accept_test.cpp @@ -0,0 +1,243 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// +// spec_accept_test -- npu/mtp/spec_accept.h, the distribution-following acceptance rule (engine#1781, gate 2). +// +// Two properties make the rule what the issue asks for, and each has an arm that a wrong rule fails: +// +// 1. at temperature 0 (one-hot rows) it is exactly "accept iff the draft is the target's argmax, else emit the +// argmax" -- the contract tests/mtp_draft_test.cpp already pins for the greedy loop; +// 2. with a sampler, the emitted token follows the TARGET's distribution p whatever the drafter's q is -- checked +// with a drafter that is deliberately wrong, and against the greedy rule, which misses p by 0.55 on the same rows. +// +// No model and no device: a fixed-seed generator, so the counts below are the same on every run and the tolerances +// are bounds on a known sample, not on luck. The sampling error of a frequency over N = 200,000 draws is at most +// sqrt(0.25 / N) = 0.0011 (one sigma), so 0.006 is more than five sigma and 0.3 is nowhere near it. +#include "mtp/spec_accept.h" + +#include +#include +#include +#include + +using onebit::npu::mtp::spec_accept_prefix; +using onebit::npu::mtp::spec_accept_step; +using onebit::npu::mtp::spec_residual_sample; +using onebit::npu::mtp::SpecResult; +using onebit::npu::mtp::SpecStep; + +static int g_fail = 0; +static void check(bool ok, const char* what) { + std::printf("%s %s\n", ok ? "PASS" : "FAIL", what); + if (!ok) g_fail++; +} + +struct Rng { // splitmix64: small, fixed, and good enough to make 200k draws look uniform + uint64_t s; + double operator()() { + uint64_t z = (s += 0x9E3779B97F4A7C15ull); + z = (z ^ (z >> 30)) * 0xBF58476D1CE4E5B9ull; + z = (z ^ (z >> 27)) * 0x94D049BB133111EBull; + z ^= z >> 31; + return double(z >> 11) * (1.0 / 9007199254740992.0); // [0, 1) + } +}; + +static int draw(const std::vector& d, double u) { + double acc = 0.0; + for (size_t i = 0; i < d.size(); ++i) { acc += d[i]; if (u < acc) return int(i); } + return int(d.size()) - 1; +} + +// Emitted-token frequencies and the acceptance rate of one position over n draws, with the drafter sampling q. +static double max_dev_from_p(const std::vector& p, const std::vector& q, int n, uint64_t seed, + double* accept_rate, bool greedy_rule = false) { + const int V = int(p.size()); + std::vector cnt(size_t(V), 0); + long acc = 0; + Rng r{seed}; + int amax = 0; + for (int i = 1; i < V; ++i) if (p[size_t(i)] > p[size_t(amax)]) amax = i; + for (int i = 0; i < n; ++i) { + const int x = draw(q, r()); + int out; + if (greedy_rule) { // the rule this header replaces: accept iff x is the target's argmax + const double ua = r(), ur = r(); (void)ua; (void)ur; + if (x == amax) acc++; + out = amax; + } else { + const SpecStep s = spec_accept_step(p.data(), q.data(), V, x, r(), r()); + if (s.accepted) acc++; + out = s.token; + } + if (out >= 0 && out < V) cnt[size_t(out)]++; + } + double worst = 0.0; + for (int i = 0; i < V; ++i) worst = std::fmax(worst, std::fabs(double(cnt[size_t(i)]) / n - double(p[size_t(i)]))); + if (accept_rate) *accept_rate = double(acc) / n; + return worst; +} + +int main() { + const int N = 200000; + char msg[256]; + + // ---- 1. temperature 0: one-hot rows, the greedy contract ------------------------------------------------------ + { + const std::vector pa = {0, 1, 0, 0}, qa = {0, 1, 0, 0}, qb = {0, 0, 1, 0}; + bool all_acc = true, none_acc = true, emit_ok = true; + Rng r{1}; + for (int i = 0; i < 1000; ++i) { + const SpecStep s = spec_accept_step(pa.data(), qa.data(), 4, 1, r(), r()); + all_acc &= s.accepted && s.token == 1; + const SpecStep t = spec_accept_step(pa.data(), qb.data(), 4, 2, r(), r()); + none_acc &= !t.accepted; + emit_ok &= t.token == 1; + } + check(all_acc, "temp 0: a draft equal to the target's argmax is accepted on every draw"); + check(none_acc, "temp 0: a draft that is not the target's argmax is rejected on every draw"); + check(emit_ok, "temp 0: the token emitted at a rejection is the target's argmax"); + } + + // ---- 2. a sampler, with a deliberately wrong drafter: the output still follows p ------------------------------ + { + const std::vector p = {0.45f, 0.35f, 0.20f}, q = {0.10f, 0.80f, 0.10f}; + double rate = 0.0; + const double dev = max_dev_from_p(p, q, N, 42, &rate); + std::snprintf(msg, sizeof msg, "poor drafter: emitted frequencies are within 0.006 of p (max deviation %.4f)", dev); + check(dev < 0.006, msg); + std::snprintf(msg, sizeof msg, "poor drafter: acceptance rate is sum(min(p, q)) = 0.55 (measured %.4f)", rate); + check(std::fabs(rate - 0.55) < 0.006, msg); + double grate = 0.0; + const double gdev = max_dev_from_p(p, q, N, 42, &grate, true); + std::snprintf(msg, sizeof msg, "the GREEDY rule on the same rows misses p by more than 0.3 (max deviation %.4f): " + "this arm tells the two rules apart", gdev); + check(gdev > 0.3, msg); + } + { + const std::vector p = {0.45f, 0.35f, 0.20f}, q = {0.40f, 0.38f, 0.22f}; + double rate = 0.0; + const double dev = max_dev_from_p(p, q, N, 7, &rate); + std::snprintf(msg, sizeof msg, "good drafter: follows p (max deviation %.4f) and accepts 0.95 (measured %.4f)", dev, rate); + check(dev < 0.006 && std::fabs(rate - 0.95) < 0.006, msg); + } + { + const std::vector p = {0.5f, 0.3f, 0.2f}; + double rate = 0.0; + const double dev = max_dev_from_p(p, p, N, 9, &rate); + std::snprintf(msg, sizeof msg, "identical drafter: every draft is accepted (rate %.6f) and the output follows p (%.4f)", rate, dev); + check(rate == 1.0 && dev < 0.006, msg); + } + { + const std::vector p = {0.6f, 0.4f, 0.0f, 0.0f}, q = {0.0f, 0.0f, 0.5f, 0.5f}; + double rate = 1.0; + const double dev = max_dev_from_p(p, q, N, 11, &rate); + std::snprintf(msg, sizeof msg, "disjoint drafter: nothing is accepted (rate %.6f) and the output still follows p (%.4f)", rate, dev); + check(rate == 0.0 && dev < 0.006, msg); + } + { // a deterministic (argmax) drafter states a one-hot q: accepted with probability p(x), residual is p without x + const std::vector p = {0.2f, 0.5f, 0.3f}, q = {0.0f, 1.0f, 0.0f}; + double rate = 0.0; + const double dev = max_dev_from_p(p, q, N, 13, &rate); + std::snprintf(msg, sizeof msg, "one-hot drafter: accepted with probability p(x) = 0.5 (measured %.4f), output follows p (%.4f)", rate, dev); + check(std::fabs(rate - 0.5) < 0.006 && dev < 0.006, msg); + } + { // a wide vocabulary with random rows: the property is not an artefact of three tokens + const int V = 257; + Rng g{2024}; + std::vector p(V), q(V); + double sp = 0, sq = 0; + for (int i = 0; i < V; ++i) { p[size_t(i)] = float(g() * g()); q[size_t(i)] = float(g() * g()); sp += p[size_t(i)]; sq += q[size_t(i)]; } + double want = 0; + for (int i = 0; i < V; ++i) { p[size_t(i)] = float(p[size_t(i)] / sp); q[size_t(i)] = float(q[size_t(i)] / sq); } + for (int i = 0; i < V; ++i) want += std::fmin(p[size_t(i)], q[size_t(i)]); + double rate = 0.0; + const double dev = max_dev_from_p(p, q, 2 * N, 5, &rate); + std::snprintf(msg, sizeof msg, "257 tokens, random rows: output within 0.003 of p (%.4f), acceptance %.4f against sum(min) %.4f", dev, rate, want); + check(dev < 0.003 && std::fabs(rate - want) < 0.006, msg); + } + + // ---- 3. the edges a wiring will hit --------------------------------------------------------------------------- + { + const std::vector p = {0.5f, 0.5f, 0.0f}, q = {1.0f, 0.0f, 0.0f}; + bool ok = true; + Rng r{3}; + for (int i = 0; i < 2000; ++i) { + const SpecStep s = spec_accept_step(p.data(), q.data(), 3, 1, r(), r()); // x has q(x) == 0 + ok &= !s.accepted && (s.token == 0 || s.token == 1); + } + check(ok, "a draft the drafter gives no mass to is rejected, and the emitted token stays in p's support"); + const SpecStep bad = spec_accept_step(p.data(), q.data(), 3, 7, 0.0, 0.5); + check(!bad.accepted && bad.token >= 0 && bad.token < 2, "an out-of-range draft id is rejected and resampled, not read"); + } + { + const std::vector p = {0.25f, 0.0f, 0.75f, 0.0f}, q = {0.25f, 0.5f, 0.25f, 0.0f}; + check(spec_residual_sample(p.data(), q.data(), 4, 0.0) == 2 && spec_residual_sample(p.data(), q.data(), 4, 0.999999999) == 2, + "the residual max(0, p - q) is sampled: only token 2 has residual mass here, at both ends of u"); + const std::vector same = {0.0f, 0.3f, 0.7f, 0.0f}; + check(spec_residual_sample(same.data(), same.data(), 4, 0.0) == 1 && spec_residual_sample(same.data(), same.data(), 4, 0.9999999999) == 2, + "an empty residual (p == q) falls back to p and never returns a zero-probability token"); + const std::vector zero = {0, 0, 0}; + check(spec_residual_sample(zero.data(), zero.data(), 3, 0.5) == -1 && spec_residual_sample(nullptr, nullptr, 3, 0.5) == -1, + "a row with no mass, or no row, returns -1 instead of a token"); + } + + // ---- 4. the chain: stop at the first rejection, two draws per position examined ------------------------------ + { + const int V = 3, k = 3; + const std::vector P = {0, 1, 0, 0, 0, 1, 1, 0, 0}; // target argmaxes 1, 2, 0 + const std::vector Q = {0, 1, 0, 0, 0, 1, 0, 1, 0}; // drafter agrees twice, then says 1 + const int drafts[3] = {1, 2, 1}; + int draws = 0; + Rng r{21}; + const SpecResult res = spec_accept_prefix(P.data(), Q.data(), V, drafts, k, [&] { draws++; return r(); }); + check(res.accepted == 2 && res.next_token == 0, "chain: two drafts accepted, the third rejected, and the target's token emitted in its place"); + check(draws == 6, "chain: two uniforms are consumed per position examined"); + const int good[3] = {1, 2, 0}; + const std::vector Q2 = P; + const SpecResult all = spec_accept_prefix(P.data(), Q2.data(), V, good, k, [&] { return r(); }); + check(all.accepted == 3 && all.next_token == -1, "chain: when all k survive, next_token is -1 (the caller samples the pass's last row)"); + const int first_bad[3] = {0, 2, 0}; + const std::vector Q3 = {1, 0, 0, 0, 0, 1, 1, 0, 0}; + draws = 0; + const SpecResult none = spec_accept_prefix(P.data(), Q3.data(), V, first_bad, k, [&] { draws++; return r(); }); + check(none.accepted == 0 && none.next_token == 1 && draws == 2, "chain: a rejection at the first position stops there, later rows are not examined"); + } + { // the chain preserves the distribution at the position where it stops, too: two sampled positions + const int V = 3; + const std::vector P = {0.45f, 0.35f, 0.20f, 0.10f, 0.60f, 0.30f}; + const std::vector Q = {0.10f, 0.80f, 0.10f, 0.30f, 0.30f, 0.40f}; + std::vector c0(3, 0), c1(3, 0); + long n1 = 0; + Rng r{77}; + for (int i = 0; i < N; ++i) { + const int d[2] = {draw({Q[0], Q[1], Q[2]}, r()), draw({Q[3], Q[4], Q[5]}, r())}; + const SpecResult res = spec_accept_prefix(P.data(), Q.data(), V, d, 2, [&] { return r(); }); + const int t0 = res.accepted >= 1 ? d[0] : res.next_token; + c0[size_t(t0)]++; + if (res.accepted >= 1) { n1++; c1[size_t(res.accepted == 2 ? d[1] : res.next_token)]++; } + } + double dev0 = 0, dev1 = 0; + for (int i = 0; i < 3; ++i) { + dev0 = std::fmax(dev0, std::fabs(double(c0[size_t(i)]) / N - double(P[size_t(i)]))); + dev1 = std::fmax(dev1, std::fabs(double(c1[size_t(i)]) / double(n1) - double(P[size_t(3 + i)]))); + } + std::snprintf(msg, sizeof msg, "chain of two sampled positions: each emitted token follows its own row of p (%.4f, %.4f)", dev0, dev1); + check(dev0 < 0.006 && dev1 < 0.008, msg); + } + + std::printf("%s\n", g_fail ? "spec_accept_test: FAILED" : "spec_accept_test: all arms passed"); + return g_fail ? 1 : 0; +}