diff --git a/common/onebit-mtp-budget.h b/common/onebit-mtp-budget.h index 018cad926a48..d8bd4184d69a 100644 --- a/common/onebit-mtp-budget.h +++ b/common/onebit-mtp-budget.h @@ -59,14 +59,14 @@ class MtpBudgetTrace { // The candidate just sampled: folds it into the running product and reports whether to draft it. // false means the running product has fallen to or below the threshold, so drafting stops here. bool keep(int32_t seq, float p1, float threshold) { - const size_t i = static_cast(seq); + const size_t i = slot(seq); budget_cum_[i] *= p1; return mtp_keep_drafting(budget_cum_[i], threshold); } // The candidate was added to the draft: remember the depth and log "D" (depth, p1, running product). void drafted(int32_t seq, size_t n_drafted, float p1, int32_t token) { - const size_t i = static_cast(seq); + const size_t i = slot(seq); drafted_[i] = n_drafted; trace_cum_[i] *= p1; if (FILE * f = file()) { @@ -79,6 +79,9 @@ class MtpBudgetTrace { // Drafting ended for this step: logs "E" with the drafted count of every sequence. void end(uint32_t n_seq) { + if (n_seq > 0) { + slot(static_cast(n_seq - 1)); + } if (FILE * f = file()) { for (uint32_t seq = 0; seq < n_seq; ++seq) { std::fprintf(f, "E t=%.3f step=%llu seq=%u drafted=%zu\n", now_ms(), (unsigned long long) step_, seq, @@ -97,7 +100,26 @@ class MtpBudgetTrace { } } + // Per-sequence state, for tests: the running product the rule has folded for seq so far. + float budget_cum(int32_t seq) const { + const size_t i = static_cast(seq); + return i < budget_cum_.size() ? budget_cum_[i] : 1.0f; + } + private: + // Index of seq's state, growing it (fresh: product 1, nothing drafted) when a caller reaches a sequence + // begin() did not size. A missing begin() once made every MTP draft write past an empty vector and + // SIGSEGV llama-server on its first request (engine #2433); with this it costs only the rule's reset. + size_t slot(int32_t seq) { + const size_t i = static_cast(seq); + if (i >= budget_cum_.size()) { + budget_cum_.resize(i + 1, 1.0f); + trace_cum_.resize(i + 1, 1.0f); + drafted_.resize(i + 1, 0); + } + return i; + } + static FILE * file() { static FILE * f = []() -> FILE * { const char * p = std::getenv("ONEBIT_MTP_TRACE"); diff --git a/common/speculative.cpp b/common/speculative.cpp index 6fe4923636f8..db3c3c0fa8ce 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -1529,6 +1529,9 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { // request's last row (engine #290). std::vector pending_pos; + // all-zero h-row for the first token of a batch that does not continue from pending_h + std::vector zero_h; + std::vector i_batch_beg; std::vector i_batch_end; @@ -1611,6 +1614,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { pending_h.assign(n_seq, std::vector(n_embd, 0.0f)); pending_pos.assign(n_seq, -1); + zero_h.assign(n_embd, 0.0f); i_last.assign(n_seq, -1); i_batch_beg.assign(n_seq, -1); @@ -1709,9 +1713,17 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { ? batch.add(t.id, t.pos[0], seq_id, false) : batch.add_embd(t.embd, t.pos.data(), seq_id, false); - const float * h_row = k == i_batch_beg[seq_id] - ? pending_h[seq_id].data() - : h_tgt + (size_t) (k - 1) * n_embd; + // pending_h is the h-row that precedes the first token of this batch, so it only + // pairs with that token when the batch really continues from it. A fresh sequence + // starts at pos 0 while pending_h still holds the previous request's last row; + // seeding the MTP head with it there made the draft's first prediction depend on + // the previous request (engine #290). Same continuity guard eagle3 applies with + // pending_pos_last. (The #327 merge dropped it; engine #2433.) + const float * h_row = h_tgt + (size_t) (k - 1) * n_embd; + if (k == i_batch_beg[seq_id]) { + const bool continues = pending_pos[seq_id] >= 0 && pending_pos[seq_id] + 1 == t.pos[0]; + h_row = continues ? pending_h[seq_id].data() : zero_h.data(); + } batch.set_embd_state(idx, { h_row, 1, (size_t) n_embd }); } @@ -1774,6 +1786,10 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { void draft(common_speculative_draft_params_vec & dparams) override { auto & ctx_dft = params.ctx_dft; + // sizes the budget/trace state to n_seq; keep()/drafted()/end() index it per sequence + // (the #327 merge dropped this call and every MTP draft wrote past an empty vector, engine #2433) + budget.begin(n_seq); + batch.clear(); // keep track of which sequences are still drafting diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d4295fad679e..35cada17b449 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -186,6 +186,7 @@ if (NOT WIN32 OR NOT BUILD_SHARED_LIBS) llama_build_and_test(test-unicode.cpp LABEL behavior LINK llama) llama_build_and_test(test-sampling.cpp LABEL behavior LINK llama) llama_build_and_test(test-reasoning-budget.cpp LABEL behavior) + llama_build_and_test(test-onebit-mtp-budget.cpp LABEL behavior) # engine #2433 llama_build_and_test(test-grammar-parser.cpp LABEL behavior LINK llama) llama_build_and_test(test-grammar-integration.cpp LABEL behavior) llama_build_and_test(test-llama-grammar.cpp LABEL behavior LINK llama) diff --git a/tests/test-onebit-mtp-budget.cpp b/tests/test-onebit-mtp-budget.cpp new file mode 100644 index 000000000000..159d738f197e --- /dev/null +++ b/tests/test-onebit-mtp-budget.cpp @@ -0,0 +1,69 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// MtpBudgetTrace's per-sequence state (common/onebit-mtp-budget.h), engine #2433: the MTP drafter calls +// keep()/drafted()/end() for every drafting sequence. A merge dropped its begin() call and every draft +// wrote past an empty vector, so the first MTP request killed llama-server. This checks that state is +// sized for any sequence the drafter reaches, with or without begin(), and that begin() still resets it. + +#include "onebit-mtp-budget.h" + +#include +#include + +static int n_fail = 0; + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + ++n_fail; \ + } \ + } while (0) + +static bool near(float a, float b) { return std::fabs(a - b) < 1e-6f; } + +int main() { + // no begin(): the shape of the #2433 drafter. Sequence 3 of 4, three draft depths. + { + onebit::MtpBudgetTrace b; + CHECK(b.keep(3, 0.9f, 0.0f)); + b.drafted(3, 1, 0.9f, 11); + CHECK(b.keep(3, 0.8f, 0.0f)); + b.drafted(3, 2, 0.8f, 12); + CHECK(near(b.budget_cum(3), 0.72f)); + CHECK(near(b.budget_cum(0), 1.0f)); // sized but untouched + b.end(4); + b.accepted(3, 2); + } + + // begin() sizes and resets: the product starts again at 1 for every sequence. + { + onebit::MtpBudgetTrace b; + b.begin(2); + CHECK(b.keep(1, 0.5f, 0.0f)); + CHECK(near(b.budget_cum(1), 0.5f)); + b.begin(2); + CHECK(near(b.budget_cum(1), 1.0f)); + // a sequence past what begin() sized still works (n_seq grew between steps) + CHECK(b.keep(5, 0.5f, 0.0f)); + CHECK(near(b.budget_cum(5), 0.5f)); + b.end(6); + } + + // the rule itself: keep while the running product stays above the threshold, off at <= 0. + { + onebit::MtpBudgetTrace b; + b.begin(1); + CHECK(b.keep(0, 0.9f, 0.6f)); // 0.9 + CHECK(b.keep(0, 0.8f, 0.6f)); // 0.72 + CHECK(!b.keep(0, 0.8f, 0.6f)); // 0.576 <= 0.6: stop + CHECK(onebit::mtp_keep_drafting(0.0f, 0.0f)); + CHECK(onebit::mtp_keep_drafting(0.0f, -1.0f)); + } + + if (n_fail == 0) { + std::printf("test-onebit-mtp-budget: ok\n"); + } + return n_fail == 0 ? 0 : 1; +}