From c28f6aeb60d5e73bcdc9de4773aa2fc4bfe7526b Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sat, 14 Nov 2026 09:48:57 -0400 Subject: [PATCH] speculative: restore the two MTP-drafter pieces the #327 merge dropped (engine #2433) The AMD-pin merge (#327) resolved common/speculative.cpp's draft-mtp conflicts onto upstream's restructured batch API and lost two onebit lines from common_speculative_impl_draft_mtp: - draft() no longer called budget.begin(n_seq). MtpBudgetTrace's per-sequence vectors stayed empty, keep()/drafted() wrote past them on the first draft, and llama-server died with SIGSEGV in draft() on the first request for any GGUF with its own MTP head (MTP is on by default). Restored, and MtpBudgetTrace now grows its state for any sequence it is handed, so a missing begin() can only reset the rule, not corrupt the heap. - process() seeded the first token of every batch with pending_h, dropping the engine #290 continuity guard: a fresh request at pos 0 drafted from the previous request's last h-row. Restored: pending_h only when pending_pos + 1 == the batch's first pos, else a zero row. tests/test-onebit-mtp-budget: drafts on sequence 3 of 4 without begin(), begin() resetting the product, and the rule. With the old indexing the first case aborts (operator[] bounds assertion). Co-Authored-By: Claude Opus 5.5 --- common/onebit-mtp-budget.h | 26 +++++++++++- common/speculative.cpp | 22 ++++++++-- tests/CMakeLists.txt | 1 + tests/test-onebit-mtp-budget.cpp | 69 ++++++++++++++++++++++++++++++++ 4 files changed, 113 insertions(+), 5 deletions(-) create mode 100644 tests/test-onebit-mtp-budget.cpp diff --git a/common/onebit-mtp-budget.h b/common/onebit-mtp-budget.h index 018cad926a48..d8bd4184d69a 100644 --- a/common/onebit-mtp-budget.h +++ b/common/onebit-mtp-budget.h @@ -59,14 +59,14 @@ class MtpBudgetTrace { // The candidate just sampled: folds it into the running product and reports whether to draft it. // false means the running product has fallen to or below the threshold, so drafting stops here. bool keep(int32_t seq, float p1, float threshold) { - const size_t i = static_cast(seq); + const size_t i = slot(seq); budget_cum_[i] *= p1; return mtp_keep_drafting(budget_cum_[i], threshold); } // The candidate was added to the draft: remember the depth and log "D" (depth, p1, running product). void drafted(int32_t seq, size_t n_drafted, float p1, int32_t token) { - const size_t i = static_cast(seq); + const size_t i = slot(seq); drafted_[i] = n_drafted; trace_cum_[i] *= p1; if (FILE * f = file()) { @@ -79,6 +79,9 @@ class MtpBudgetTrace { // Drafting ended for this step: logs "E" with the drafted count of every sequence. void end(uint32_t n_seq) { + if (n_seq > 0) { + slot(static_cast(n_seq - 1)); + } if (FILE * f = file()) { for (uint32_t seq = 0; seq < n_seq; ++seq) { std::fprintf(f, "E t=%.3f step=%llu seq=%u drafted=%zu\n", now_ms(), (unsigned long long) step_, seq, @@ -97,7 +100,26 @@ class MtpBudgetTrace { } } + // Per-sequence state, for tests: the running product the rule has folded for seq so far. + float budget_cum(int32_t seq) const { + const size_t i = static_cast(seq); + return i < budget_cum_.size() ? budget_cum_[i] : 1.0f; + } + private: + // Index of seq's state, growing it (fresh: product 1, nothing drafted) when a caller reaches a sequence + // begin() did not size. A missing begin() once made every MTP draft write past an empty vector and + // SIGSEGV llama-server on its first request (engine #2433); with this it costs only the rule's reset. + size_t slot(int32_t seq) { + const size_t i = static_cast(seq); + if (i >= budget_cum_.size()) { + budget_cum_.resize(i + 1, 1.0f); + trace_cum_.resize(i + 1, 1.0f); + drafted_.resize(i + 1, 0); + } + return i; + } + static FILE * file() { static FILE * f = []() -> FILE * { const char * p = std::getenv("ONEBIT_MTP_TRACE"); diff --git a/common/speculative.cpp b/common/speculative.cpp index 6fe4923636f8..db3c3c0fa8ce 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -1529,6 +1529,9 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { // request's last row (engine #290). std::vector pending_pos; + // all-zero h-row for the first token of a batch that does not continue from pending_h + std::vector zero_h; + std::vector i_batch_beg; std::vector i_batch_end; @@ -1611,6 +1614,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { pending_h.assign(n_seq, std::vector(n_embd, 0.0f)); pending_pos.assign(n_seq, -1); + zero_h.assign(n_embd, 0.0f); i_last.assign(n_seq, -1); i_batch_beg.assign(n_seq, -1); @@ -1709,9 +1713,17 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { ? batch.add(t.id, t.pos[0], seq_id, false) : batch.add_embd(t.embd, t.pos.data(), seq_id, false); - const float * h_row = k == i_batch_beg[seq_id] - ? pending_h[seq_id].data() - : h_tgt + (size_t) (k - 1) * n_embd; + // pending_h is the h-row that precedes the first token of this batch, so it only + // pairs with that token when the batch really continues from it. A fresh sequence + // starts at pos 0 while pending_h still holds the previous request's last row; + // seeding the MTP head with it there made the draft's first prediction depend on + // the previous request (engine #290). Same continuity guard eagle3 applies with + // pending_pos_last. (The #327 merge dropped it; engine #2433.) + const float * h_row = h_tgt + (size_t) (k - 1) * n_embd; + if (k == i_batch_beg[seq_id]) { + const bool continues = pending_pos[seq_id] >= 0 && pending_pos[seq_id] + 1 == t.pos[0]; + h_row = continues ? pending_h[seq_id].data() : zero_h.data(); + } batch.set_embd_state(idx, { h_row, 1, (size_t) n_embd }); } @@ -1774,6 +1786,10 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl { void draft(common_speculative_draft_params_vec & dparams) override { auto & ctx_dft = params.ctx_dft; + // sizes the budget/trace state to n_seq; keep()/drafted()/end() index it per sequence + // (the #327 merge dropped this call and every MTP draft wrote past an empty vector, engine #2433) + budget.begin(n_seq); + batch.clear(); // keep track of which sequences are still drafting diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index d4295fad679e..35cada17b449 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -186,6 +186,7 @@ if (NOT WIN32 OR NOT BUILD_SHARED_LIBS) llama_build_and_test(test-unicode.cpp LABEL behavior LINK llama) llama_build_and_test(test-sampling.cpp LABEL behavior LINK llama) llama_build_and_test(test-reasoning-budget.cpp LABEL behavior) + llama_build_and_test(test-onebit-mtp-budget.cpp LABEL behavior) # engine #2433 llama_build_and_test(test-grammar-parser.cpp LABEL behavior LINK llama) llama_build_and_test(test-grammar-integration.cpp LABEL behavior) llama_build_and_test(test-llama-grammar.cpp LABEL behavior LINK llama) diff --git a/tests/test-onebit-mtp-budget.cpp b/tests/test-onebit-mtp-budget.cpp new file mode 100644 index 000000000000..159d738f197e --- /dev/null +++ b/tests/test-onebit-mtp-budget.cpp @@ -0,0 +1,69 @@ +// Copyright 2026 bong-water-water-bong +// SPDX-License-Identifier: Apache-2.0 +// +// MtpBudgetTrace's per-sequence state (common/onebit-mtp-budget.h), engine #2433: the MTP drafter calls +// keep()/drafted()/end() for every drafting sequence. A merge dropped its begin() call and every draft +// wrote past an empty vector, so the first MTP request killed llama-server. This checks that state is +// sized for any sequence the drafter reaches, with or without begin(), and that begin() still resets it. + +#include "onebit-mtp-budget.h" + +#include +#include + +static int n_fail = 0; + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \ + ++n_fail; \ + } \ + } while (0) + +static bool near(float a, float b) { return std::fabs(a - b) < 1e-6f; } + +int main() { + // no begin(): the shape of the #2433 drafter. Sequence 3 of 4, three draft depths. + { + onebit::MtpBudgetTrace b; + CHECK(b.keep(3, 0.9f, 0.0f)); + b.drafted(3, 1, 0.9f, 11); + CHECK(b.keep(3, 0.8f, 0.0f)); + b.drafted(3, 2, 0.8f, 12); + CHECK(near(b.budget_cum(3), 0.72f)); + CHECK(near(b.budget_cum(0), 1.0f)); // sized but untouched + b.end(4); + b.accepted(3, 2); + } + + // begin() sizes and resets: the product starts again at 1 for every sequence. + { + onebit::MtpBudgetTrace b; + b.begin(2); + CHECK(b.keep(1, 0.5f, 0.0f)); + CHECK(near(b.budget_cum(1), 0.5f)); + b.begin(2); + CHECK(near(b.budget_cum(1), 1.0f)); + // a sequence past what begin() sized still works (n_seq grew between steps) + CHECK(b.keep(5, 0.5f, 0.0f)); + CHECK(near(b.budget_cum(5), 0.5f)); + b.end(6); + } + + // the rule itself: keep while the running product stays above the threshold, off at <= 0. + { + onebit::MtpBudgetTrace b; + b.begin(1); + CHECK(b.keep(0, 0.9f, 0.6f)); // 0.9 + CHECK(b.keep(0, 0.8f, 0.6f)); // 0.72 + CHECK(!b.keep(0, 0.8f, 0.6f)); // 0.576 <= 0.6: stop + CHECK(onebit::mtp_keep_drafting(0.0f, 0.0f)); + CHECK(onebit::mtp_keep_drafting(0.0f, -1.0f)); + } + + if (n_fail == 0) { + std::printf("test-onebit-mtp-budget: ok\n"); + } + return n_fail == 0 ? 0 : 1; +}