Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 24 additions & 2 deletions common/onebit-mtp-budget.h
Original file line number Diff line number Diff line change
Expand Up @@ -59,14 +59,14 @@ class MtpBudgetTrace {
// The candidate just sampled: folds it into the running product and reports whether to draft it.
// false means the running product has fallen to or below the threshold, so drafting stops here.
bool keep(int32_t seq, float p1, float threshold) {
const size_t i = static_cast<size_t>(seq);
const size_t i = slot(seq);
budget_cum_[i] *= p1;
return mtp_keep_drafting(budget_cum_[i], threshold);
}

// The candidate was added to the draft: remember the depth and log "D" (depth, p1, running product).
void drafted(int32_t seq, size_t n_drafted, float p1, int32_t token) {
const size_t i = static_cast<size_t>(seq);
const size_t i = slot(seq);
drafted_[i] = n_drafted;
trace_cum_[i] *= p1;
if (FILE * f = file()) {
Expand All @@ -79,6 +79,9 @@ class MtpBudgetTrace {

// Drafting ended for this step: logs "E" with the drafted count of every sequence.
void end(uint32_t n_seq) {
if (n_seq > 0) {
slot(static_cast<int32_t>(n_seq - 1));
}
if (FILE * f = file()) {
for (uint32_t seq = 0; seq < n_seq; ++seq) {
std::fprintf(f, "E t=%.3f step=%llu seq=%u drafted=%zu\n", now_ms(), (unsigned long long) step_, seq,
Expand All @@ -97,7 +100,26 @@ class MtpBudgetTrace {
}
}

// Per-sequence state, for tests: the running product the rule has folded for seq so far.
float budget_cum(int32_t seq) const {
const size_t i = static_cast<size_t>(seq);
return i < budget_cum_.size() ? budget_cum_[i] : 1.0f;
}

private:
// Index of seq's state, growing it (fresh: product 1, nothing drafted) when a caller reaches a sequence
// begin() did not size. A missing begin() once made every MTP draft write past an empty vector and
// SIGSEGV llama-server on its first request (engine #2433); with this it costs only the rule's reset.
size_t slot(int32_t seq) {
const size_t i = static_cast<size_t>(seq);
if (i >= budget_cum_.size()) {
budget_cum_.resize(i + 1, 1.0f);
trace_cum_.resize(i + 1, 1.0f);
drafted_.resize(i + 1, 0);
}
return i;
}

static FILE * file() {
static FILE * f = []() -> FILE * {
const char * p = std::getenv("ONEBIT_MTP_TRACE");
Expand Down
22 changes: 19 additions & 3 deletions common/speculative.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1529,6 +1529,9 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
// request's last row (engine #290).
std::vector<llama_pos> pending_pos;

// all-zero h-row for the first token of a batch that does not continue from pending_h
std::vector<float> zero_h;

std::vector<int32_t> i_batch_beg;
std::vector<int32_t> i_batch_end;

Expand Down Expand Up @@ -1611,6 +1614,7 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {

pending_h.assign(n_seq, std::vector<float>(n_embd, 0.0f));
pending_pos.assign(n_seq, -1);
zero_h.assign(n_embd, 0.0f);

i_last.assign(n_seq, -1);
i_batch_beg.assign(n_seq, -1);
Expand Down Expand Up @@ -1709,9 +1713,17 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
? batch.add(t.id, t.pos[0], seq_id, false)
: batch.add_embd(t.embd, t.pos.data(), seq_id, false);

const float * h_row = k == i_batch_beg[seq_id]
? pending_h[seq_id].data()
: h_tgt + (size_t) (k - 1) * n_embd;
// pending_h is the h-row that precedes the first token of this batch, so it only
// pairs with that token when the batch really continues from it. A fresh sequence
// starts at pos 0 while pending_h still holds the previous request's last row;
// seeding the MTP head with it there made the draft's first prediction depend on
// the previous request (engine #290). Same continuity guard eagle3 applies with
// pending_pos_last. (The #327 merge dropped it; engine #2433.)
const float * h_row = h_tgt + (size_t) (k - 1) * n_embd;
if (k == i_batch_beg[seq_id]) {
const bool continues = pending_pos[seq_id] >= 0 && pending_pos[seq_id] + 1 == t.pos[0];
h_row = continues ? pending_h[seq_id].data() : zero_h.data();
}

batch.set_embd_state(idx, { h_row, 1, (size_t) n_embd });
}
Expand Down Expand Up @@ -1774,6 +1786,10 @@ struct common_speculative_impl_draft_mtp : public common_speculative_impl {
void draft(common_speculative_draft_params_vec & dparams) override {
auto & ctx_dft = params.ctx_dft;

// sizes the budget/trace state to n_seq; keep()/drafted()/end() index it per sequence
// (the #327 merge dropped this call and every MTP draft wrote past an empty vector, engine #2433)
budget.begin(n_seq);

batch.clear();

// keep track of which sequences are still drafting
Expand Down
1 change: 1 addition & 0 deletions tests/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -186,6 +186,7 @@ if (NOT WIN32 OR NOT BUILD_SHARED_LIBS)
llama_build_and_test(test-unicode.cpp LABEL behavior LINK llama)
llama_build_and_test(test-sampling.cpp LABEL behavior LINK llama)
llama_build_and_test(test-reasoning-budget.cpp LABEL behavior)
llama_build_and_test(test-onebit-mtp-budget.cpp LABEL behavior) # engine #2433
llama_build_and_test(test-grammar-parser.cpp LABEL behavior LINK llama)
llama_build_and_test(test-grammar-integration.cpp LABEL behavior)
llama_build_and_test(test-llama-grammar.cpp LABEL behavior LINK llama)
Expand Down
69 changes: 69 additions & 0 deletions tests/test-onebit-mtp-budget.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
// Copyright 2026 bong-water-water-bong
// SPDX-License-Identifier: Apache-2.0
//
// MtpBudgetTrace's per-sequence state (common/onebit-mtp-budget.h), engine #2433: the MTP drafter calls
// keep()/drafted()/end() for every drafting sequence. A merge dropped its begin() call and every draft
// wrote past an empty vector, so the first MTP request killed llama-server. This checks that state is
// sized for any sequence the drafter reaches, with or without begin(), and that begin() still resets it.

#include "onebit-mtp-budget.h"

#include <cmath>
#include <cstdio>

static int n_fail = 0;

#define CHECK(cond) \
do { \
if (!(cond)) { \
std::fprintf(stderr, "FAIL %s:%d: %s\n", __FILE__, __LINE__, #cond); \
++n_fail; \
} \
} while (0)

static bool near(float a, float b) { return std::fabs(a - b) < 1e-6f; }

int main() {
// no begin(): the shape of the #2433 drafter. Sequence 3 of 4, three draft depths.
{
onebit::MtpBudgetTrace b;
CHECK(b.keep(3, 0.9f, 0.0f));
b.drafted(3, 1, 0.9f, 11);
CHECK(b.keep(3, 0.8f, 0.0f));
b.drafted(3, 2, 0.8f, 12);
CHECK(near(b.budget_cum(3), 0.72f));
CHECK(near(b.budget_cum(0), 1.0f)); // sized but untouched
b.end(4);
b.accepted(3, 2);
}

// begin() sizes and resets: the product starts again at 1 for every sequence.
{
onebit::MtpBudgetTrace b;
b.begin(2);
CHECK(b.keep(1, 0.5f, 0.0f));
CHECK(near(b.budget_cum(1), 0.5f));
b.begin(2);
CHECK(near(b.budget_cum(1), 1.0f));
// a sequence past what begin() sized still works (n_seq grew between steps)
CHECK(b.keep(5, 0.5f, 0.0f));
CHECK(near(b.budget_cum(5), 0.5f));
b.end(6);
}

// the rule itself: keep while the running product stays above the threshold, off at <= 0.
{
onebit::MtpBudgetTrace b;
b.begin(1);
CHECK(b.keep(0, 0.9f, 0.6f)); // 0.9
CHECK(b.keep(0, 0.8f, 0.6f)); // 0.72
CHECK(!b.keep(0, 0.8f, 0.6f)); // 0.576 <= 0.6: stop
CHECK(onebit::mtp_keep_drafting(0.0f, 0.0f));
CHECK(onebit::mtp_keep_drafting(0.0f, -1.0f));
}

if (n_fail == 0) {
std::printf("test-onebit-mtp-budget: ok\n");
}
return n_fail == 0 ? 0 : 1;
}