From cb7f767f40a7b90662a560138865e768b9dcd2c6 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sat, 14 Nov 2026 16:55:51 -0400 Subject: [PATCH] common/sampling: fast-greedy predicate and argmax as functions, an offline test, and the min_p clamp (follow-up to #385) Review of #385 asked for the equivalence claim to be pinned in the repo and for MIN_P to be admitted only where it provably keeps the top token. - common_sampler_fast_greedy_eligible(params, has_grammar, has_reasoning_budget) and common_sampler_fast_greedy_argmax(logits, n_vocab, mask) carry the logic common_sampler_init / common_sampler_sample used inline; behaviour unchanged except MIN_P now requires min_p <= 1 (it keeps p >= min_p * p_max). - tests/test-sampling-fast-greedy (no model): every stage that can move the argmax turns the fast path off (temp, dynatemp, mirostat, n_probs, backend sampling, XTC, typical, min_p > 1, DRY, penalties, a finite bias, temperature not last, grammar, reasoning budget); a -inf bias does not. On 200 random 4096-entry logit vectors the fast argmax equals the llama.h chain's pick (logit bias + top-k + top-p + min-p + temp 0 + greedy), with and without a -inf bias on the argmax (both move to the runner-up); NaN and all-masked return -1 so the full path decides. Co-Authored-By: Claude Opus 5.5 --- common/sampling.cpp | 110 +++++++++++++----------- common/sampling.h | 6 ++ tests/CMakeLists.txt | 1 + tests/test-sampling-fast-greedy.cpp | 129 ++++++++++++++++++++++++++++ 4 files changed, 197 insertions(+), 49 deletions(-) create mode 100644 tests/test-sampling-fast-greedy.cpp diff --git a/common/sampling.cpp b/common/sampling.cpp index f9fe52e4e80f..41ca9e8534c0 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -195,6 +195,61 @@ std::string common_params_sampling::print() const { return std::string(result); } +bool common_sampler_fast_greedy_eligible(const common_params_sampling & params, bool has_grammar, bool has_reasoning_budget) { + bool ok = params.mirostat == 0 && params.n_probs == 0 && !has_grammar && !has_reasoning_budget && !params.backend_sampling && + !params.samplers.empty() && params.samplers.back() == COMMON_SAMPLER_TYPE_TEMPERATURE && + params.temp == 0.0f && params.dynatemp_range == 0.0f; + for (const auto type : params.samplers) { + switch (type) { + case COMMON_SAMPLER_TYPE_TOP_K: + case COMMON_SAMPLER_TYPE_TOP_P: + case COMMON_SAMPLER_TYPE_TOP_N_SIGMA: + case COMMON_SAMPLER_TYPE_TEMPERATURE: + break; // never remove the top token + case COMMON_SAMPLER_TYPE_MIN_P: + ok &= params.min_p <= 1.0f; // keeps p >= min_p * p_max, which includes the top token only for min_p <= 1 + break; + case COMMON_SAMPLER_TYPE_PENALTIES: + ok &= params.penalty_last_n == 0 || + (params.penalty_repeat == 1.0f && params.penalty_freq == 0.0f && params.penalty_present == 0.0f); + break; + case COMMON_SAMPLER_TYPE_DRY: + ok &= params.dry_multiplier == 0.0f; + break; + case COMMON_SAMPLER_TYPE_XTC: + ok &= params.xtc_probability <= 0.0f; + break; + case COMMON_SAMPLER_TYPE_TYPICAL_P: + ok &= params.typ_p >= 1.0f; + break; + default: + ok = false; // infill, adaptive-p, anything new + } + } + for (const auto & b : params.logit_bias) { + if (b.bias != -INFINITY && b.bias != 0.0f) { + ok = false; // a finite bias can move the argmax + } + } + return ok; +} + +int32_t common_sampler_fast_greedy_argmax(const float * logits, int32_t n_vocab, const char * mask) { + int32_t best = -1; + float best_v = -INFINITY; + for (int32_t i = 0; i < n_vocab; ++i) { + const float v = logits[i]; + if (std::isnan(v)) { + return -1; // the full path reports it (engine#123/#315) + } + if (v > best_v && (mask == nullptr || !mask[i])) { + best_v = v; + best = i; + } + } + return best; +} + struct common_sampler * common_sampler_init( const struct llama_model * model, struct common_params_sampling & params) { @@ -451,42 +506,13 @@ struct common_sampler * common_sampler_init( }; { - bool ok = params.mirostat == 0 && params.n_probs == 0 && !grmr && !rbudget && !params.backend_sampling && - !params.samplers.empty() && params.samplers.back() == COMMON_SAMPLER_TYPE_TEMPERATURE && - params.temp == 0.0f && params.dynatemp_range == 0.0f; - for (const auto type : params.samplers) { - switch (type) { - case COMMON_SAMPLER_TYPE_TOP_K: - case COMMON_SAMPLER_TYPE_TOP_P: - case COMMON_SAMPLER_TYPE_MIN_P: - case COMMON_SAMPLER_TYPE_TOP_N_SIGMA: - case COMMON_SAMPLER_TYPE_TEMPERATURE: - break; // never remove the top token - case COMMON_SAMPLER_TYPE_PENALTIES: - ok &= params.penalty_last_n == 0 || - (params.penalty_repeat == 1.0f && params.penalty_freq == 0.0f && params.penalty_present == 0.0f); - break; - case COMMON_SAMPLER_TYPE_DRY: - ok &= params.dry_multiplier == 0.0f; - break; - case COMMON_SAMPLER_TYPE_XTC: - ok &= params.xtc_probability <= 0.0f; - break; - case COMMON_SAMPLER_TYPE_TYPICAL_P: - ok &= params.typ_p >= 1.0f; - break; - default: - ok = false; // infill, adaptive-p, anything new - } - } + bool ok = common_sampler_fast_greedy_eligible(params, grmr != nullptr, rbudget != nullptr); if (ok) { const int32_t n_vocab = llama_vocab_n_tokens(vocab); result->fast_greedy_mask.assign(n_vocab, 0); for (const auto & b : params.logit_bias) { - if (b.bias == -INFINITY) { - if (b.token >= 0 && b.token < n_vocab) result->fast_greedy_mask[b.token] = 1; - } else if (b.bias != 0.0f) { - ok = false; // a finite bias can move the argmax + if (b.bias == -INFINITY && b.token >= 0 && b.token < n_vocab) { + result->fast_greedy_mask[b.token] = 1; } } int32_t n_suppress = 0; @@ -670,24 +696,10 @@ llama_token common_sampler_sample(struct common_sampler * gsmpl, struct llama_co const float * logits = llama_get_logits_ith(ctx, idx); const int32_t n_vocab = (int32_t) gsmpl->fast_greedy_mask.size(); if (logits != nullptr && n_vocab > 0) { - const char * mask = gsmpl->fast_greedy_mask.data(); - int32_t best = -1; - float best_v = -INFINITY; - bool nan = false; - for (int32_t i = 0; i < n_vocab; ++i) { - const float v = logits[i]; - if (std::isnan(v)) { - nan = true; // the full path below reports it (engine#123/#315) - break; - } - if (v > best_v && !mask[i]) { - best_v = v; - best = i; - } - } - if (!nan && best >= 0) { + const int32_t best = common_sampler_fast_greedy_argmax(logits, n_vocab, gsmpl->fast_greedy_mask.data()); + if (best >= 0) { gsmpl->cur.resize(1); - gsmpl->cur[0] = { best, best_v, 1.0f }; + gsmpl->cur[0] = { best, logits[best], 1.0f }; gsmpl->cur_p = { gsmpl->cur.data(), 1, 0, true }; return best; } diff --git a/common/sampling.h b/common/sampling.h index f19bd27c40f8..0a28e93a27c2 100644 --- a/common/sampling.h +++ b/common/sampling.h @@ -67,6 +67,12 @@ struct llama_sampler * common_sampler_get(const struct common_sampler * gsmpl); // llama_token common_sampler_sample(struct common_sampler * gsmpl, struct llama_context * ctx, int idx, bool grammar_first = false); +// greedy fast path (common_sampler_sample): true when every stage of the chain built from params provably keeps the top +// token, so sampling is an argmax over the logits minus the -inf-biased / suppressed ids +bool common_sampler_fast_greedy_eligible(const common_params_sampling & params, bool has_grammar, bool has_reasoning_budget); +// argmax over logits skipping mask[i] != 0 (mask may be null); -1 when a logit is NaN or every id is masked / -inf +int32_t common_sampler_fast_greedy_argmax(const float * logits, int32_t n_vocab, const char * mask); + // generalized version of common_sampler_sample // // will cross-reference the sampled tokens with a batch of draft tokens and accept those that match diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index 8d7ac0dabfff..2b746e25f5ea 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -199,6 +199,7 @@ if (NOT WIN32 OR NOT BUILD_SHARED_LIBS) target_include_directories(test-kv-zero PRIVATE ${PROJECT_SOURCE_DIR}/src) llama_build_and_test(test-unicode.cpp LABEL behavior LINK llama) llama_build_and_test(test-sampling.cpp LABEL behavior LINK llama) + llama_build_and_test(test-sampling-fast-greedy.cpp LABEL behavior) llama_build_and_test(test-reasoning-budget.cpp LABEL behavior) llama_build_and_test(test-onebit-mtp-budget.cpp LABEL behavior) # engine #2433 llama_build_and_test(test-grammar-parser.cpp LABEL behavior LINK llama) diff --git a/tests/test-sampling-fast-greedy.cpp b/tests/test-sampling-fast-greedy.cpp new file mode 100644 index 000000000000..e89167a49c2c --- /dev/null +++ b/tests/test-sampling-fast-greedy.cpp @@ -0,0 +1,129 @@ +// common_sampler greedy fast path (llama.cpp#385): the eligibility predicate turns the fast path off for every stage that +// can move the argmax, and on the same logits the fast-path argmax picks the token the equivalent sampler chain picks. +// Offline: no model, the chain is built from llama.h samplers. + +#include "sampling.h" +#include "llama.h" + +#include +#include +#include +#include +#include + +static int failures = 0; + +#define CHECK(cond) \ + do { \ + if (!(cond)) { \ + std::fprintf(stderr, "%s:%d: CHECK failed: %s\n", __FILE__, __LINE__, #cond); \ + ++failures; \ + } \ + } while (0) + +static common_params_sampling greedy_params() { + common_params_sampling p; + p.temp = 0.0f; // the default sampler order ends in TEMPERATURE + return p; +} + +static void test_eligibility() { + { + const common_params_sampling p = greedy_params(); + CHECK(common_sampler_fast_greedy_eligible(p, false, false)); + CHECK(!common_sampler_fast_greedy_eligible(p, true, false)); // grammar + CHECK(!common_sampler_fast_greedy_eligible(p, false, true)); // reasoning budget + } + auto off = [](void (*edit)(common_params_sampling &)) { + common_params_sampling p = greedy_params(); + edit(p); + return !common_sampler_fast_greedy_eligible(p, false, false); + }; + CHECK(off([](common_params_sampling & p) { p.temp = 0.7f; })); + CHECK(off([](common_params_sampling & p) { p.dynatemp_range = 0.5f; })); + CHECK(off([](common_params_sampling & p) { p.mirostat = 2; })); + CHECK(off([](common_params_sampling & p) { p.n_probs = 1; })); + CHECK(off([](common_params_sampling & p) { p.backend_sampling = true; })); + CHECK(off([](common_params_sampling & p) { p.xtc_probability = 0.5f; })); + CHECK(off([](common_params_sampling & p) { p.typ_p = 0.9f; })); + CHECK(off([](common_params_sampling & p) { p.min_p = 1.5f; })); + CHECK(off([](common_params_sampling & p) { p.dry_multiplier = 0.8f; })); + CHECK(off([](common_params_sampling & p) { p.penalty_repeat = 1.1f; })); + CHECK(off([](common_params_sampling & p) { p.logit_bias.push_back({ 7, 2.5f }); })); // finite bias + CHECK(off([](common_params_sampling & p) { p.samplers.push_back(COMMON_SAMPLER_TYPE_TOP_K); })); // temp not last + { + common_params_sampling p = greedy_params(); + p.logit_bias.push_back({ 7, -INFINITY }); // a suppression is a mask, not a reason to fall back + CHECK(common_sampler_fast_greedy_eligible(p, false, false)); + p.penalty_last_n = 0; + p.penalty_repeat = 1.3f; // penalties over zero tokens do nothing + CHECK(common_sampler_fast_greedy_eligible(p, false, false)); + } +} + +// the chain common_sampler_init builds for greedy_params(), minus the stages that are identity at their defaults +static llama_token chain_pick(const std::vector & logits, const std::vector & bias) { + const int32_t n = (int32_t) logits.size(); + llama_sampler * chain = llama_sampler_chain_init(llama_sampler_chain_default_params()); + if (!bias.empty()) { + llama_sampler_chain_add(chain, llama_sampler_init_logit_bias(n, (int32_t) bias.size(), bias.data())); + } + llama_sampler_chain_add(chain, llama_sampler_init_top_k(40)); + llama_sampler_chain_add(chain, llama_sampler_init_top_p(0.95f, 1)); + llama_sampler_chain_add(chain, llama_sampler_init_min_p(0.05f, 1)); + llama_sampler_chain_add(chain, llama_sampler_init_temp_ext(0.0f, 0.0f, 1.0f)); + llama_sampler_chain_add(chain, llama_sampler_init_greedy()); + + std::vector cur(n); + for (int32_t i = 0; i < n; ++i) { + cur[i] = { i, logits[i], 0.0f }; + } + llama_token_data_array cur_p = { cur.data(), cur.size(), -1, false }; + llama_sampler_apply(chain, &cur_p); + const llama_token id = cur_p.data[cur_p.selected].id; + llama_sampler_free(chain); + return id; +} + +static void test_equivalence() { + std::mt19937 rng(1234); + std::uniform_real_distribution dist(-8.0f, 8.0f); + const int32_t n = 4096; + for (int trial = 0; trial < 200; ++trial) { + std::vector logits(n); + for (auto & v : logits) { + v = dist(rng); + } + // plain greedy + CHECK(common_sampler_fast_greedy_argmax(logits.data(), n, nullptr) == chain_pick(logits, {})); + + // a -inf bias on the argmax: both paths must move to the runner-up + const int32_t top = common_sampler_fast_greedy_argmax(logits.data(), n, nullptr); + std::vector mask(n, 0); + mask[top] = 1; + const int32_t fast = common_sampler_fast_greedy_argmax(logits.data(), n, mask.data()); + CHECK(fast != top); + CHECK(fast == chain_pick(logits, { { top, -INFINITY } })); + } +} + +static void test_nan_and_empty() { + std::vector logits = { 0.5f, 2.0f, NAN, 1.0f }; + CHECK(common_sampler_fast_greedy_argmax(logits.data(), (int32_t) logits.size(), nullptr) == -1); // full path reports it + + std::vector all_masked = { 1.0f, 2.0f }; + std::vector mask = { 1, 1 }; + CHECK(common_sampler_fast_greedy_argmax(all_masked.data(), 2, mask.data()) == -1); // nothing left: full path decides +} + +int main() { + test_eligibility(); + test_equivalence(); + test_nan_and_empty(); + if (failures != 0) { + std::fprintf(stderr, "test-sampling-fast-greedy: %d failure(s)\n", failures); + return 1; + } + std::printf("test-sampling-fast-greedy: OK\n"); + return 0; +}