Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
110 changes: 61 additions & 49 deletions common/sampling.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,61 @@ std::string common_params_sampling::print() const {
return std::string(result);
}

bool common_sampler_fast_greedy_eligible(const common_params_sampling & params, bool has_grammar, bool has_reasoning_budget) {
bool ok = params.mirostat == 0 && params.n_probs == 0 && !has_grammar && !has_reasoning_budget && !params.backend_sampling &&
!params.samplers.empty() && params.samplers.back() == COMMON_SAMPLER_TYPE_TEMPERATURE &&
params.temp == 0.0f && params.dynatemp_range == 0.0f;
for (const auto type : params.samplers) {
switch (type) {
case COMMON_SAMPLER_TYPE_TOP_K:
case COMMON_SAMPLER_TYPE_TOP_P:
case COMMON_SAMPLER_TYPE_TOP_N_SIGMA:
case COMMON_SAMPLER_TYPE_TEMPERATURE:
break; // never remove the top token
case COMMON_SAMPLER_TYPE_MIN_P:
ok &= params.min_p <= 1.0f; // keeps p >= min_p * p_max, which includes the top token only for min_p <= 1
break;
case COMMON_SAMPLER_TYPE_PENALTIES:
ok &= params.penalty_last_n == 0 ||
(params.penalty_repeat == 1.0f && params.penalty_freq == 0.0f && params.penalty_present == 0.0f);
break;
case COMMON_SAMPLER_TYPE_DRY:
ok &= params.dry_multiplier == 0.0f;
break;
case COMMON_SAMPLER_TYPE_XTC:
ok &= params.xtc_probability <= 0.0f;
break;
case COMMON_SAMPLER_TYPE_TYPICAL_P:
ok &= params.typ_p >= 1.0f;
break;
default:
ok = false; // infill, adaptive-p, anything new
}
}
for (const auto & b : params.logit_bias) {
if (b.bias != -INFINITY && b.bias != 0.0f) {
ok = false; // a finite bias can move the argmax
}
}
return ok;
}

int32_t common_sampler_fast_greedy_argmax(const float * logits, int32_t n_vocab, const char * mask) {
int32_t best = -1;
float best_v = -INFINITY;
for (int32_t i = 0; i < n_vocab; ++i) {
const float v = logits[i];
if (std::isnan(v)) {
return -1; // the full path reports it (engine#123/#315)
}
if (v > best_v && (mask == nullptr || !mask[i])) {
best_v = v;
best = i;
}
}
return best;
}

struct common_sampler * common_sampler_init(
const struct llama_model * model,
struct common_params_sampling & params) {
Expand Down Expand Up @@ -451,42 +506,13 @@ struct common_sampler * common_sampler_init(
};

{
bool ok = params.mirostat == 0 && params.n_probs == 0 && !grmr && !rbudget && !params.backend_sampling &&
!params.samplers.empty() && params.samplers.back() == COMMON_SAMPLER_TYPE_TEMPERATURE &&
params.temp == 0.0f && params.dynatemp_range == 0.0f;
for (const auto type : params.samplers) {
switch (type) {
case COMMON_SAMPLER_TYPE_TOP_K:
case COMMON_SAMPLER_TYPE_TOP_P:
case COMMON_SAMPLER_TYPE_MIN_P:
case COMMON_SAMPLER_TYPE_TOP_N_SIGMA:
case COMMON_SAMPLER_TYPE_TEMPERATURE:
break; // never remove the top token
case COMMON_SAMPLER_TYPE_PENALTIES:
ok &= params.penalty_last_n == 0 ||
(params.penalty_repeat == 1.0f && params.penalty_freq == 0.0f && params.penalty_present == 0.0f);
break;
case COMMON_SAMPLER_TYPE_DRY:
ok &= params.dry_multiplier == 0.0f;
break;
case COMMON_SAMPLER_TYPE_XTC:
ok &= params.xtc_probability <= 0.0f;
break;
case COMMON_SAMPLER_TYPE_TYPICAL_P:
ok &= params.typ_p >= 1.0f;
break;
default:
ok = false; // infill, adaptive-p, anything new
}
}
bool ok = common_sampler_fast_greedy_eligible(params, grmr != nullptr, rbudget != nullptr);
if (ok) {
const int32_t n_vocab = llama_vocab_n_tokens(vocab);
result->fast_greedy_mask.assign(n_vocab, 0);
for (const auto & b : params.logit_bias) {
if (b.bias == -INFINITY) {
if (b.token >= 0 && b.token < n_vocab) result->fast_greedy_mask[b.token] = 1;
} else if (b.bias != 0.0f) {
ok = false; // a finite bias can move the argmax
if (b.bias == -INFINITY && b.token >= 0 && b.token < n_vocab) {
result->fast_greedy_mask[b.token] = 1;
}
}
int32_t n_suppress = 0;
Expand Down Expand Up @@ -670,24 +696,10 @@ llama_token common_sampler_sample(struct common_sampler * gsmpl, struct llama_co
const float * logits = llama_get_logits_ith(ctx, idx);
const int32_t n_vocab = (int32_t) gsmpl->fast_greedy_mask.size();
if (logits != nullptr && n_vocab > 0) {
const char * mask = gsmpl->fast_greedy_mask.data();
int32_t best = -1;
float best_v = -INFINITY;
bool nan = false;
for (int32_t i = 0; i < n_vocab; ++i) {
const float v = logits[i];
if (std::isnan(v)) {
nan = true; // the full path below reports it (engine#123/#315)
break;
}
if (v > best_v && !mask[i]) {
best_v = v;
best = i;
}
}
if (!nan && best >= 0) {
const int32_t best = common_sampler_fast_greedy_argmax(logits, n_vocab, gsmpl->fast_greedy_mask.data());
if (best >= 0) {
gsmpl->cur.resize(1);
gsmpl->cur[0] = { best, best_v, 1.0f };
gsmpl->cur[0] = { best, logits[best], 1.0f };
gsmpl->cur_p = { gsmpl->cur.data(), 1, 0, true };
return best;
}
Expand Down
6 changes: 6 additions & 0 deletions common/sampling.h
Original file line number Diff line number Diff line change
Expand Up @@ -67,6 +67,12 @@ struct llama_sampler * common_sampler_get(const struct common_sampler * gsmpl);
//
llama_token common_sampler_sample(struct common_sampler * gsmpl, struct llama_context * ctx, int idx, bool grammar_first = false);

// greedy fast path (common_sampler_sample): true when every stage of the chain built from params provably keeps the top
// token, so sampling is an argmax over the logits minus the -inf-biased / suppressed ids
bool common_sampler_fast_greedy_eligible(const common_params_sampling & params, bool has_grammar, bool has_reasoning_budget);
// argmax over logits skipping mask[i] != 0 (mask may be null); -1 when a logit is NaN or every id is masked / -inf
int32_t common_sampler_fast_greedy_argmax(const float * logits, int32_t n_vocab, const char * mask);

// generalized version of common_sampler_sample
//
// will cross-reference the sampled tokens with a batch of draft tokens and accept those that match
Expand Down
1 change: 1 addition & 0 deletions tests/CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -199,6 +199,7 @@ if (NOT WIN32 OR NOT BUILD_SHARED_LIBS)
target_include_directories(test-kv-zero PRIVATE ${PROJECT_SOURCE_DIR}/src)
llama_build_and_test(test-unicode.cpp LABEL behavior LINK llama)
llama_build_and_test(test-sampling.cpp LABEL behavior LINK llama)
llama_build_and_test(test-sampling-fast-greedy.cpp LABEL behavior)
llama_build_and_test(test-reasoning-budget.cpp LABEL behavior)
llama_build_and_test(test-onebit-mtp-budget.cpp LABEL behavior) # engine #2433
llama_build_and_test(test-grammar-parser.cpp LABEL behavior LINK llama)
Expand Down
129 changes: 129 additions & 0 deletions tests/test-sampling-fast-greedy.cpp
Original file line number Diff line number Diff line change
@@ -0,0 +1,129 @@
// common_sampler greedy fast path (llama.cpp#385): the eligibility predicate turns the fast path off for every stage that
// can move the argmax, and on the same logits the fast-path argmax picks the token the equivalent sampler chain picks.
// Offline: no model, the chain is built from llama.h samplers.

#include "sampling.h"
#include "llama.h"

#include <cmath>
#include <cstdio>
#include <cstdlib>
#include <random>
#include <vector>

static int failures = 0;

#define CHECK(cond) \
do { \
if (!(cond)) { \
std::fprintf(stderr, "%s:%d: CHECK failed: %s\n", __FILE__, __LINE__, #cond); \
++failures; \
} \
} while (0)

static common_params_sampling greedy_params() {
common_params_sampling p;
p.temp = 0.0f; // the default sampler order ends in TEMPERATURE
return p;
}

static void test_eligibility() {
{
const common_params_sampling p = greedy_params();
CHECK(common_sampler_fast_greedy_eligible(p, false, false));
CHECK(!common_sampler_fast_greedy_eligible(p, true, false)); // grammar
CHECK(!common_sampler_fast_greedy_eligible(p, false, true)); // reasoning budget
}
auto off = [](void (*edit)(common_params_sampling &)) {
common_params_sampling p = greedy_params();
edit(p);
return !common_sampler_fast_greedy_eligible(p, false, false);
};
CHECK(off([](common_params_sampling & p) { p.temp = 0.7f; }));
CHECK(off([](common_params_sampling & p) { p.dynatemp_range = 0.5f; }));
CHECK(off([](common_params_sampling & p) { p.mirostat = 2; }));
CHECK(off([](common_params_sampling & p) { p.n_probs = 1; }));
CHECK(off([](common_params_sampling & p) { p.backend_sampling = true; }));
CHECK(off([](common_params_sampling & p) { p.xtc_probability = 0.5f; }));
CHECK(off([](common_params_sampling & p) { p.typ_p = 0.9f; }));
CHECK(off([](common_params_sampling & p) { p.min_p = 1.5f; }));
CHECK(off([](common_params_sampling & p) { p.dry_multiplier = 0.8f; }));
CHECK(off([](common_params_sampling & p) { p.penalty_repeat = 1.1f; }));
CHECK(off([](common_params_sampling & p) { p.logit_bias.push_back({ 7, 2.5f }); })); // finite bias
CHECK(off([](common_params_sampling & p) { p.samplers.push_back(COMMON_SAMPLER_TYPE_TOP_K); })); // temp not last
{
common_params_sampling p = greedy_params();
p.logit_bias.push_back({ 7, -INFINITY }); // a suppression is a mask, not a reason to fall back
CHECK(common_sampler_fast_greedy_eligible(p, false, false));
p.penalty_last_n = 0;
p.penalty_repeat = 1.3f; // penalties over zero tokens do nothing
CHECK(common_sampler_fast_greedy_eligible(p, false, false));
}
}

// the chain common_sampler_init builds for greedy_params(), minus the stages that are identity at their defaults
static llama_token chain_pick(const std::vector<float> & logits, const std::vector<llama_logit_bias> & bias) {
const int32_t n = (int32_t) logits.size();
llama_sampler * chain = llama_sampler_chain_init(llama_sampler_chain_default_params());
if (!bias.empty()) {
llama_sampler_chain_add(chain, llama_sampler_init_logit_bias(n, (int32_t) bias.size(), bias.data()));
}
llama_sampler_chain_add(chain, llama_sampler_init_top_k(40));
llama_sampler_chain_add(chain, llama_sampler_init_top_p(0.95f, 1));
llama_sampler_chain_add(chain, llama_sampler_init_min_p(0.05f, 1));
llama_sampler_chain_add(chain, llama_sampler_init_temp_ext(0.0f, 0.0f, 1.0f));
llama_sampler_chain_add(chain, llama_sampler_init_greedy());

std::vector<llama_token_data> cur(n);
for (int32_t i = 0; i < n; ++i) {
cur[i] = { i, logits[i], 0.0f };
}
llama_token_data_array cur_p = { cur.data(), cur.size(), -1, false };
llama_sampler_apply(chain, &cur_p);
const llama_token id = cur_p.data[cur_p.selected].id;
llama_sampler_free(chain);
return id;
}

static void test_equivalence() {
std::mt19937 rng(1234);
std::uniform_real_distribution<float> dist(-8.0f, 8.0f);
const int32_t n = 4096;
for (int trial = 0; trial < 200; ++trial) {
std::vector<float> logits(n);
for (auto & v : logits) {
v = dist(rng);
}
// plain greedy
CHECK(common_sampler_fast_greedy_argmax(logits.data(), n, nullptr) == chain_pick(logits, {}));

// a -inf bias on the argmax: both paths must move to the runner-up
const int32_t top = common_sampler_fast_greedy_argmax(logits.data(), n, nullptr);
std::vector<char> mask(n, 0);
mask[top] = 1;
const int32_t fast = common_sampler_fast_greedy_argmax(logits.data(), n, mask.data());
CHECK(fast != top);
CHECK(fast == chain_pick(logits, { { top, -INFINITY } }));
}
}

static void test_nan_and_empty() {
std::vector<float> logits = { 0.5f, 2.0f, NAN, 1.0f };
CHECK(common_sampler_fast_greedy_argmax(logits.data(), (int32_t) logits.size(), nullptr) == -1); // full path reports it

std::vector<float> all_masked = { 1.0f, 2.0f };
std::vector<char> mask = { 1, 1 };
CHECK(common_sampler_fast_greedy_argmax(all_masked.data(), 2, mask.data()) == -1); // nothing left: full path decides
}

int main() {
test_eligibility();
test_equivalence();
test_nan_and_empty();
if (failures != 0) {
std::fprintf(stderr, "test-sampling-fast-greedy: %d failure(s)\n", failures);
return 1;
}
std::printf("test-sampling-fast-greedy: OK\n");
return 0;
}