Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
56 changes: 55 additions & 1 deletion common/sampling.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -234,7 +234,7 @@ bool common_sampler_fast_greedy_eligible(const common_params_sampling & params,
return ok;
}

int32_t common_sampler_fast_greedy_argmax(const float * logits, int32_t n_vocab, const char * mask) {
static int32_t fast_greedy_argmax_scalar(const float * logits, int32_t n_vocab, const char * mask) {
int32_t best = -1;
float best_v = -INFINITY;
for (int32_t i = 0; i < n_vocab; ++i) {
Expand All @@ -250,6 +250,60 @@ int32_t common_sampler_fast_greedy_argmax(const float * logits, int32_t n_vocab,
return best;
}

#if (defined(__x86_64__) || defined(__i386__)) && (defined(__GNUC__) || defined(__clang__))
// The same contract with 8-wide max and NaN lanes (engine C4-STREAMS: the scalar scan of a 248,320-entry vocab was
// 0.37 ms per slot in-server, 89 us -> 23.5 us offline with this body). Pass 1 takes the maximum over all logits; pass 2
// returns the first unmasked index holding it, which is the scalar answer whenever that maximum is reachable. When every
// copy of it is masked, or nothing is above -inf, the scalar loop decides, so masks and ties need no vector logic.
typedef float fg_v8f __attribute__((vector_size(32)));
typedef int32_t fg_v8i __attribute__((vector_size(32)));

__attribute__((target("avx2"))) static int32_t fast_greedy_argmax_avx2(const float * logits, int32_t n_vocab,
const char * mask) {
fg_v8f m = { -INFINITY, -INFINITY, -INFINITY, -INFINITY, -INFINITY, -INFINITY, -INFINITY, -INFINITY };
fg_v8i nan = { 0, 0, 0, 0, 0, 0, 0, 0 };
int32_t i = 0;
for (; i + 8 <= n_vocab; i += 8) {
fg_v8f v;
std::memcpy(&v, logits + i, sizeof(v));
nan |= (v != v);
m = v > m ? v : m;
}
float best_v = -INFINITY;
bool any_nan = false;
for (int l = 0; l < 8; ++l) {
any_nan |= nan[l] != 0;
best_v = m[l] > best_v ? m[l] : best_v;
}
for (; i < n_vocab; ++i) {
const float v = logits[i];
any_nan |= v != v;
best_v = v > best_v ? v : best_v;
}
if (any_nan) {
return -1; // the full path reports it (engine#123/#315)
}
if (best_v != -INFINITY) {
for (int32_t j = 0; j < n_vocab; ++j) {
if (logits[j] == best_v && (mask == nullptr || !mask[j])) {
return j;
}
}
}
return fast_greedy_argmax_scalar(logits, n_vocab, mask);
}
#endif

int32_t common_sampler_fast_greedy_argmax(const float * logits, int32_t n_vocab, const char * mask) {
#if (defined(__x86_64__) || defined(__i386__)) && (defined(__GNUC__) || defined(__clang__))
static const bool avx2 = __builtin_cpu_supports("avx2");
if (avx2) {
return fast_greedy_argmax_avx2(logits, n_vocab, mask);
}
#endif
return fast_greedy_argmax_scalar(logits, n_vocab, mask);
}

struct common_sampler * common_sampler_init(
const struct llama_model * model,
struct common_params_sampling & params) {
Expand Down
72 changes: 72 additions & 0 deletions tests/test-sampling-fast-greedy.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
#include "sampling.h"
#include "llama.h"

#include <algorithm>
#include <cmath>
#include <cstdio>
#include <cstdlib>
Expand Down Expand Up @@ -116,8 +117,79 @@ static void test_nan_and_empty() {
CHECK(common_sampler_fast_greedy_argmax(all_masked.data(), 2, mask.data()) == -1); // nothing left: full path decides
}

// the scalar contract the vectorised path must reproduce exactly (lowest index on ties, masks skipped, NaN -> -1)
static int32_t reference_argmax(const float * logits, int32_t n, const char * mask) {
int32_t best = -1;
float best_v = -INFINITY;
for (int32_t i = 0; i < n; ++i) {
if (std::isnan(logits[i])) {
return -1;
}
if (logits[i] > best_v && (mask == nullptr || !mask[i])) {
best_v = logits[i];
best = i;
}
}
return best;
}

// common_sampler_fast_greedy_argmax dispatches to an AVX2 body on x86 hosts that have it (C4-STREAMS); it must agree
// with the scalar contract on every input shape: ties, masks over the maximum, NaN / inf, and lengths that are not a
// multiple of the vector width.
static void test_vector_matches_scalar() {
std::mt19937 rng(4321);
std::uniform_real_distribution<float> dist(-20.0f, 20.0f);
const int32_t n = 248320;
std::vector<float> logits(n);
std::vector<char> mask(n, 0);
for (int trial = 0; trial < 120; ++trial) {
for (auto & v : logits) {
v = dist(rng);
}
std::fill(mask.begin(), mask.end(), 0);
if (trial % 3 == 1) {
for (int k = 0; k < 50; ++k) {
mask[rng() % n] = 1;
}
}
if (trial % 5 == 2) { // exact tie: the lowest id wins
logits[rng() % n] = 50.0f;
logits[rng() % n] = 50.0f;
}
if (trial % 7 == 3) { // the global maximum is masked: the next one wins
const int32_t a = rng() % n;
logits[a] = 60.0f;
mask[a] = 1;
}
if (trial % 17 == 6) { // a tie where one copy is masked
const int32_t a = rng() % n;
logits[a] = 70.0f;
mask[a] = 1;
logits[rng() % n] = 70.0f;
}
if (trial % 11 == 4) {
logits[rng() % n] = NAN;
}
if (trial % 13 == 5) {
logits[rng() % n] = INFINITY;
}
const char * m = (trial % 2) ? mask.data() : nullptr;
CHECK(common_sampler_fast_greedy_argmax(logits.data(), n, m) == reference_argmax(logits.data(), n, m));
}
for (const int32_t k : { 1, 7, 8, 9, 15, 16, 17, 31 }) { // the vector tail
std::vector<float> s(k);
for (auto & v : s) {
v = dist(rng);
}
CHECK(common_sampler_fast_greedy_argmax(s.data(), k, nullptr) == reference_argmax(s.data(), k, nullptr));
}
std::vector<float> all_inf(16, -INFINITY);
CHECK(common_sampler_fast_greedy_argmax(all_inf.data(), 16, nullptr) == -1);
}

int main() {
test_eligibility();
test_vector_matches_scalar();
test_equivalence();
test_nan_and_empty();
if (failures != 0) {
Expand Down