diff --git a/common/sampling.cpp b/common/sampling.cpp index 8f69bf320468..d2c91cf9eb56 100644 --- a/common/sampling.cpp +++ b/common/sampling.cpp @@ -234,7 +234,7 @@ bool common_sampler_fast_greedy_eligible(const common_params_sampling & params, return ok; } -int32_t common_sampler_fast_greedy_argmax(const float * logits, int32_t n_vocab, const char * mask) { +static int32_t fast_greedy_argmax_scalar(const float * logits, int32_t n_vocab, const char * mask) { int32_t best = -1; float best_v = -INFINITY; for (int32_t i = 0; i < n_vocab; ++i) { @@ -250,6 +250,60 @@ int32_t common_sampler_fast_greedy_argmax(const float * logits, int32_t n_vocab, return best; } +#if (defined(__x86_64__) || defined(__i386__)) && (defined(__GNUC__) || defined(__clang__)) +// The same contract with 8-wide max and NaN lanes (engine C4-STREAMS: the scalar scan of a 248,320-entry vocab was +// 0.37 ms per slot in-server, 89 us -> 23.5 us offline with this body). Pass 1 takes the maximum over all logits; pass 2 +// returns the first unmasked index holding it, which is the scalar answer whenever that maximum is reachable. When every +// copy of it is masked, or nothing is above -inf, the scalar loop decides, so masks and ties need no vector logic. +typedef float fg_v8f __attribute__((vector_size(32))); +typedef int32_t fg_v8i __attribute__((vector_size(32))); + +__attribute__((target("avx2"))) static int32_t fast_greedy_argmax_avx2(const float * logits, int32_t n_vocab, + const char * mask) { + fg_v8f m = { -INFINITY, -INFINITY, -INFINITY, -INFINITY, -INFINITY, -INFINITY, -INFINITY, -INFINITY }; + fg_v8i nan = { 0, 0, 0, 0, 0, 0, 0, 0 }; + int32_t i = 0; + for (; i + 8 <= n_vocab; i += 8) { + fg_v8f v; + std::memcpy(&v, logits + i, sizeof(v)); + nan |= (v != v); + m = v > m ? v : m; + } + float best_v = -INFINITY; + bool any_nan = false; + for (int l = 0; l < 8; ++l) { + any_nan |= nan[l] != 0; + best_v = m[l] > best_v ? m[l] : best_v; + } + for (; i < n_vocab; ++i) { + const float v = logits[i]; + any_nan |= v != v; + best_v = v > best_v ? v : best_v; + } + if (any_nan) { + return -1; // the full path reports it (engine#123/#315) + } + if (best_v != -INFINITY) { + for (int32_t j = 0; j < n_vocab; ++j) { + if (logits[j] == best_v && (mask == nullptr || !mask[j])) { + return j; + } + } + } + return fast_greedy_argmax_scalar(logits, n_vocab, mask); +} +#endif + +int32_t common_sampler_fast_greedy_argmax(const float * logits, int32_t n_vocab, const char * mask) { +#if (defined(__x86_64__) || defined(__i386__)) && (defined(__GNUC__) || defined(__clang__)) + static const bool avx2 = __builtin_cpu_supports("avx2"); + if (avx2) { + return fast_greedy_argmax_avx2(logits, n_vocab, mask); + } +#endif + return fast_greedy_argmax_scalar(logits, n_vocab, mask); +} + struct common_sampler * common_sampler_init( const struct llama_model * model, struct common_params_sampling & params) { diff --git a/tests/test-sampling-fast-greedy.cpp b/tests/test-sampling-fast-greedy.cpp index e89167a49c2c..e45d9d6cfe90 100644 --- a/tests/test-sampling-fast-greedy.cpp +++ b/tests/test-sampling-fast-greedy.cpp @@ -5,6 +5,7 @@ #include "sampling.h" #include "llama.h" +#include #include #include #include @@ -116,8 +117,79 @@ static void test_nan_and_empty() { CHECK(common_sampler_fast_greedy_argmax(all_masked.data(), 2, mask.data()) == -1); // nothing left: full path decides } +// the scalar contract the vectorised path must reproduce exactly (lowest index on ties, masks skipped, NaN -> -1) +static int32_t reference_argmax(const float * logits, int32_t n, const char * mask) { + int32_t best = -1; + float best_v = -INFINITY; + for (int32_t i = 0; i < n; ++i) { + if (std::isnan(logits[i])) { + return -1; + } + if (logits[i] > best_v && (mask == nullptr || !mask[i])) { + best_v = logits[i]; + best = i; + } + } + return best; +} + +// common_sampler_fast_greedy_argmax dispatches to an AVX2 body on x86 hosts that have it (C4-STREAMS); it must agree +// with the scalar contract on every input shape: ties, masks over the maximum, NaN / inf, and lengths that are not a +// multiple of the vector width. +static void test_vector_matches_scalar() { + std::mt19937 rng(4321); + std::uniform_real_distribution dist(-20.0f, 20.0f); + const int32_t n = 248320; + std::vector logits(n); + std::vector mask(n, 0); + for (int trial = 0; trial < 120; ++trial) { + for (auto & v : logits) { + v = dist(rng); + } + std::fill(mask.begin(), mask.end(), 0); + if (trial % 3 == 1) { + for (int k = 0; k < 50; ++k) { + mask[rng() % n] = 1; + } + } + if (trial % 5 == 2) { // exact tie: the lowest id wins + logits[rng() % n] = 50.0f; + logits[rng() % n] = 50.0f; + } + if (trial % 7 == 3) { // the global maximum is masked: the next one wins + const int32_t a = rng() % n; + logits[a] = 60.0f; + mask[a] = 1; + } + if (trial % 17 == 6) { // a tie where one copy is masked + const int32_t a = rng() % n; + logits[a] = 70.0f; + mask[a] = 1; + logits[rng() % n] = 70.0f; + } + if (trial % 11 == 4) { + logits[rng() % n] = NAN; + } + if (trial % 13 == 5) { + logits[rng() % n] = INFINITY; + } + const char * m = (trial % 2) ? mask.data() : nullptr; + CHECK(common_sampler_fast_greedy_argmax(logits.data(), n, m) == reference_argmax(logits.data(), n, m)); + } + for (const int32_t k : { 1, 7, 8, 9, 15, 16, 17, 31 }) { // the vector tail + std::vector s(k); + for (auto & v : s) { + v = dist(rng); + } + CHECK(common_sampler_fast_greedy_argmax(s.data(), k, nullptr) == reference_argmax(s.data(), k, nullptr)); + } + std::vector all_inf(16, -INFINITY); + CHECK(common_sampler_fast_greedy_argmax(all_inf.data(), 16, nullptr) == -1); +} + int main() { test_eligibility(); + test_vector_matches_scalar(); test_equivalence(); test_nan_and_empty(); if (failures != 0) {