diff --git a/src/llama-context.cpp b/src/llama-context.cpp index f286bd1da2a..0394737ee3b 100644 --- a/src/llama-context.cpp +++ b/src/llama-context.cpp @@ -1802,6 +1802,9 @@ int llama_context::decode(const llama_batch & batch_inp) { int64_t n_outputs_prev = 0; int64_t n_tokens_prev = 0; + bool has_next_ubatch = false; + bool mtp_multi_ubatch = false; + do { const auto & ubatch = mctx->get_ubatch(); @@ -1977,7 +1980,15 @@ int llama_context::decode(const llama_batch & batch_inp) { n_outputs_prev += n_outputs; n_tokens_prev += ubatch.n_tokens; - } while (mctx->next()); + + has_next_ubatch = mctx->next(); + mtp_multi_ubatch |= has_next_ubatch; + + // MTP ubatches update the same KV cache and must complete in order. + if (cparams.ctx_type == LLAMA_CONTEXT_TYPE_MTP && mtp_multi_ubatch) { + synchronize(); + } + } while (has_next_ubatch); // set to total number of outputs in the batch, for use in llama_get_logits_ith n_outputs = n_outputs_all; diff --git a/tests/CMakeLists.txt b/tests/CMakeLists.txt index c46377c7623..7ac458f625a 100644 --- a/tests/CMakeLists.txt +++ b/tests/CMakeLists.txt @@ -197,6 +197,12 @@ if (NOT WIN32 OR NOT BUILD_SHARED_LIBS) # llama_build_and_test(test-double-float.cpp) # SLOW llama_build_and_test(test-llama-archs.cpp) + llama_test( + test-llama-archs + NAME test-mtp-ubatch-sync + LABEL main + ARGS --test-mtp-ubatch-sync + ) set(MODEL_DIR "${CMAKE_CURRENT_BINARY_DIR}/test-models/") file(MAKE_DIRECTORY "${MODEL_DIR}") diff --git a/tests/test-llama-archs.cpp b/tests/test-llama-archs.cpp index b2ea245ab84..30d17fd8561 100644 --- a/tests/test-llama-archs.cpp +++ b/tests/test-llama-archs.cpp @@ -65,7 +65,7 @@ static void set_tensor_data(struct ggml_tensor * tensor, void * userdata) { } static void usage(char ** argv) { - printf("Usage: %s [-a/--arch arch] [-s/--seed seed] [-o/--out dir] [-v N] [-h/--help]\n", argv[0]); + printf("Usage: %s [-a/--arch arch] [-s/--seed seed] [-o/--out dir] [-v N] [-h/--help] [--test-mtp-ubatch-sync]\n", argv[0]); } static std::vector get_tokens(const uint32_t n_tokens, const uint32_t n_vocab, const size_t seed){ @@ -79,7 +79,7 @@ static std::vector get_tokens(const uint32_t n_tokens, const uint32 return ret; } -static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { +static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe, const bool mtp = false) { gguf_context_ptr ret(gguf_init_empty()); llama_model_saver ms(arch, ret.get()); const uint32_t n_ctx = 256; @@ -145,6 +145,9 @@ static gguf_context_ptr get_gguf_ctx(const llm_arch arch, const bool moe) { ms.add_kv(LLM_KV_FEATURES_LENGTH, n_embd); ms.add_kv(LLM_KV_BLOCK_COUNT, n_layer); ms.add_kv(LLM_KV_LEADING_DENSE_BLOCK_COUNT, uint32_t(1)); + if (mtp) { + ms.add_kv(LLM_KV_NEXTN_PREDICT_LAYERS, uint32_t(1)); + } if (arch == LLM_ARCH_NEMOTRON_H || arch == LLM_ARCH_NEMOTRON_H_MOE) { std::vector n_ff_per_layer; @@ -402,6 +405,99 @@ static std::pair get_model_and_ctx( return std::make_pair(std::move(model), std::move(lctx)); } +struct mtp_sync_test_data { + llama_context * ctx; + int32_t n_tokens; + bool saw_sync; +}; + +static bool mtp_sync_test_abort(void * user_data) { + auto * data = (mtp_sync_test_data *) user_data; + // synchronize() accounts queued tokens before the next ubatch starts. + if (llama_perf_context(data->ctx).n_p_eval >= data->n_tokens) { + data->saw_sync = true; + } + return false; +} + +static bool mtp_sync_test_decode(llama_model * model, uint32_t n_ubatch) { + const int32_t n_tokens = 4; + const int32_t n_embd = llama_model_n_embd_out(model); + + llama_context_params ctx_params = llama_context_default_params(); + ctx_params.n_ctx = 8; + ctx_params.n_batch = n_tokens; + ctx_params.n_ubatch = n_ubatch; + ctx_params.n_threads = 4; + ctx_params.n_threads_batch = 4; + ctx_params.ctx_type = LLAMA_CONTEXT_TYPE_MTP; + + llama_context_ptr ctx(llama_init_from_model(model, ctx_params)); + if (!ctx) { + throw std::runtime_error("failed to create MTP context"); + } + + std::vector token(n_tokens); + std::vector embd((size_t) n_tokens * n_embd, 1.0e-2f); + std::vector pos(n_tokens); + std::vector n_seq_id(n_tokens, 1); + std::vector seq_id_data(n_tokens, 0); + std::vector seq_id(n_tokens); + std::vector logits(n_tokens, 0); + + for (int32_t i = 0; i < n_tokens; ++i) { + token[i] = i; + pos[i] = i; + seq_id[i] = &seq_id_data[i]; + } + logits.back() = 1; + + llama_batch batch = { + /*.n_tokens =*/ n_tokens, + /*.token =*/ token.data(), + /*.embd =*/ embd.data(), + /*.pos =*/ pos.data(), + /*.n_seq_id =*/ n_seq_id.data(), + /*.seq_id =*/ seq_id.data(), + /*.logits =*/ logits.data(), + }; + + mtp_sync_test_data data = { ctx.get(), n_tokens, false }; + llama_perf_context_reset(ctx.get()); + llama_set_abort_callback(ctx.get(), mtp_sync_test_abort, &data); + const int32_t ret = llama_decode(ctx.get(), batch); + llama_set_abort_callback(ctx.get(), nullptr, nullptr); + if (ret != 0) { + throw std::runtime_error("failed to decode MTP batch"); + } + + return data.saw_sync; +} + +static int test_mtp_ubatch_sync(const size_t seed) { + gguf_context_ptr gguf_ctx = get_gguf_ctx(LLM_ARCH_QWEN35, false, true); + llama_model_params model_params = llama_model_default_params(); + model_params.progress_callback = silent_model_load_progress; + model_params.load_mtp = true; + + size_t tmp = seed; + llama_model_ptr model(llama_model_init_from_user(gguf_ctx.get(), set_tensor_data, &tmp, model_params)); + if (!model) { + throw std::runtime_error("failed to create MTP model"); + } + + if (!mtp_sync_test_decode(model.get(), 2)) { + fprintf(stderr, "MTP ubatches were not synchronized\n"); + return 1; + } + if (mtp_sync_test_decode(model.get(), 4)) { + fprintf(stderr, "single MTP ubatch was synchronized\n"); + return 1; + } + + return 0; +} + static std::vector get_logits( llama_model * model, llama_context * lctx, const std::vector & tokens, bool encode = false) { const uint32_t n_vocab = llama_vocab_n_tokens(llama_model_get_vocab(model)); @@ -798,6 +894,7 @@ int main(int argc, char ** argv) { llm_arch arch = LLM_ARCH_UNKNOWN; size_t seed = rd(); std::string out; + bool test_mtp_sync = false; int verbosity = LOG_LEVEL_ERROR; @@ -843,6 +940,9 @@ int main(int argc, char ** argv) { return 1; } } + if (strcmp(argv[i], "--test-mtp-ubatch-sync") == 0) { + test_mtp_sync = true; + } } printf("%s: using seed %zu\n", __func__, seed); @@ -850,6 +950,9 @@ int main(int argc, char ** argv) { if (!out.empty()) { return save_models(arch, seed, verbosity, out); } + if (test_mtp_sync) { + return test_mtp_ubatch_sync(seed); + } return test_backends(arch, seed, verbosity); } catch (const std::exception & err) { fprintf(stderr, "encountered runtime error: %s\n", err.what());