diff --git a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-mul-mat.cpp b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-mul-mat.cpp index 4d3c6ab569e8..1bebf0833ca9 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-mul-mat.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-mul-mat.cpp @@ -306,7 +306,10 @@ static bool matches_embedded_external_concat_projection(const Graph & graph, second_producer != nullptr && second_producer->op == GGML_OP_GET_ROWS && second_producer->inputs.size() == 2; const bool first_is_external = first_producer == nullptr; const bool second_is_external = second_producer == nullptr; - return (first_is_embedding && second_is_external) || (second_is_embedding && first_is_external); + // DECODE-BW: the MTP draft graph (common/speculative.cpp draft-mtp) gets the token embedding as an input + // (mtp_tok_embd), not a GET_ROWS in the graph, so both branches are external there. + return (first_is_embedding && second_is_external) || (second_is_embedding && first_is_external) || + (first_is_external && second_is_external); } static bool has_embedded_external_concat_projection_ancestor(const Graph & graph, @@ -446,6 +449,47 @@ static bool match_symmetric_i4_low_row_dispatch(const DispatchMatchContext & con return dispatch_match.status.success(); } +// DECODE-BW: the q6_k shortlist writes -inf outside its refined candidates. A reader of those logits is safe if it only +// looks at the best few: a TOP_K whose k fits in the candidates, or a GET_ROWS of the values at such a TOP_K's ids. +// No consumer at all (a graph output) is the original case. The draft backend sampler (llama_sampler_init_top_k(10), +// common/speculative.cpp) builds result_output -> PAD -> (reshape) -> TOP_K, plus GET_ROWS(logits, top_k). +static bool logits_read_only_by_candidate_readers(const Graph & graph, ValueId value, int64_t candidate_count, int depth) { + if (depth > 6) { + return false; + } + const std::vector aliases = layout_alias_consumers(graph, value); + for (const GraphNode * consumer : graph.index().consumers(value)) { + if (consumer == nullptr) { + return false; + } + const bool is_alias = std::find(aliases.begin(), aliases.end(), consumer) != aliases.end(); + if (is_alias || consumer->op == GGML_OP_PAD) { + if (!logits_read_only_by_candidate_readers(graph, consumer->output, candidate_count, depth + 1)) { + return false; + } + continue; + } + if (consumer->op == GGML_OP_TOP_K) { + const Value * out = common_graph_value(graph, consumer->output); + if (out == nullptr || out->ne[0] > candidate_count) { + return false; + } + continue; + } + if (consumer->op == GGML_OP_GET_ROWS && consumer->inputs.size() == 2 && consumer->inputs[0] == value) { + const GraphNode * ids = graph.index().producer(consumer->inputs[1]); + const Value * ids_value = ids != nullptr ? common_graph_value(graph, ids->output) : nullptr; + if (ids == nullptr || ids->op != GGML_OP_TOP_K || ids_value == nullptr || + ids_value->ne[0] > candidate_count) { + return false; + } + continue; + } + return false; + } + return true; +} + static bool match_q6_k_token1_shortlist_dispatch(const DispatchMatchContext & context, DispatchMatch & dispatch_match) { constexpr int64_t max_selected_groups = 96; constexpr int64_t candidate_count = 128; @@ -457,8 +501,9 @@ static bool match_q6_k_token1_shortlist_dispatch(const DispatchMatchContext & co if (!match.matched() || !context.graph.has_index() || match.weight->type != GGML_TYPE_Q6_K || match.token_count != 1 || match.weight->alias_source.value >= 0 || match.input_size % 256 != 0 || match.input_size > 8192 || match.output_size < 65536 || match.output_size % 64 != 0 || - match.output_size < 16 * match.input_size || !context.graph.index().consumers(match.output->id).empty() || - !has_embedded_external_concat_projection_ancestor(context.graph, match.input->id, match.input_size)) { + match.output_size < 16 * match.input_size || + !has_embedded_external_concat_projection_ancestor(context.graph, match.input->id, match.input_size) || + !logits_read_only_by_candidate_readers(context.graph, match.output->id, candidate_count, 0)) { return false; } diff --git a/tests/hrx-backend-test.cpp b/tests/hrx-backend-test.cpp index d949262fc672..8e96bac3127a 100644 --- a/tests/hrx-backend-test.cpp +++ b/tests/hrx-backend-test.cpp @@ -4073,6 +4073,70 @@ static void schedule_q6k_token1_shortlist_group_count(int64_t hidden_size, int64 ggml_free(ctx); } +// DECODE-BW: the MTP draft head (common/speculative.cpp draft-mtp) as the fork builds it. The token embedding arrives as +// a graph input (mtp_tok_embd), not a GET_ROWS, and with draft backend sampling (the default) the logits feed a +// TOP_K(10). The shortlist must take that head. It must decline a TOP_K wider than its 128 refined candidates and a +// plain lm_head with no MTP concat projection above it. +enum class DraftEmbedding { External, GetRows, NoConcat }; +// chain: 0 = TOP_K straight on the logits (top_k > 0) or none; 1 = the backend sampler's chain as the fork builds it +// (PAD -> reshape -> TOP_K, plus GET_ROWS of the values at the TOP_K ids); 2 = that chain plus a SCALE over all the +// logits, which reads values the shortlist does not compute. +static void schedule_mtp_draft_shortlist(DraftEmbedding embedding, int top_k, bool expect_shortlist, int chain = 0) { + constexpr int64_t hidden_size = 2560; + constexpr int64_t vocab_size = 65536; + ggml_init_params params = {}; + params.mem_size = 4 * 1024 * 1024; + params.no_alloc = true; + ggml_context * ctx = ggml_init(params); + REQUIRE(ctx != nullptr); + ggml_tensor * external = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, hidden_size, 1); + ggml_tensor * scale_a = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, hidden_size, 1); + ggml_tensor * scale_b = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, hidden_size, 1); + ggml_tensor * projection = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, 2 * hidden_size, hidden_size); + ggml_tensor * head = ggml_new_tensor_2d(ctx, GGML_TYPE_Q6_K, hidden_size, vocab_size); + ggml_tensor * head_input = nullptr; + if (embedding == DraftEmbedding::NoConcat) { + head_input = ggml_mul(ctx, ggml_rms_norm(ctx, external, 0.000001f), scale_b); + } else { + ggml_tensor * token = nullptr; + if (embedding == DraftEmbedding::GetRows) { + ggml_tensor * table = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, hidden_size, 8); + ggml_tensor * ids = ggml_new_tensor_1d(ctx, GGML_TYPE_I32, 1); + token = ggml_get_rows(ctx, table, ids); + } else { + token = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, hidden_size, 1); + } + ggml_tensor * branch_a = ggml_mul(ctx, ggml_rms_norm(ctx, token, 0.000001f), scale_a); + ggml_tensor * branch_b = ggml_mul(ctx, ggml_rms_norm(ctx, external, 0.000001f), scale_b); + head_input = ggml_mul_mat(ctx, projection, ggml_concat(ctx, branch_a, branch_b, 0)); + } + ggml_tensor * logits = ggml_mul_mat(ctx, head, head_input); + ggml_cgraph * graph = ggml_new_graph(ctx); + if (chain == 0) { + ggml_build_forward_expand(graph, top_k > 0 ? ggml_top_k(ctx, logits, top_k) : logits); + } else { + ggml_tensor * padded = ggml_pad(ctx, logits, 0, 1, 0, 0); + ggml_tensor * row = ggml_reshape_2d(ctx, ggml_view_1d(ctx, padded, vocab_size, 0), vocab_size, 1); + ggml_tensor * ids = ggml_top_k(ctx, row, top_k); + ggml_tensor * values = ggml_get_rows(ctx, ggml_reshape_2d(ctx, row, 1, vocab_size), ids); + ggml_build_forward_expand(graph, values); + if (chain == 2) { + ggml_build_forward_expand(graph, ggml_scale(ctx, logits, 0.5f)); + } + } + ggml::hrx::GraphImportResult imported = ggml::hrx::import_ggml_graph(*graph); + REQUIRE(imported.valid()); + ggml::hrx::DispatchScheduler scheduler; + REQUIRE(scheduler.schedule_graph(imported.graph, test_dispatch_target())); + REQUIRE(scheduler.plan().valid()); + size_t scan_count = 0; + for (const ggml::hrx::Dispatch & dispatch : scheduler.plan().dispatches) { + scan_count += kernel_name_for_id(dispatch.kernel.kernel_id) == "loom_libs:ggml_mul_mat_q6_k_symmetric_i2_scan_token1"; + } + REQUIRE(scan_count == (expect_shortlist ? 1u : 0u)); + ggml_free(ctx); +} + static void schedule_qwen_terminal_q6k_q8_command(int64_t token_count) { ggml_init_params params = {}; params.mem_size = 4 * 1024 * 1024; @@ -16143,6 +16207,13 @@ int main(int argc, char ** argv) { [] { schedule_packed_layout_follows_weight(GGML_TYPE_Q6_K, true, 100, true); } }, { "packed_layout_follows_weight.q6_k.file_mapped.stored_copy.tokens12", [] { schedule_packed_layout_follows_weight(GGML_TYPE_Q6_K, true, 12, true); } }, + { "mtp_draft_shortlist.external_embedding", [] { schedule_mtp_draft_shortlist(DraftEmbedding::External, 0, true); } }, + { "mtp_draft_shortlist.external_embedding.top_k10", [] { schedule_mtp_draft_shortlist(DraftEmbedding::External, 10, true); } }, + { "mtp_draft_shortlist.get_rows_embedding.top_k10", [] { schedule_mtp_draft_shortlist(DraftEmbedding::GetRows, 10, true); } }, + { "mtp_draft_shortlist.top_k_wider_than_candidates", [] { schedule_mtp_draft_shortlist(DraftEmbedding::External, 200, false); } }, + { "mtp_draft_shortlist.sampler_chain", [] { schedule_mtp_draft_shortlist(DraftEmbedding::External, 10, true, 1); } }, + { "mtp_draft_shortlist.sampler_chain_plus_full_reader", [] { schedule_mtp_draft_shortlist(DraftEmbedding::External, 10, false, 2); } }, + { "mtp_draft_shortlist.plain_lm_head_declines", [] { schedule_mtp_draft_shortlist(DraftEmbedding::NoConcat, 10, false); } }, { "q6k_token1_shortlist_group_count.256", [] { schedule_q6k_token1_shortlist_group_count(256, 65536, 8); } }, { "q6k_token1_shortlist_group_count.2560", [] { schedule_q6k_token1_shortlist_group_count(2560, 248320, 80); } },