From 4d53bd17e659fcebbf5a1b4c548f1631d37e033e Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sat, 14 Nov 2026 12:17:47 -0400 Subject: [PATCH 1/2] hrx: one-token decode with mixed-format gate/up runs unfused (DECODE-BW) On Qwen3.8-27B Q4_K_M, 17 layers pair a Q4_K gate with a Q6_K up. The fused direct-dot SwiGLU body reads them at ~181 GB/s; two separate decode GEMVs plus the GLU dispatch are faster. Same-format pairs and 2..5-token batches keep the fused body. Co-Authored-By: Claude Opus 5.5 --- .../common/dispatch-gated-mul-mat.cpp | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-gated-mul-mat.cpp b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-gated-mul-mat.cpp index 7f5f231d55fc..1918ff5b17cc 100644 --- a/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-gated-mul-mat.cpp +++ b/ggml/src/ggml-hrx/dispatch_registration/common/dispatch-gated-mul-mat.cpp @@ -733,6 +733,13 @@ static bool match_mul_mat_swiglu_dispatch(const DispatchMatchContext & context, Dispatch dispatch; const bool use_direct_dot = match.token_count <= 5 && match.input_size % 256 == 0; + // One-token decode with gate and up in DIFFERENT formats (Qwen3.8-27B Q4_K_M: Q4_K gate + Q6_K up on 17 layers): the + // fused direct-dot body runs at ~181 GB/s, while two separate decode GEMVs + the GLU dispatch reach the weights' own + // kernels. Measured 11.79/11.80/11.80 -> 11.97/11.98/11.97 tok/s (tg64, three interleaved pairs, greedy text + // identical; DECODE-BW 11-14). Same-format pairs and 2..5-token batches (MTP verify) keep the fused body. + if (use_direct_dot && !use_q8 && match.token_count == 1 && match.gate_format != match.up_format) { + return false; + } const bool use_wmma = !use_q8 && !use_direct_dot; if (use_wmma && match.token_count < 2) { return false; From 5d1243db837c59a4f92279ad54ba55e121ea705c Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Sat, 14 Nov 2026 13:05:49 -0400 Subject: [PATCH 2/2] tests: the mixed-format one-token SwiGLU decode runs unfused; 2 tokens and one format stay fused Three host cases at Qwen3.8-27B's FFN shape (5120 -> 17408), with the FFN input built as the model builds it (RMSNorm x weight, which publishes the q8_1 copy that sends the pair past kquant.swiglu.decode_f32 to the f32 SwiGLU matcher). In an add-on build the add-on's few-token kernel takes the 2-token pair; labelled, and hrx-backend-test-loom still asserts it. Co-Authored-By: Claude Opus 5.5 --- tests/hrx-backend-test.cpp | 43 ++++++++++++++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/tests/hrx-backend-test.cpp b/tests/hrx-backend-test.cpp index bc04d8e2b6d8..b517d04c96c1 100644 --- a/tests/hrx-backend-test.cpp +++ b/tests/hrx-backend-test.cpp @@ -3797,6 +3797,44 @@ static void schedule_swiglu_q4k_prefill_layout(bool stored_copy_exists) { ggml_free(ctx); } +// DECODE-BW (11-14): a one-token decode whose gate and up weights are in DIFFERENT formats (Qwen3.8-27B Q4_K_M: Q4_K gate, +// Q6_K up) does not take the fused direct-dot SwiGLU body; separate decode GEMVs + the GLU dispatch were 1.5 % faster +// end to end. Two tokens (an MTP verify batch) keep the fused body, and so does a one-token pair in ONE format. +static void schedule_mixed_format_swiglu_decode(ggml_type gate_type, ggml_type up_type, int64_t token_count, + bool expect_fused) { + // Qwen3.8-27B's FFN shape: at a small hidden size kquant.swiglu.decode_f32 takes the pair and this matcher never runs + constexpr int64_t input_size = 5120; + constexpr int64_t hidden = 17408; + ggml_init_params params = {}; + params.mem_size = 4 * 1024 * 1024; + params.no_alloc = true; + ggml_context * ctx = ggml_init(params); + REQUIRE(ctx != nullptr); + ggml_tensor * gate_weight = ggml_new_tensor_2d(ctx, gate_type, input_size, hidden); + ggml_tensor * up_weight = ggml_new_tensor_2d(ctx, up_type, input_size, hidden); + // the FFN input as the model builds it: RMSNorm x weight, which publishes a q8_1 copy the kquant SwiGLU decode + // kernel declines (it reads f32), so the pair reaches the f32 SwiGLU matcher as on the real model + ggml_tensor * hidden_in = ggml_new_tensor_2d(ctx, GGML_TYPE_F32, input_size, token_count); + ggml_tensor * norm_weight = ggml_new_tensor_1d(ctx, GGML_TYPE_F32, input_size); + ggml_tensor * input = ggml_mul(ctx, ggml_rms_norm(ctx, hidden_in, 1e-6f), norm_weight); + ggml_tensor * gate = ggml_mul_mat(ctx, gate_weight, input); + ggml_tensor * up = ggml_mul_mat(ctx, up_weight, input); + ggml_tensor * output = ggml_swiglu_split(ctx, gate, up); + ggml_cgraph * graph = ggml_new_graph(ctx); + ggml_build_forward_expand(graph, output); + ggml::hrx::GraphImportResult imported = ggml::hrx::import_ggml_graph(*graph); + REQUIRE(imported.valid()); + ggml::hrx::DispatchScheduler scheduler; + REQUIRE(scheduler.schedule_graph(imported.graph, test_dispatch_target())); + size_t fused = 0; + for (const ggml::hrx::Dispatch & dispatch : scheduler.plan().dispatches) { + fused += kernel_name_for_id(dispatch.kernel.kernel_id) == "loom_libs:ggml_mul_mat_swiglu_f32_f32_lowtoken_dot"; + } + REQUIRE(fused == (expect_fused ? 1u : 0u)); + REQUIRE(!scheduler.plan().dispatches.empty()); // the unfused case is still planned, not dropped + ggml_free(ctx); +} + static void schedule_packed_layout_follows_weight(ggml_type weight_type, bool file_mapped, int64_t token_count = 100, bool stored_copy_exists = false) { if (std::getenv("GGML_HRX_Q4K_PACKED_TAIL") != nullptr || std::getenv("GGML_HRX_Q6K_PACKED_ROWS") != nullptr) { @@ -15554,6 +15592,8 @@ static const HipAddonLabel * hip_addon_known_fail(const char * name) { // the add-on's few-token kernel takes the tied weight instead of the Loom kquant kernel { "tied_weight_few_tokens_keeps_kquant.q4_k.tokens6", kPlan, "kquant == 1" }, { "tied_weight_few_tokens_keeps_kquant.q4_k.tokens8", kPlan, "kquant == 1" }, + // the add-on's few-token kernel takes the 2-token mixed-format gate/up pair (the Loom fused body is the -loom row's) + { "mixed_format_swiglu_decode.q4k_q6k.t2", kPlan, "fused == (expect_fused ? 1u : 0u)" }, // C1: the norm publishes a q8 alternate no dispatch reads, and the add-on consumer takes no f16 one { "rmsnorm_binary_f16_output", "#2392 C1 add-on consumer reads neither alternate", "publication->alternate_value" }, // hip.moe_tokens.* publishes the routed gate/up plane as Q8_1, not as f16 at 2^-6 @@ -16070,6 +16110,9 @@ int main(int argc, char ** argv) { { "q4k_aligned_prefill_layout.stored_copy", [] { schedule_q4k_aligned_prefill_layout(true); } }, { "swiglu_q4k_prefill_layout.packed", [] { schedule_swiglu_q4k_prefill_layout(false); } }, { "swiglu_q4k_prefill_layout.stored_copy", [] { schedule_swiglu_q4k_prefill_layout(true); } }, + { "mixed_format_swiglu_decode.q4k_q6k.t1", [] { schedule_mixed_format_swiglu_decode(GGML_TYPE_Q4_K, GGML_TYPE_Q6_K, 1, false); } }, + { "mixed_format_swiglu_decode.q4k_q6k.t2", [] { schedule_mixed_format_swiglu_decode(GGML_TYPE_Q4_K, GGML_TYPE_Q6_K, 2, true); } }, + { "mixed_format_swiglu_decode.q6k_q6k.t1", [] { schedule_mixed_format_swiglu_decode(GGML_TYPE_Q6_K, GGML_TYPE_Q6_K, 1, true); } }, { "packed_layout_follows_weight.q4_k.file_mapped.stored_copy", [] { schedule_packed_layout_follows_weight(GGML_TYPE_Q4_K, true, 100, true); } }, { "packed_layout_follows_weight.q6_k.file_mapped.stored_copy",