diff --git a/ggml/src/ggml-hrx/qwen4exp-hrx-support.h b/ggml/src/ggml-hrx/qwen4exp-hrx-support.h index 01b8dd3ebc09..1b0ae7a908e7 100644 --- a/ggml/src/ggml-hrx/qwen4exp-hrx-support.h +++ b/ggml/src/ggml-hrx/qwen4exp-hrx-support.h @@ -69,16 +69,40 @@ inline bool rope_mode_is_collapsible_imrope(const RopeParams & params) { return params.mode == GGML_ROPE_TYPE_IMROPE && imrope_collapses_to_neox(params, params.n_dims); } -// Expected positions ne0 for a supported ROPE node. IMROPE carries four position +// Non-interleaved M-RoPE (GGML_ROPE_TYPE_MROPE, ggml_rope_multi) whose every rotated pair reads p_t: in +// ggml_mrope_cache_init a pair takes theta_t when (pair % sect_dims) < sections[0], and the rotation pairs i with +// i + n_dims/2 like NEOX. So it is a NEOX rotation driven by p_t, the first token_count entries of the positions +// buffer, whatever p_h/p_w/p_e hold. DFlash drafts for M-RoPE targets get this mode (llama-model.cpp, "drafts +// for M-RoPE targets carry rope sections"): Qwen3.8-27B's DFlash2 has sections {64,0,0,0} at n_dims 128. +inline bool rope_mode_is_collapsible_mrope(const RopeParams & params) { + if (params.mode != GGML_ROPE_TYPE_MROPE || params.n_dims < 2) { + return false; + } + const int64_t sect_dims = static_cast(params.sections[0]) + params.sections[1] + params.sections[2] + + params.sections[3]; + if (params.sections[0] <= 0 || params.sections[1] < 0 || params.sections[2] < 0 || params.sections[3] < 0 || + sect_dims > params.n_dims) { + return false; + } + for (int64_t pair = 0; pair < params.n_dims / 2; ++pair) { + if (pair % sect_dims >= params.sections[0]) { + return false; // this pair reads p_h, p_w or p_e + } + } + return true; +} + +// Expected positions ne0 for a supported ROPE node. IMROPE and MROPE carry four position // sections of token_count each; the collapse path reads only the first (p_t). inline int64_t rope_expected_positions_ne0(const RopeParams & params, int64_t token_count) { - return params.mode == GGML_ROPE_TYPE_IMROPE ? token_count * 4 : token_count; + return params.mode == GGML_ROPE_TYPE_IMROPE || params.mode == GGML_ROPE_TYPE_MROPE ? token_count * 4 : token_count; } // Mode value handed to the rope_f32 / rope_set_rows_f32 kernels. A collapsible -// IMROPE runs the kernel's NEOX path. +// IMROPE or MROPE runs the kernel's NEOX path. inline int rope_kernel_mode(const RopeParams & params) { - return params.mode == GGML_ROPE_TYPE_IMROPE ? GGML_ROPE_TYPE_NEOX : params.mode; + return params.mode == GGML_ROPE_TYPE_IMROPE || params.mode == GGML_ROPE_TYPE_MROPE ? GGML_ROPE_TYPE_NEOX + : params.mode; } // Mode 3 of the generic rope_f32 / rope_set_rows_f32 kernels: an IMROPE whose sections leave some pairs on the fourth diff --git a/ggml/src/ggml-hrx/rope-claim-params.h b/ggml/src/ggml-hrx/rope-claim-params.h index f013f47a514b..b43ef06be52f 100644 --- a/ggml/src/ggml-hrx/rope-claim-params.h +++ b/ggml/src/ggml-hrx/rope-claim-params.h @@ -59,7 +59,8 @@ inline bool rope_params_supported_by_rope_set_rows(const RopeParams & params, in // the same n_offs guard as the attention predicate above: neither rope kernel on this line applies an offset if (params.n_offs != 0) { return false; } const bool supported_mode = params.mode == GGML_ROPE_TYPE_NORMAL || params.mode == GGML_ROPE_TYPE_NEOX || - rope_mode_is_collapsible_imrope(params) || rope_mode_is_sectioned_imrope(params); + rope_mode_is_collapsible_imrope(params) || rope_mode_is_sectioned_imrope(params) || + rope_mode_is_collapsible_mrope(params); return params.n_dims >= 4 && params.n_dims <= head_size && params.n_dims % 4 == 0 && supported_mode && std::isfinite(params.freq_base) && params.freq_base > 0.0f && std::isfinite(params.freq_scale) && params.freq_scale > 0.0f && std::isfinite(params.ext_factor) && std::isfinite(params.attn_factor); diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index e6874826c78e..de7005fd165e 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -6800,6 +6800,10 @@ struct test_rope : public test_case { } else { GGML_ASSERT(n_dims/3 > 0); int rope_sections[4] = {n_dims/3, n_dims/3, n_dims/3, 0}; + if (v & 8) { + // every rotated pair reads p_t: the sections of a DFlash draft for an M-RoPE target ({64,0,0,0} at n_dims 128) + rope_sections[0] = n_dims/2; rope_sections[1] = 0; rope_sections[2] = 0; + } if (forward) { if (inplace) { out = ggml_rope_multi_inplace(ctx, a, pos, freq, n_dims, rope_sections, mode, 0, 10000.0f, fs, ef, af, 1.0f, 1.0f); @@ -10756,6 +10760,12 @@ static std::vector> make_test_cases_eval() { } } test_cases.emplace_back(new test_rope_set_rows(GGML_TYPE_F32, GGML_TYPE_I32, { 128, 32, 8, 1 }, GGML_ROPE_TYPE_IMROPE)); + // M-RoPE whose sections leave every pair on p_t (Qwen3.8-27B DFlash2: {64,0,0,0} at n_dims 128, 32 q / 8 kv heads) + for (int64_t ne1 : {32, 8}) { + for (int64_t ne2 : {2, 512}) { + test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {128, ne1, ne2, 1}, 128, GGML_ROPE_TYPE_MROPE, 512, 1.0f, 0.0f, 1.0f, false, 8)); + } + } // partial rotation for (int mode : { GGML_ROPE_TYPE_NORMAL, GGML_ROPE_TYPE_NEOX, GGML_ROPE_TYPE_MROPE, GGML_ROPE_TYPE_IMROPE }) { for (ggml_type type : {GGML_TYPE_F16, GGML_TYPE_F32}) { diff --git a/tests/test-hrx-rope-claim-params.cpp b/tests/test-hrx-rope-claim-params.cpp index 0471d2225bbd..96c32feaf136 100644 --- a/tests/test-hrx-rope-claim-params.cpp +++ b/tests/test-hrx-rope-claim-params.cpp @@ -164,6 +164,44 @@ int main() { REQUIRE(rope_params_supported_by_any_fusion(params, 64)); } + { + // A DFlash draft for an M-RoPE target (Qwen3.8-27B DFlash2): mode MROPE, sections {64,0,0,0} at n_dims + // 128. Every rotated pair reads p_t, so it is NEOX on the first token_count positions and the generic + // rope kernel may run it; before, the claim refused all MROPE and strict placement aborted the load. + RopeParams params = text_rotation(128, 128); + params.mode = GGML_ROPE_TYPE_MROPE; + params.freq_base = 10000000.0f; + params.sections[0] = 64; + REQUIRE(rope_mode_is_collapsible_mrope(params)); + REQUIRE(rope_params_supported_by_rope_set_rows(params, 128)); + REQUIRE(rope_params_supported_by_any_fusion(params, 128)); + REQUIRE(rope_expected_positions_ne0(params, 2) == 8); + REQUIRE(rope_kernel_mode(params) == GGML_ROPE_TYPE_NEOX); + REQUIRE(rope_generic_kernel_mode(params) == GGML_ROPE_TYPE_NEOX); + } + { + // An MROPE whose sections send pairs to p_h/p_w (test-backend-ops' {n/3, n/3, n/3, 0}, or {32,32,0,0}: + // pairs 32..63 read p_h) is not a NEOX rotation, so it stays unclaimed. + RopeParams params = text_rotation(128, 128); + params.mode = GGML_ROPE_TYPE_MROPE; + params.sections[0] = 42; + params.sections[1] = 42; + params.sections[2] = 42; + REQUIRE(!rope_mode_is_collapsible_mrope(params)); + REQUIRE(!rope_params_supported_by_rope_set_rows(params, 128)); + params.sections[0] = 32; + params.sections[1] = 32; + params.sections[2] = 0; + REQUIRE(!rope_mode_is_collapsible_mrope(params)); + REQUIRE(!rope_params_supported_by_rope_set_rows(params, 128)); + // the vision rotation carries the MROPE bit (24 & 8) but is its own mode: still refused + params.mode = GGML_ROPE_TYPE_VISION; + params.sections[0] = 64; + params.sections[1] = 0; + REQUIRE(!rope_mode_is_collapsible_mrope(params)); + REQUIRE(!rope_params_supported_by_rope_set_rows(params, 128)); + } + // The regression proper: the claim must refuse the vision rotation and keep the text one. check_claim_for(/*vision*/ true, /*want_claimed*/ false); check_claim_for(/*vision*/ false, /*want_claimed*/ true);