From f3533d251fe52085e3d758a8f5e340c48639b64b Mon Sep 17 00:00:00 2001 From: Todor Boinovski Date: Sat, 15 Aug 2026 18:47:36 -0700 Subject: [PATCH 1/8] hexagon: vectorize RoPE theta cache on v75 --- ggml/src/ggml-hexagon/htp/hvx-sin-cos.h | 110 ++++++++--------- ggml/src/ggml-hexagon/htp/rope-ops.c | 149 ++++++++++++++---------- 2 files changed, 139 insertions(+), 120 deletions(-) diff --git a/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h b/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h index c5b9a5d47c17..8648af0e5b95 100644 --- a/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h +++ b/ggml/src/ggml-hexagon/htp/hvx-sin-cos.h @@ -4,87 +4,75 @@ #include "hvx-base.h" #include "hvx-floor.h" -static inline HVX_Vector hvx_vec_cos_f32(HVX_Vector x) { - HVX_Vector const_inv_pi = hvx_vec_splat_f32(0.3183098861837907f); - HVX_Vector const_half = hvx_vec_splat_f32(0.5f); - HVX_Vector const_pi = hvx_vec_splat_f32(3.141592653589793f); - HVX_Vector const_one = hvx_vec_splat_f32(1.0f); +// Range-reduce x to y in [-pi/2, pi/2] and the quadrant sign (-1)^n. +// Floor/truncate need IEEE bits, so convert qf32 back to sf before them. +static inline void hvx_vec_sincos_reduce_f32(HVX_Vector x, HVX_Vector * y, HVX_Vector * sign) { + HVX_Vector const_inv_pi = hvx_vec_splat_f32(0.3183098861837907f); + HVX_Vector const_half = hvx_vec_splat_f32(0.5f); + HVX_Vector const_pi = hvx_vec_splat_f32(3.141592653589793f); + HVX_Vector const_one = hvx_vec_splat_f32(1.0f); HVX_Vector const_neg_one = hvx_vec_splat_f32(-1.0f); + HVX_Vector const_one_i = Q6_V_vsplat_R(1); + + HVX_Vector x_over_pi = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(x, const_inv_pi)); + x_over_pi = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(x_over_pi, const_half)); - // n = floor(x * (1/pi) + 0.5) - HVX_Vector n_float = hvx_vec_floor_f32(hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(x, const_inv_pi), const_half)); + HVX_Vector n_float = hvx_vec_floor_f32(x_over_pi); + HVX_Vector n_int = hvx_vec_truncate_f32(n_float); - // y = x - n * pi - HVX_Vector y = hvx_vec_sub_f32_f32(x, hvx_vec_mul_f32_f32(n_float, const_pi)); + HVX_Vector n_pi = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(n_float, const_pi)); + *y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(x, n_pi)); - // Sign determination: if n is odd, sign is -1.0f, else 1.0f - // half_n = n * 0.5f - HVX_Vector half_n = hvx_vec_mul_f32_f32(n_float, const_half); - // floor_half_n = floor(half_n) - HVX_Vector floor_half_n = hvx_vec_floor_f32(half_n); - // is_odd = half_n > floor_half_n - HVX_VectorPred is_odd = Q6_Q_vcmp_gt_VsfVsf(half_n, floor_half_n); - // sign = vmux(is_odd, -1.0f, 1.0f) - HVX_Vector sign = Q6_V_vmux_QVV(is_odd, const_neg_one, const_one); + HVX_VectorPred is_odd = Q6_Q_vcmp_eq_VwVw(Q6_V_vand_VV(n_int, const_one_i), const_one_i); + *sign = Q6_V_vmux_QVV(is_odd, const_neg_one, const_one); +} - // z = y^2 - HVX_Vector z = hvx_vec_mul_f32_f32(y, y); +static inline void hvx_vec_sincos_f32(HVX_Vector x, HVX_Vector * vcos, HVX_Vector * vsin) { + HVX_Vector y; + HVX_Vector sign; + hvx_vec_sincos_reduce_f32(x, &y, &sign); + + HVX_Vector z = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(y, y)); - // Chebyshev approximation for cos(y) HVX_Vector c4 = hvx_vec_splat_f32(2.3557242013849433e-05f); HVX_Vector c3 = hvx_vec_splat_f32(-0.0013871428263450528f); HVX_Vector c2 = hvx_vec_splat_f32(0.041665895266688284f); HVX_Vector c1 = hvx_vec_splat_f32(-0.4999999360426369f); HVX_Vector c0 = hvx_vec_splat_f32(0.9999999999071725f); - HVX_Vector cos_y = hvx_vec_add_f32_f32(c3, hvx_vec_mul_f32_f32(z, c4)); - cos_y = hvx_vec_add_f32_f32(c2, hvx_vec_mul_f32_f32(z, cos_y)); - cos_y = hvx_vec_add_f32_f32(c1, hvx_vec_mul_f32_f32(z, cos_y)); - cos_y = hvx_vec_add_f32_f32(c0, hvx_vec_mul_f32_f32(z, cos_y)); - - return hvx_vec_mul_f32_f32(cos_y, sign); -} - -static inline HVX_Vector hvx_vec_sin_f32(HVX_Vector x) { - HVX_Vector const_inv_pi = hvx_vec_splat_f32(0.3183098861837907f); - HVX_Vector const_half = hvx_vec_splat_f32(0.5f); - HVX_Vector const_pi = hvx_vec_splat_f32(3.141592653589793f); - HVX_Vector const_one = hvx_vec_splat_f32(1.0f); - HVX_Vector const_neg_one = hvx_vec_splat_f32(-1.0f); - - // n = floor(x * (1/pi) + 0.5) - HVX_Vector n_float = hvx_vec_floor_f32(hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(x, const_inv_pi), const_half)); + HVX_Vector cos_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(c3, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, c4)))); + cos_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(c2, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, cos_y)))); + cos_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(c1, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, cos_y)))); + cos_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(c0, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, cos_y)))); - // y = x - n * pi - HVX_Vector y = hvx_vec_sub_f32_f32(x, hvx_vec_mul_f32_f32(n_float, const_pi)); - - // Sign determination: if n is odd, sign is -1.0f, else 1.0f - // half_n = n * 0.5f - HVX_Vector half_n = hvx_vec_mul_f32_f32(n_float, const_half); - // floor_half_n = floor(half_n) - HVX_Vector floor_half_n = hvx_vec_floor_f32(half_n); - // is_odd = half_n > floor_half_n - HVX_VectorPred is_odd = Q6_Q_vcmp_gt_VsfVsf(half_n, floor_half_n); - // sign = vmux(is_odd, -1.0f, 1.0f) - HVX_Vector sign = Q6_V_vmux_QVV(is_odd, const_neg_one, const_one); - - // z = y^2 - HVX_Vector z = hvx_vec_mul_f32_f32(y, y); - - // Chebyshev approximation for sin(y) HVX_Vector s4 = hvx_vec_splat_f32(2.642186986152672e-06f); HVX_Vector s3 = hvx_vec_splat_f32(-0.00019825318964070864f); HVX_Vector s2 = hvx_vec_splat_f32(0.00833326283319605f); HVX_Vector s1 = hvx_vec_splat_f32(-0.16666666082087775f); HVX_Vector s0 = hvx_vec_splat_f32(0.999999999915155f); - HVX_Vector sin_y = hvx_vec_add_f32_f32(s3, hvx_vec_mul_f32_f32(z, s4)); - sin_y = hvx_vec_add_f32_f32(s2, hvx_vec_mul_f32_f32(z, sin_y)); - sin_y = hvx_vec_add_f32_f32(s1, hvx_vec_mul_f32_f32(z, sin_y)); - sin_y = hvx_vec_add_f32_f32(s0, hvx_vec_mul_f32_f32(z, sin_y)); - sin_y = hvx_vec_mul_f32_f32(y, sin_y); + HVX_Vector sin_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(s3, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, s4)))); + sin_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(s2, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, sin_y)))); + sin_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(s1, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, sin_y)))); + sin_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(s0, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, sin_y)))); + sin_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(y, sin_y)); + + *vcos = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(cos_y, sign)); + *vsin = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(sin_y, sign)); +} - return hvx_vec_mul_f32_f32(sin_y, sign); +static inline HVX_Vector hvx_vec_cos_f32(HVX_Vector x) { + HVX_Vector vcos; + HVX_Vector vsin; + hvx_vec_sincos_f32(x, &vcos, &vsin); + return vcos; +} + +static inline HVX_Vector hvx_vec_sin_f32(HVX_Vector x) { + HVX_Vector vcos; + HVX_Vector vsin; + hvx_vec_sincos_f32(x, &vcos, &vsin); + return vsin; } #endif /* HVX_SIN_COS_H */ diff --git a/ggml/src/ggml-hexagon/htp/rope-ops.c b/ggml/src/ggml-hexagon/htp/rope-ops.c index 6c689824934f..86cecc6fa0a1 100644 --- a/ggml/src/ggml-hexagon/htp/rope-ops.c +++ b/ggml/src/ggml-hexagon/htp/rope-ops.c @@ -116,26 +116,48 @@ static inline void rope_yarn_one(float theta, float freq_scale, float * corr_dim cache[i0 + 1] = sinf(theta_final) * mscale_final; } +// 32 thetas -> 32 interleaved (cos, sin) pairs at cache[i0]. +static inline void rope_cache_hvx_32(float * cache, uint32_t i0, + HVX_Vector v_theta, + const float * freq_factors, + HVX_Vector v_freq_scale, + HVX_Vector v_mscale) { + if (freq_factors) { + HVX_Vector v_ff = hvx_vmemu(freq_factors + i0 / 2); + v_theta = hvx_vec_mul_f32_f32(v_theta, hvx_vec_inverse_f32(v_ff)); + } + + HVX_Vector v_theta_final = hvx_vec_mul_f32_f32(v_theta, v_freq_scale); + HVX_Vector vcos; + HVX_Vector vsin; + hvx_vec_sincos_f32(v_theta_final, &vcos, &vsin); + vcos = hvx_vec_mul_f32_f32(vcos, v_mscale); + vsin = hvx_vec_mul_f32_f32(vsin, v_mscale); + HVX_VectorPair vstore = Q6_W_vshuff_VVR(vsin, vcos, -4); + + if (((uintptr_t) (cache + i0)) % 128 == 0) { + hvx_vmem(cache + i0 + 0) = Q6_V_lo_W(vstore); + hvx_vmem(cache + i0 + 32) = Q6_V_hi_W(vstore); + } else { + hvx_vec_store_u(cache + i0 + 0, 32 * sizeof(float), Q6_V_lo_W(vstore)); + hvx_vec_store_u(cache + i0 + 32, 32 * sizeof(float), Q6_V_hi_W(vstore)); + } +} + static __attribute__((noinline)) void rope_cache_init(const float theta_base, const float freq_scale, const float * freq_factors, float * corr_dims, - const uint32_t ne0, + const uint32_t n_cache, const float ext_factor, const float mscale, float * cache, const float theta_scale) { // ref: https://github.com/jquesnelle/yarn/blob/master/scaled_rope/LlamaYaRNScaledRotaryEmbedding.py -#if __HVX_ARCH__ >= 79 - const bool is_v79_or_newer = true; -#else - const bool is_v79_or_newer = false; -#endif - - if (is_v79_or_newer && ext_factor == 0.0f) { + if (ext_factor == 0.0f) { // Fast path: fully vectorized // We process 32 pairs (64 elements) per iteration. - const uint32_t n_blocks = ne0 / 64; + const uint32_t n_blocks = n_cache / 64; // Initialize theta scale powers: [1.0f, theta_scale, theta_scale^2, ..., theta_scale^31] float __attribute__((aligned(128))) theta_powers[32]; @@ -143,7 +165,7 @@ static __attribute__((noinline)) void rope_cache_init(const float theta_base, for (int j = 1; j < 32; j++) { theta_powers[j] = theta_powers[j - 1] * theta_scale; } - HVX_Vector v_theta_powers = hvx_vmem(theta_powers); + HVX_Vector v_theta_powers = hvx_vmemu(theta_powers); HVX_Vector v_freq_scale = hvx_vec_splat_f32(freq_scale); HVX_Vector v_mscale = hvx_vec_splat_f32(mscale); @@ -160,46 +182,20 @@ static __attribute__((noinline)) void rope_cache_init(const float theta_base, uint32_t i0 = b * 64; HVX_Vector v_theta_base = hvx_vec_splat_f32(theta_block); HVX_Vector v_theta = hvx_vec_mul_f32_f32(v_theta_base, v_theta_powers); - - if (freq_factors) { - // Load 32 elements of freq_factors - HVX_Vector v_ff = hvx_vmemu(freq_factors + i0 / 2); - HVX_Vector v_inv_ff = hvx_vec_inverse_f32(v_ff); - v_theta = hvx_vec_mul_f32_f32(v_theta, v_inv_ff); - } - - HVX_Vector v_theta_final = hvx_vec_mul_f32_f32(v_theta, v_freq_scale); - - HVX_Vector vcos = hvx_vec_cos_f32(v_theta_final); - HVX_Vector vsin = hvx_vec_sin_f32(v_theta_final); - - vcos = hvx_vec_mul_f32_f32(vcos, v_mscale); - vsin = hvx_vec_mul_f32_f32(vsin, v_mscale); - - HVX_VectorPair vstore = Q6_W_vshuff_VVR(vsin, vcos, -4); - - if (((uintptr_t)cache) % 128 == 0) { - hvx_vmem(cache + i0 + 0) = Q6_V_lo_W(vstore); - hvx_vmem(cache + i0 + 32) = Q6_V_hi_W(vstore); - } else { - hvx_vec_store_u(cache + i0 + 0, 32 * sizeof(float), Q6_V_lo_W(vstore)); - hvx_vec_store_u(cache + i0 + 32, 32 * sizeof(float), Q6_V_hi_W(vstore)); - } - + rope_cache_hvx_32(cache, i0, v_theta, freq_factors, v_freq_scale, v_mscale); theta_block *= theta_scale_32; } // Leftovers float theta = theta_block; - for (uint32_t i0 = n_blocks * 64; i0 < ne0; i0 += 2) { + for (uint32_t i0 = n_blocks * 64; i0 < n_cache; i0 += 2) { const float ff = freq_factors ? freq_factors[i0 / 2] : 1.0f; rope_yarn_one(theta / ff, freq_scale, corr_dims, i0, ext_factor, mscale, cache); theta *= theta_scale; } } else { - // Fallback to original scalar loop float theta = theta_base; - for (uint32_t i0 = 0; i0 < ne0; i0 += 2) { + for (uint32_t i0 = 0; i0 < n_cache; i0 += 2) { const float ff = freq_factors ? freq_factors[i0 / 2] : 1.0f; rope_yarn_one(theta / ff, freq_scale, corr_dims, i0, ext_factor, mscale, cache); theta *= theta_scale; @@ -207,6 +203,21 @@ static __attribute__((noinline)) void rope_cache_init(const float theta_base, } } +static inline float mrope_pick_theta(float theta_t, float theta_h, float theta_w, float theta_e, + int sector, const int32_t sections[4], int sec_w, int sec_e, + bool is_imrope) { + if (is_imrope) { + if (sector % 3 == 0 && sector < 3 * sections[0]) { return theta_t; } + else if (sector % 3 == 1 && sector < 3 * sections[1]) { return theta_h; } + else if (sector % 3 == 2 && sector < 3 * sections[2]) { return theta_w; } + else { return theta_e; } + } + if (sector < sections[0]) { return theta_t; } + else if (sector < sec_w) { return theta_h; } + else if (sector < sec_e) { return theta_w; } + else { return theta_e; } +} + // pos_t/h/w/e: the four position ids for this sequence step (t=time, h=height, w=width, e=extra). // sections[4]: number of head dims assigned to each position component. static __attribute__((noinline)) void mrope_cache_init(const float pos_t, @@ -219,7 +230,7 @@ static __attribute__((noinline)) void mrope_cache_init(const float pos_t, const float freq_scale, const float * freq_factors, float * corr_dims, - const uint32_t ne0, + const uint32_t n_cache, const float ext_factor, const float mscale, float * cache, @@ -233,9 +244,16 @@ static __attribute__((noinline)) void mrope_cache_init(const float pos_t, float theta_w = pos_w; float theta_e = pos_e; - for (uint32_t i0 = 0; i0 < ne0; i0 += 2) { - const float ff = freq_factors ? freq_factors[i0 / 2] : 1.0f; - const int sector = (i0 / 2) % sect_dims; + const bool use_hvx = (ext_factor == 0.0f); + float __attribute__((aligned(128))) thetas[32]; + uint32_t n_thetas = 0; + uint32_t block_i0 = 0; + + HVX_Vector v_freq_scale = hvx_vec_splat_f32(freq_scale); + HVX_Vector v_mscale = hvx_vec_splat_f32(mscale); + + for (uint32_t i0 = 0; i0 < n_cache; i0 += 2) { + const int sector = (i0 / 2) % sect_dims; if (indep_sects) { // Reset theta when crossing into a new section. @@ -245,28 +263,34 @@ static __attribute__((noinline)) void mrope_cache_init(const float pos_t, else if (sector == sec_e) { theta_e = pos_e; } } - float theta; - if (is_imrope) { - // Interleaved: sector mod 3 selects component - if (sector % 3 == 0 && sector < 3 * sections[0]) { theta = theta_t; } - else if (sector % 3 == 1 && sector < 3 * sections[1]) { theta = theta_h; } - else if (sector % 3 == 2 && sector < 3 * sections[2]) { theta = theta_w; } - else { theta = theta_e; } + const float theta = mrope_pick_theta(theta_t, theta_h, theta_w, theta_e, + sector, sections, sec_w, sec_e, is_imrope); + + if (use_hvx) { + if (n_thetas == 0) { + block_i0 = i0; + } + thetas[n_thetas++] = theta; + if (n_thetas == 32) { + rope_cache_hvx_32(cache, block_i0, hvx_vmemu(thetas), freq_factors, v_freq_scale, v_mscale); + n_thetas = 0; + } } else { - // Contiguous sections - if (sector < sections[0]) { theta = theta_t; } - else if (sector < sec_w) { theta = theta_h; } - else if (sector < sec_e) { theta = theta_w; } - else { theta = theta_e; } + const float ff = freq_factors ? freq_factors[i0 / 2] : 1.0f; + rope_yarn_one(theta / ff, freq_scale, corr_dims, i0, ext_factor, mscale, cache); } - rope_yarn_one(theta / ff, freq_scale, corr_dims, i0, ext_factor, mscale, cache); - theta_t *= theta_scale; theta_h *= theta_scale; theta_w *= theta_scale; theta_e *= theta_scale; } + + for (uint32_t k = 0; k < n_thetas; k++) { + const uint32_t i0 = block_i0 + 2 * k; + const float ff = freq_factors ? freq_factors[i0 / 2] : 1.0f; + rope_yarn_one(thetas[k] / ff, freq_scale, corr_dims, i0, ext_factor, mscale, cache); + } } #define M_PI 3.1415926535897932384626433 @@ -491,6 +515,7 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { uint8_t * dst_spad_base = octx->dst_spad.data + (ith * octx->dst_spad.size_per_thread); dma_queue * dma_queue = octx->ctx->dma[ith]; + struct htp_thread_trace * tr = &octx->ctx->trace[ith]; const int32_t * pos = (const int32_t *) src1->data; const float * freq_factors = src2 ? (const float *) src2->data : NULL; @@ -542,6 +567,9 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { if (i2 != prev_i2) { prev_i2 = i2; + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_A_PREP, i2); + // VISION rotates the full row; other modes only rotate n_dims. + const uint32_t n_cache = is_vision ? ne0 : (uint32_t) rctx->n_dims; const bool is_mrope = (rctx->mode & HTP_ROPE_TYPE_MROPE) != 0; if (is_mrope) { // src1 holds four position arrays stacked along ne0: @@ -554,13 +582,14 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { (float) pos[i2 + ne2 * 3], rctx->sections, is_imrope, is_vision, rctx->freq_scale, freq_factors, rctx->corr_dims, - ne0, rctx->ext_factor, rctx->attn_factor, + n_cache, rctx->ext_factor, rctx->attn_factor, theta_cache, rctx->theta_scale); } else { rope_cache_init(pos[i2], rctx->freq_scale, freq_factors, rctx->corr_dims, - ne0, rctx->ext_factor, rctx->attn_factor, + n_cache, rctx->ext_factor, rctx->attn_factor, theta_cache, rctx->theta_scale); } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, i2); } // Skip output DMA transactions from prev block (if any) @@ -578,6 +607,7 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { // FARF(HIGH, "rope-compute %u: ir %u i1 %u i2 %u i3 %u src-spad %p cnr %u : usec %u", ith, ir, i1, i2, i3, src_spad, cnr, // (unsigned) HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - rctx->t_start)); + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir); if (is_vision) { rope_vision_f32(rctx, dst_spad, src_spad, cnr, ne0, theta_cache); } else if (is_neox) { @@ -585,6 +615,7 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { } else { rope_basic_f32(rctx, dst_spad, src_spad, cnr, ne0, theta_cache); } + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir); uint8_t * dst_addr = (uint8_t *) dst->data + i3 * nb3 + i2 * nb2 + i1 * nb1; From 9732ab37aa448c370026aacd839c6325a15ed5a8 Mon Sep 17 00:00:00 2001 From: Todor Boinovski Date: Sat, 15 Aug 2026 21:48:59 -0700 Subject: [PATCH 2/8] hexagon: vectorize MROPE/IMROPE theta pick --- ggml/src/ggml-hexagon/htp/rope-ops.c | 122 +++++++++++++++++++++++---- 1 file changed, 105 insertions(+), 17 deletions(-) diff --git a/ggml/src/ggml-hexagon/htp/rope-ops.c b/ggml/src/ggml-hexagon/htp/rope-ops.c index 86cecc6fa0a1..8f823a2ed4d6 100644 --- a/ggml/src/ggml-hexagon/htp/rope-ops.c +++ b/ggml/src/ggml-hexagon/htp/rope-ops.c @@ -65,6 +65,8 @@ struct htp_rope_context { float beta_fast; float beta_slow; float theta_scale; + float theta_scale_32; + float theta_powers[32]; float corr_dims[2]; uint32_t src0_nrows_per_thread; @@ -152,31 +154,20 @@ static __attribute__((noinline)) void rope_cache_init(const float theta_base, const float ext_factor, const float mscale, float * cache, - const float theta_scale) { + const float theta_scale, + const float * theta_powers, + const float theta_scale_32) { // ref: https://github.com/jquesnelle/yarn/blob/master/scaled_rope/LlamaYaRNScaledRotaryEmbedding.py if (ext_factor == 0.0f) { // Fast path: fully vectorized // We process 32 pairs (64 elements) per iteration. const uint32_t n_blocks = n_cache / 64; - // Initialize theta scale powers: [1.0f, theta_scale, theta_scale^2, ..., theta_scale^31] - float __attribute__((aligned(128))) theta_powers[32]; - theta_powers[0] = 1.0f; - for (int j = 1; j < 32; j++) { - theta_powers[j] = theta_powers[j - 1] * theta_scale; - } HVX_Vector v_theta_powers = hvx_vmemu(theta_powers); - HVX_Vector v_freq_scale = hvx_vec_splat_f32(freq_scale); HVX_Vector v_mscale = hvx_vec_splat_f32(mscale); - // Base theta starts at theta_base float theta_block = theta_base; - // The scale factor for the next block is theta_scale^32 - float theta_scale_32 = 1.0f; - for (int j = 0; j < 32; j++) { - theta_scale_32 *= theta_scale; - } for (uint32_t b = 0; b < n_blocks; b++) { uint32_t i0 = b * 64; @@ -218,6 +209,57 @@ static inline float mrope_pick_theta(float theta_t, float theta_h, float theta_w else { return theta_e; } } +// lane j is 1 when (j % 3) == rem +static const float __attribute__((aligned(128))) mrope_mod3_eq0[32] = { + 1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0 +}; +static const float __attribute__((aligned(128))) mrope_mod3_eq1[32] = { + 0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1 +}; +static const float __attribute__((aligned(128))) mrope_mod3_eq2[32] = { + 0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0,1,0,0 +}; + +static const float __attribute__((aligned(128))) mrope_k_ramp[32] = { + 0,1,2,3,4,5,6,7,8,9,10,11,12,13,14,15, + 16,17,18,19,20,21,22,23,24,25,26,27,28,29,30,31 +}; + +static inline HVX_VectorPred mrope_mask_eq1(const float * m) { + return Q6_Q_vcmp_gt_VsfVsf(hvx_vmemu(m), Q6_V_vzero()); +} + +// IMROPE without wrap: theta[k] = pos[k % 3] * scale^k +static inline HVX_Vector mrope_thetas_imrope_mod3(float pos_t, float pos_h, float pos_w, + uint32_t k0, HVX_Vector v_powers, float scale_block) { + const int r = (int) (k0 % 3); + const float * mt = (r == 0) ? mrope_mod3_eq0 : (r == 1) ? mrope_mod3_eq2 : mrope_mod3_eq1; + const float * mh = (r == 0) ? mrope_mod3_eq1 : (r == 1) ? mrope_mod3_eq0 : mrope_mod3_eq2; + + HVX_Vector v = hvx_vec_splat_f32(pos_w); + v = Q6_V_vmux_QVV(mrope_mask_eq1(mh), hvx_vec_splat_f32(pos_h), v); + v = Q6_V_vmux_QVV(mrope_mask_eq1(mt), hvx_vec_splat_f32(pos_t), v); + v = hvx_vec_mul_f32_f32(v, v_powers); + return hvx_vec_mul_f32_f32(v, hvx_vec_splat_f32(scale_block)); +} + +// Contiguous MROPE without wrap: theta[k] = pos[section(k)] * scale^k +static inline HVX_Vector mrope_thetas_contig(float pos_t, float pos_h, float pos_w, float pos_e, + uint32_t k0, int s0, int sec_w, int sec_e, + HVX_Vector v_powers, float scale_block) { + HVX_Vector v_k = hvx_vec_add_f32_f32(hvx_vec_splat_f32((float) k0), hvx_vmemu(mrope_k_ramp)); + HVX_VectorPred lt_s0 = Q6_Q_vcmp_gt_VsfVsf(hvx_vec_splat_f32((float) s0), v_k); + HVX_VectorPred lt_sw = Q6_Q_vcmp_gt_VsfVsf(hvx_vec_splat_f32((float) sec_w), v_k); + HVX_VectorPred lt_se = Q6_Q_vcmp_gt_VsfVsf(hvx_vec_splat_f32((float) sec_e), v_k); + + HVX_Vector v = hvx_vec_splat_f32(pos_e); + v = Q6_V_vmux_QVV(lt_se, hvx_vec_splat_f32(pos_w), v); + v = Q6_V_vmux_QVV(lt_sw, hvx_vec_splat_f32(pos_h), v); + v = Q6_V_vmux_QVV(lt_s0, hvx_vec_splat_f32(pos_t), v); + v = hvx_vec_mul_f32_f32(v, v_powers); + return hvx_vec_mul_f32_f32(v, hvx_vec_splat_f32(scale_block)); +} + // pos_t/h/w/e: the four position ids for this sequence step (t=time, h=height, w=width, e=extra). // sections[4]: number of head dims assigned to each position component. static __attribute__((noinline)) void mrope_cache_init(const float pos_t, @@ -234,10 +276,51 @@ static __attribute__((noinline)) void mrope_cache_init(const float pos_t, const float ext_factor, const float mscale, float * cache, - const float theta_scale) { + const float theta_scale, + const float * theta_powers, + const float theta_scale_32) { const int sect_dims = sections[0] + sections[1] + sections[2] + sections[3]; const int sec_w = sections[0] + sections[1]; const int sec_e = sec_w + sections[2]; + const uint32_t n_pairs = n_cache / 2; + + const bool no_wrap = (sect_dims > 0) && (n_pairs <= (uint32_t) sect_dims); + const bool imrope_mod3 = is_imrope && !indep_sects && no_wrap + && sections[0] > 0 && sections[1] > 0 && sections[2] > 0 + && n_pairs <= (uint32_t) (3 * sections[0]) + && n_pairs <= (uint32_t) (3 * sections[1]) + && n_pairs <= (uint32_t) (3 * sections[2]); + const bool contig = !is_imrope && !indep_sects && no_wrap; + + if (ext_factor == 0.0f && (imrope_mod3 || contig)) { + HVX_Vector v_powers = hvx_vmemu(theta_powers); + HVX_Vector v_freq_scale = hvx_vec_splat_f32(freq_scale); + HVX_Vector v_mscale = hvx_vec_splat_f32(mscale); + float scale_block = 1.0f; + const uint32_t n_blocks = n_cache / 64; + + for (uint32_t b = 0; b < n_blocks; b++) { + const uint32_t i0 = b * 64; + const uint32_t k0 = b * 32; + HVX_Vector v_theta = imrope_mod3 + ? mrope_thetas_imrope_mod3(pos_t, pos_h, pos_w, k0, v_powers, scale_block) + : mrope_thetas_contig(pos_t, pos_h, pos_w, pos_e, k0, sections[0], sec_w, sec_e, + v_powers, scale_block); + rope_cache_hvx_32(cache, i0, v_theta, freq_factors, v_freq_scale, v_mscale); + scale_block *= theta_scale_32; + } + + float theta_k = scale_block; + for (uint32_t k = n_blocks * 32; k < n_pairs; k++) { + const uint32_t i0 = 2 * k; + const float pos = mrope_pick_theta(pos_t, pos_h, pos_w, pos_e, + (int) k, sections, sec_w, sec_e, is_imrope); + const float ff = freq_factors ? freq_factors[k] : 1.0f; + rope_yarn_one(pos * theta_k / ff, freq_scale, corr_dims, i0, ext_factor, mscale, cache); + theta_k *= theta_scale; + } + return; + } float theta_t = pos_t; float theta_h = pos_h; @@ -583,11 +666,11 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { rctx->sections, is_imrope, is_vision, rctx->freq_scale, freq_factors, rctx->corr_dims, n_cache, rctx->ext_factor, rctx->attn_factor, - theta_cache, rctx->theta_scale); + theta_cache, rctx->theta_scale, rctx->theta_powers, rctx->theta_scale_32); } else { rope_cache_init(pos[i2], rctx->freq_scale, freq_factors, rctx->corr_dims, n_cache, rctx->ext_factor, rctx->attn_factor, - theta_cache, rctx->theta_scale); + theta_cache, rctx->theta_scale, rctx->theta_powers, rctx->theta_scale_32); } htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, i2); } @@ -724,6 +807,11 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) { memcpy(&rctx.sections, (int32_t *) op_params + 11, sizeof(int) * 4); rctx.theta_scale = powf(rctx.freq_base, -2.0f / rctx.n_dims); + rctx.theta_powers[0] = 1.0f; + for (int j = 1; j < 32; j++) { + rctx.theta_powers[j] = rctx.theta_powers[j - 1] * rctx.theta_scale; + } + rctx.theta_scale_32 = rctx.theta_powers[31] * rctx.theta_scale; rope_corr_dims(rctx.n_dims, rctx.n_ctx_orig, rctx.freq_base, rctx.beta_fast, rctx.beta_slow, rctx.corr_dims); From ee634ee6b9d59875fd8034f59aece7f8aeff4a86 Mon Sep 17 00:00:00 2001 From: Todor Boinovski Date: Fri, 28 Aug 2026 12:36:44 -0700 Subject: [PATCH 3/8] hexagon: tighten NEOX RoPE rotate and aligned tail copy --- ggml/src/ggml-hexagon/htp/dma-queue.h | 12 +- ggml/src/ggml-hexagon/htp/rope-ops.c | 172 ++++++++++++++++---------- tests/test-backend-ops.cpp | 13 ++ 3 files changed, 127 insertions(+), 70 deletions(-) diff --git a/ggml/src/ggml-hexagon/htp/dma-queue.h b/ggml/src/ggml-hexagon/htp/dma-queue.h index 190ca3a9b9e1..03e4d024c988 100644 --- a/ggml/src/ggml-hexagon/htp/dma-queue.h +++ b/ggml/src/ggml-hexagon/htp/dma-queue.h @@ -185,7 +185,7 @@ static inline bool dma_queue_push_single_1d(dma_queue * q, dma_ptr dptr, size_t return true; } -static inline bool dma_queue_push_single_2d(dma_queue * q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { +static inline bool dma_queue_push_single_2d_order(dma_queue * q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows, unsigned order) { dma_ring * r = q->ring; if (((r->push_idx + 1) & r->idx_mask) == r->pop_idx) { return false; @@ -201,7 +201,7 @@ static inline bool dma_queue_push_single_2d(dma_queue * q, dma_ptr dptr, size_t desc->dst_bypass = dma_is_vtcm(q, dptr.dst) ? 1 : q->nocache; desc->src_comp = 0; desc->dst_comp = 0; - desc->order = 0; + desc->order = order ? 1 : 0; desc->done = 0; desc->src_stride = src_stride; desc->dst_stride = dst_stride; @@ -236,6 +236,14 @@ static inline bool dma_queue_push_single_2d(dma_queue * q, dma_ptr dptr, size_t return true; } +static inline bool dma_queue_push_single_2d(dma_queue * q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { + return dma_queue_push_single_2d_order(q, dptr, dst_stride, src_stride, row_size, nrows, 0); +} + +static inline bool dma_queue_push_single_2d_ordered(dma_queue * q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { + return dma_queue_push_single_2d_order(q, dptr, dst_stride, src_stride, row_size, nrows, 1); +} + static inline dma_ptr dma_queue_pop(dma_queue * q) { dma_ring * r = q->ring; dma_ptr dptr = { NULL }; diff --git a/ggml/src/ggml-hexagon/htp/rope-ops.c b/ggml/src/ggml-hexagon/htp/rope-ops.c index 8f823a2ed4d6..f528ad8040fd 100644 --- a/ggml/src/ggml-hexagon/htp/rope-ops.c +++ b/ggml/src/ggml-hexagon/htp/rope-ops.c @@ -390,52 +390,67 @@ static void rope_corr_dims(int n_dims, dims[1] = MIN(n_dims - 1, end); } +// Inverse of the (cos, sin) interleave stored by rope_cache_hvx_32. +// Full 32-pair blocks become [cos[32] | sin[32]]. Leftover pairs stay interleaved. +static inline void rope_cache_deinterleave(float * cache, uint32_t n_cache) { + const uint32_t n_blocks = n_cache / 64; + for (uint32_t b = 0; b < n_blocks; b++) { + HVX_Vector * v = (HVX_Vector *) (cache + b * 64); + HVX_VectorPair cs = Q6_W_vdeal_VVR(v[1], v[0], -4); + v[0] = Q6_V_lo_W(cs); + v[1] = Q6_V_hi_W(cs); + } +} + +static inline void hvx_rope_neox_mul(HVX_Vector v0, HVX_Vector v1, HVX_Vector vcos, HVX_Vector vsin, + HVX_Vector * o0, HVX_Vector * o1) { + HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(v0, vcos); + HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(v0, vsin); + HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(v1, vcos); + HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(v1, vsin); + *o0 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_Vqf32Vqf32(vx0_c, vx1_s)); + *o1 = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_Vqf32Vqf32(vx0_s, vx1_c)); +} + +// theta_cache full 32-pair blocks are deinterleaved [cos | sin]. static inline void hvx_rope_neox_f32_aa(float * restrict dst, const float * restrict src0, uint32_t ne, const float * restrict theta_cache) { const uint32_t he = ne / 2; const uint32_t nvec = he / 32; const uint32_t nloe = he % 32; - for (uint32_t i = 0; i < nvec; i++) { - HVX_Vector v0 = ((const HVX_Vector *) src0)[i]; - HVX_Vector v1 = hvx_vmemu(src0 + he + i * 32); - - HVX_Vector v2 = ((const HVX_Vector *) theta_cache)[i * 2 + 0]; - HVX_Vector v3 = ((const HVX_Vector *) theta_cache)[i * 2 + 1]; - - HVX_VectorPair vcos_sin = Q6_W_vdeal_VVR(v3, v2, -4); - - HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(v0, Q6_V_lo_W(vcos_sin)); - HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(v0, Q6_V_hi_W(vcos_sin)); - HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(v1, Q6_V_lo_W(vcos_sin)); - HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(v1, Q6_V_hi_W(vcos_sin)); - - HVX_Vector v4 = Q6_Vqf32_vsub_Vqf32Vqf32(vx0_c, vx1_s); - HVX_Vector v5 = Q6_Vqf32_vadd_Vqf32Vqf32(vx0_s, vx1_c); - - ((HVX_Vector *) dst)[i] = Q6_Vsf_equals_Vqf32(v4); - hvx_vmemu(dst + he + i * 32) = Q6_Vsf_equals_Vqf32(v5); + if (nloe == 0) { + const HVX_Vector * vs = (const HVX_Vector *) src0; + const HVX_Vector * vt = (const HVX_Vector *) theta_cache; + HVX_Vector * vd = (HVX_Vector *) dst; + for (uint32_t i = 0; i < nvec; i++) { + HVX_Vector o0, o1; + hvx_rope_neox_mul(vs[i], vs[nvec + i], vt[i * 2 + 0], vt[i * 2 + 1], &o0, &o1); + vd[i] = o0; + vd[nvec + i] = o1; + } + return; } - if (nloe > 0) { - HVX_Vector v0 = hvx_vmemu(src0 + nvec * 32); - HVX_Vector v1 = hvx_vmemu(src0 + he + nvec * 32); - - HVX_Vector v2 = ((const HVX_Vector *) theta_cache)[nvec * 2 + 0]; - HVX_Vector v3 = ((const HVX_Vector *) theta_cache)[nvec * 2 + 1]; - - HVX_VectorPair vcos_sin = Q6_W_vdeal_VVR(v3, v2, -4); - - HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(v0, Q6_V_lo_W(vcos_sin)); - HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(v0, Q6_V_hi_W(vcos_sin)); - HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(v1, Q6_V_lo_W(vcos_sin)); - HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(v1, Q6_V_hi_W(vcos_sin)); - - HVX_Vector v4 = Q6_Vqf32_vsub_Vqf32Vqf32(vx0_c, vx1_s); - HVX_Vector v5 = Q6_Vqf32_vadd_Vqf32Vqf32(vx0_s, vx1_c); - - hvx_vec_store_u(dst + nvec * 32, nloe * sizeof(float), Q6_Vsf_equals_Vqf32(v4)); - hvx_vec_store_u(dst + he + nvec * 32, nloe * sizeof(float), Q6_Vsf_equals_Vqf32(v5)); + for (uint32_t i = 0; i < nvec; i++) { + HVX_Vector o0, o1; + hvx_rope_neox_mul(((const HVX_Vector *) src0)[i], + hvx_vmemu(src0 + he + i * 32), + ((const HVX_Vector *) theta_cache)[i * 2 + 0], + ((const HVX_Vector *) theta_cache)[i * 2 + 1], + &o0, &o1); + ((HVX_Vector *) dst)[i] = o0; + hvx_vmemu(dst + he + i * 32) = o1; } + + HVX_Vector v0 = hvx_vmemu(src0 + nvec * 32); + HVX_Vector v1 = hvx_vmemu(src0 + he + nvec * 32); + HVX_Vector v2 = hvx_vmemu(theta_cache + nvec * 64); + HVX_Vector v3 = hvx_vmemu(theta_cache + nvec * 64 + 32); + HVX_VectorPair vcos_sin = Q6_W_vdeal_VVR(v3, v2, -4); + HVX_Vector o0, o1; + hvx_rope_neox_mul(v0, v1, Q6_V_lo_W(vcos_sin), Q6_V_hi_W(vcos_sin), &o0, &o1); + hvx_vec_store_u(dst + nvec * 32, nloe * sizeof(float), o0); + hvx_vec_store_u(dst + he + nvec * 32, nloe * sizeof(float), o1); } static inline void hvx_rope_f32_aa(float * restrict dst, const float * restrict src0, uint32_t ne, const float * restrict theta_cache) { @@ -512,42 +527,47 @@ static inline void hvx_rope_f32_aa(float * restrict dst, const float * restrict } static void inline rope_basic_f32(struct htp_rope_context * rctx, uint8_t * restrict dst, uint8_t * restrict src, - uint32_t nr, uint32_t ne0, const float * restrict theta_cache) { - const uint32_t n_offs = rctx->n_offs; // VLEN-aligned (enforced by supports_op) + uint32_t nr, const float * restrict theta_cache) { + // Only reached when there is no pass-through region (n_dims == ne0, hence n_offs == 0); + // the n_offs>0 / partial-n_dims cases go through the *_inplace path (see rope_job_f32). #pragma unroll(4) for (uint32_t i = 0; i < nr; i++) { float * d = (float *) (dst + i * rctx->dst_row_size_aligned); float * s = (float *) (src + i * rctx->src0_row_size_aligned); - - hvx_rope_f32_aa(d + n_offs, s + n_offs, rctx->n_dims, theta_cache); - - // fill the remain channels with data from src tensor - if (n_offs > 0) { - hvx_copy_f32_uu((uint8_t *) d, (uint8_t *) s, n_offs); - } - if (n_offs + rctx->n_dims < ne0) { - hvx_copy_f32_uu((uint8_t *)(d + n_offs + rctx->n_dims), (uint8_t *)(s + n_offs + rctx->n_dims), ne0 - n_offs - rctx->n_dims); - } + hvx_rope_f32_aa(d, s, rctx->n_dims, theta_cache); } } static void inline rope_neox_f32(struct htp_rope_context * rctx, uint8_t * restrict dst, uint8_t * restrict src, - uint32_t nr, uint32_t ne0, const float * restrict theta_cache) { - const uint32_t n_offs = rctx->n_offs; // VLEN-aligned (enforced by supports_op) + uint32_t nr, const float * restrict theta_cache) { + // See rope_basic_f32: only the full-row (n_offs == 0) case reaches here. #pragma unroll(4) for (uint32_t i = 0; i < nr; i++) { float * d = (float *) (dst + i * rctx->dst_row_size_aligned); float * s = (float *) (src + i * rctx->src0_row_size_aligned); + hvx_rope_neox_f32_aa(d, s, rctx->n_dims, theta_cache); + } +} - hvx_rope_neox_f32_aa(d + n_offs, s + n_offs, rctx->n_dims, theta_cache); +static void inline rope_basic_f32_inplace(struct htp_rope_context * rctx, uint8_t * src, + uint32_t nr, const float * restrict theta_cache) { + const uint32_t n_offs = rctx->n_offs; // VLEN-aligned (enforced by supports_op) + #pragma unroll(4) + for (uint32_t i = 0; i < nr; i++) { + float * s = (float *) (src + i * rctx->src0_row_size_aligned); + // Rotate the n_dims window at offset n_offs in place; the pass-through prefix [0,n_offs) + // and tail [n_offs+n_dims,ne0) stay untouched in src_spad and survive the whole-row writeback. + hvx_rope_f32_aa(s + n_offs, s + n_offs, rctx->n_dims, theta_cache); + } +} - // fill the remain channels with data from src tensor - if (n_offs > 0) { - hvx_copy_f32_uu((uint8_t *) d, (uint8_t *) s, n_offs); - } - if (n_offs + rctx->n_dims < ne0) { - hvx_copy_f32_uu((uint8_t *)(d + n_offs + rctx->n_dims), (uint8_t *)(s + n_offs + rctx->n_dims), ne0 - n_offs - rctx->n_dims); - } +static void inline rope_neox_f32_inplace(struct htp_rope_context * rctx, uint8_t * src, + uint32_t nr, const float * restrict theta_cache) { + const uint32_t n_offs = rctx->n_offs; // VLEN-aligned (enforced by supports_op) + #pragma unroll(4) + for (uint32_t i = 0; i < nr; i++) { + float * s = (float *) (src + i * rctx->src0_row_size_aligned); + hvx_rope_neox_f32_aa(s + n_offs, s + n_offs, rctx->n_dims, theta_cache); } } @@ -557,7 +577,6 @@ static void inline rope_vision_f32(struct htp_rope_context * rctx, uint8_t * res for (uint32_t i = 0; i < nr; i++) { float * d = (float *) (dst + i * rctx->dst_row_size_aligned); float * s = (float *) (src + i * rctx->src0_row_size_aligned); - hvx_rope_neox_f32_aa(d, s, ne0, theta_cache); } } @@ -590,6 +609,7 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { // MROPE, IMROPE and VISION use NEOX-style pairing for the rotation const bool is_neox = (mode & HTP_ROPE_TYPE_NEOX) || (mode & HTP_ROPE_TYPE_MROPE); const bool is_vision = (mode == HTP_ROPE_TYPE_VISION); + const bool inplace_tail = !is_vision && rctx->n_dims < (int32_t) ne0; // VTCM setup uint8_t * src0_spad_base = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread); @@ -672,6 +692,9 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { n_cache, rctx->ext_factor, rctx->attn_factor, theta_cache, rctx->theta_scale, rctx->theta_powers, rctx->theta_scale_32); } + if (is_neox) { + rope_cache_deinterleave(theta_cache, n_cache); + } htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, i2); } @@ -693,18 +716,25 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir); if (is_vision) { rope_vision_f32(rctx, dst_spad, src_spad, cnr, ne0, theta_cache); + } else if (inplace_tail) { + if (is_neox) { + rope_neox_f32_inplace(rctx, src_spad, cnr, theta_cache); + } else { + rope_basic_f32_inplace(rctx, src_spad, cnr, theta_cache); + } } else if (is_neox) { - rope_neox_f32(rctx, dst_spad, src_spad, cnr, ne0, theta_cache); + rope_neox_f32(rctx, dst_spad, src_spad, cnr, theta_cache); } else { - rope_basic_f32(rctx, dst_spad, src_spad, cnr, ne0, theta_cache); + rope_basic_f32(rctx, dst_spad, src_spad, cnr, theta_cache); } htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir); uint8_t * dst_addr = (uint8_t *) dst->data + i3 * nb3 + i2 * nb2 + i1 * nb1; + uint8_t * wr_spad = inplace_tail ? src_spad : dst_spad; + const size_t wr_pitch = inplace_tail ? rctx->src0_row_size_aligned : rctx->dst_row_size_aligned; - // Write only the row payload while striding the DDR dst - dma_queue_push(dma_queue, dma_make_ptr(dst_addr, dst_spad), - rctx->dst_row_stride, rctx->dst_row_size_aligned, rctx->dst_row_size, cnr); + dma_queue_push(dma_queue, dma_make_ptr(dst_addr, wr_spad), + rctx->dst_row_stride, wr_pitch, rctx->dst_row_size, cnr); // Prefetch more rows (if any) if ((cr + HTP_ROPE_SPAD_NROWS) < nrows) { @@ -713,8 +743,14 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { uint32_t pir = ir + HTP_ROPE_SPAD_NROWS; const uint8_t * src_addr = (const uint8_t *) src0->data + i3 * nb03 + i2 * nb02 + pi1 * nb01; - dma_queue_push(dma_queue, dma_make_ptr(src_spad, src_addr), - rctx->src0_row_size_aligned, rctx->src0_row_stride, rctx->src0_row_size, pnr); + if (inplace_tail) { + // Write and this fetch share src_spad, so the fetch waits for the write. + dma_queue_push_single_2d_ordered(dma_queue, dma_make_ptr(src_spad, src_addr), + rctx->src0_row_size_aligned, rctx->src0_row_stride, rctx->src0_row_size, pnr); + } else { + dma_queue_push(dma_queue, dma_make_ptr(src_spad, src_addr), + rctx->src0_row_size_aligned, rctx->src0_row_stride, rctx->src0_row_size, pnr); + } // FARF(HIGH, "rope-prefetch %u: pr %u i1 %u i2 %u i3 %u src-spad %p src-addr %p pnr %u", ith, pir, pi1, i2, i3, src_spad, src_addr, pnr); } diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index 3342253d0ec9..a244c84b4f01 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -10251,6 +10251,13 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_rope(type, {128, 32, 2, 1}, 32, GGML_ROPE_TYPE_NEOX, 512, 1.4245f, 0.7465f, 1.4245f, false, 0, true, true, 32)); } + // Real-model RoPE: F32 forward, packed Q, 512-token prefill. + test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {256, 8, 512, 1}, 64, GGML_ROPE_TYPE_IMROPE, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // qwen3.5 0.8B + test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {256, 16, 512, 1}, 64, GGML_ROPE_TYPE_IMROPE, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // qwen3.5 4B + test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {256, 8, 512, 1}, 256, GGML_ROPE_TYPE_NEOX, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // gemma4 E2B sliding + test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {512, 8, 512, 1}, 128, GGML_ROPE_TYPE_NEOX, 512, 1.0f, 0.0f, 1.0f, true, 0, true)); // gemma4 E4B global + test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {128, 32, 512, 1}, 128, GGML_ROPE_TYPE_NORMAL, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // llama 7B + for (int v : { 0, 1, 2, 3 }) { for (int dim : { 0, 1, 2, 3, }) { test_cases.emplace_back(new test_concat(GGML_TYPE_F32, {11, 12, 13, 14}, 7, dim, v)); @@ -11142,6 +11149,12 @@ static std::vector> make_test_cases_perf() { } } + // Real-model RoPE: F32 forward, packed Q, 512-token prefill. + test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {256, 8, 512, 1}, 64, GGML_ROPE_TYPE_IMROPE, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // qwen3.5 0.8B + test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {256, 16, 512, 1}, 64, GGML_ROPE_TYPE_IMROPE, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // qwen3.5 4B + test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {256, 8, 512, 1}, 256, GGML_ROPE_TYPE_NEOX, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // gemma4 E2B sliding + test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {512, 8, 512, 1}, 128, GGML_ROPE_TYPE_NEOX, 512, 1.0f, 0.0f, 1.0f, true, 0, true)); // gemma4 E4B global + std::vector> reduce_rows_cases = { { 8192, 1, 1, 1 }, { 8192, 8192, 1, 1 }, From 900ad6d8f528c6953156a7c9755a7b4eb810cd95 Mon Sep 17 00:00:00 2001 From: Max Krasnyansky Date: Mon, 7 Sep 2026 15:25:39 -0700 Subject: [PATCH 4/8] hex-rope: use inplace rope for all scenarios --- ggml/src/ggml-hexagon/htp/dma-queue.h | 12 +- ggml/src/ggml-hexagon/htp/rope-ops.c | 263 ++++++++++---------------- 2 files changed, 100 insertions(+), 175 deletions(-) diff --git a/ggml/src/ggml-hexagon/htp/dma-queue.h b/ggml/src/ggml-hexagon/htp/dma-queue.h index 03e4d024c988..190ca3a9b9e1 100644 --- a/ggml/src/ggml-hexagon/htp/dma-queue.h +++ b/ggml/src/ggml-hexagon/htp/dma-queue.h @@ -185,7 +185,7 @@ static inline bool dma_queue_push_single_1d(dma_queue * q, dma_ptr dptr, size_t return true; } -static inline bool dma_queue_push_single_2d_order(dma_queue * q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows, unsigned order) { +static inline bool dma_queue_push_single_2d(dma_queue * q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { dma_ring * r = q->ring; if (((r->push_idx + 1) & r->idx_mask) == r->pop_idx) { return false; @@ -201,7 +201,7 @@ static inline bool dma_queue_push_single_2d_order(dma_queue * q, dma_ptr dptr, s desc->dst_bypass = dma_is_vtcm(q, dptr.dst) ? 1 : q->nocache; desc->src_comp = 0; desc->dst_comp = 0; - desc->order = order ? 1 : 0; + desc->order = 0; desc->done = 0; desc->src_stride = src_stride; desc->dst_stride = dst_stride; @@ -236,14 +236,6 @@ static inline bool dma_queue_push_single_2d_order(dma_queue * q, dma_ptr dptr, s return true; } -static inline bool dma_queue_push_single_2d(dma_queue * q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { - return dma_queue_push_single_2d_order(q, dptr, dst_stride, src_stride, row_size, nrows, 0); -} - -static inline bool dma_queue_push_single_2d_ordered(dma_queue * q, dma_ptr dptr, size_t dst_stride, size_t src_stride, size_t row_size, size_t nrows) { - return dma_queue_push_single_2d_order(q, dptr, dst_stride, src_stride, row_size, nrows, 1); -} - static inline dma_ptr dma_queue_pop(dma_queue * q) { dma_ring * r = q->ring; dma_ptr dptr = { NULL }; diff --git a/ggml/src/ggml-hexagon/htp/rope-ops.c b/ggml/src/ggml-hexagon/htp/rope-ops.c index f528ad8040fd..3fa8632dc2b4 100644 --- a/ggml/src/ggml-hexagon/htp/rope-ops.c +++ b/ggml/src/ggml-hexagon/htp/rope-ops.c @@ -27,8 +27,13 @@ #define HTP_ROPE_TYPE_VISION 24 #define HTP_ROPE_TYPE_IMROPE 40 -#define HTP_ROPE_SPAD_NROWS 16 -#define HTP_ROPE_SPAD_BLOCK (HTP_ROPE_SPAD_NROWS/2) +#define HTP_ROPE_SPAD_BLOCK 8 +#define HTP_ROPE_SPAD_NSLOTS 4 +#define HTP_ROPE_SPAD_NROWS (HTP_ROPE_SPAD_BLOCK * HTP_ROPE_SPAD_NSLOTS) + +static inline uint8_t * rope_spad_slot(uint8_t * base, uint32_t slot, size_t row_size_aligned) { + return base + (slot * HTP_ROPE_SPAD_BLOCK) * row_size_aligned; +} #define htp_rope_preamble \ const uint32_t ne00 = src0->ne[0]; \ @@ -114,11 +119,13 @@ static inline void rope_yarn_one(float theta, float freq_scale, float * corr_dim mscale_final *= 1.0f + 0.1f * logf(1.0f / freq_scale); } - cache[i0 + 0] = cosf(theta_final) * mscale_final; - cache[i0 + 1] = sinf(theta_final) * mscale_final; + const uint32_t b = i0 / 64; + const uint32_t k = (i0 % 64) / 2; + cache[b * 64 + k] = cosf(theta_final) * mscale_final; + cache[b * 64 + 32 + k] = sinf(theta_final) * mscale_final; } -// 32 thetas -> 32 interleaved (cos, sin) pairs at cache[i0]. +// 32 thetas -> 32 deinterleaved pairs [cos[32] | sin[32]] at cache[i0]. static inline void rope_cache_hvx_32(float * cache, uint32_t i0, HVX_Vector v_theta, const float * freq_factors, @@ -135,14 +142,13 @@ static inline void rope_cache_hvx_32(float * cache, uint32_t i0, hvx_vec_sincos_f32(v_theta_final, &vcos, &vsin); vcos = hvx_vec_mul_f32_f32(vcos, v_mscale); vsin = hvx_vec_mul_f32_f32(vsin, v_mscale); - HVX_VectorPair vstore = Q6_W_vshuff_VVR(vsin, vcos, -4); if (((uintptr_t) (cache + i0)) % 128 == 0) { - hvx_vmem(cache + i0 + 0) = Q6_V_lo_W(vstore); - hvx_vmem(cache + i0 + 32) = Q6_V_hi_W(vstore); + hvx_vmem(cache + i0 + 0) = vcos; + hvx_vmem(cache + i0 + 32) = vsin; } else { - hvx_vec_store_u(cache + i0 + 0, 32 * sizeof(float), Q6_V_lo_W(vstore)); - hvx_vec_store_u(cache + i0 + 32, 32 * sizeof(float), Q6_V_hi_W(vstore)); + hvx_vec_store_u(cache + i0 + 0, 32 * sizeof(float), vcos); + hvx_vec_store_u(cache + i0 + 32, 32 * sizeof(float), vsin); } } @@ -390,18 +396,6 @@ static void rope_corr_dims(int n_dims, dims[1] = MIN(n_dims - 1, end); } -// Inverse of the (cos, sin) interleave stored by rope_cache_hvx_32. -// Full 32-pair blocks become [cos[32] | sin[32]]. Leftover pairs stay interleaved. -static inline void rope_cache_deinterleave(float * cache, uint32_t n_cache) { - const uint32_t n_blocks = n_cache / 64; - for (uint32_t b = 0; b < n_blocks; b++) { - HVX_Vector * v = (HVX_Vector *) (cache + b * 64); - HVX_VectorPair cs = Q6_W_vdeal_VVR(v[1], v[0], -4); - v[0] = Q6_V_lo_W(cs); - v[1] = Q6_V_hi_W(cs); - } -} - static inline void hvx_rope_neox_mul(HVX_Vector v0, HVX_Vector v1, HVX_Vector vcos, HVX_Vector vsin, HVX_Vector * o0, HVX_Vector * o1) { HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(v0, vcos); @@ -444,11 +438,10 @@ static inline void hvx_rope_neox_f32_aa(float * restrict dst, const float * rest HVX_Vector v0 = hvx_vmemu(src0 + nvec * 32); HVX_Vector v1 = hvx_vmemu(src0 + he + nvec * 32); - HVX_Vector v2 = hvx_vmemu(theta_cache + nvec * 64); - HVX_Vector v3 = hvx_vmemu(theta_cache + nvec * 64 + 32); - HVX_VectorPair vcos_sin = Q6_W_vdeal_VVR(v3, v2, -4); + HVX_Vector vcos = hvx_vmemu(theta_cache + nvec * 64); + HVX_Vector vsin = hvx_vmemu(theta_cache + nvec * 64 + 32); HVX_Vector o0, o1; - hvx_rope_neox_mul(v0, v1, Q6_V_lo_W(vcos_sin), Q6_V_hi_W(vcos_sin), &o0, &o1); + hvx_rope_neox_mul(v0, v1, vcos, vsin, &o0, &o1); hvx_vec_store_u(dst + nvec * 32, nloe * sizeof(float), o0); hvx_vec_store_u(dst + he + nvec * 32, nloe * sizeof(float), o1); } @@ -461,16 +454,15 @@ static inline void hvx_rope_f32_aa(float * restrict dst, const float * restrict HVX_Vector v0 = ((const HVX_Vector *) src0)[i * 2 + 0]; HVX_Vector v1 = ((const HVX_Vector *) src0)[i * 2 + 1]; - HVX_Vector v2 = ((const HVX_Vector *) theta_cache)[i * 2 + 0]; - HVX_Vector v3 = ((const HVX_Vector *) theta_cache)[i * 2 + 1]; + HVX_Vector vcos = ((const HVX_Vector *) theta_cache)[i * 2 + 0]; + HVX_Vector vsin = ((const HVX_Vector *) theta_cache)[i * 2 + 1]; - HVX_VectorPair vx0_x1 = Q6_W_vdeal_VVR(v1, v0, -4); - HVX_VectorPair vcos_sin = Q6_W_vdeal_VVR(v3, v2, -4); + HVX_VectorPair vx0_x1 = Q6_W_vdeal_VVR(v1, v0, -4); - HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), Q6_V_lo_W(vcos_sin)); - HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), Q6_V_hi_W(vcos_sin)); - HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), Q6_V_lo_W(vcos_sin)); - HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), Q6_V_hi_W(vcos_sin)); + HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), vcos); + HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), vsin); + HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), vcos); + HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), vsin); HVX_Vector v4 = Q6_Vqf32_vsub_Vqf32Vqf32(vx0_c, vx1_s); HVX_Vector v5 = Q6_Vqf32_vadd_Vqf32Vqf32(vx0_s, vx1_c); @@ -484,15 +476,15 @@ static inline void hvx_rope_f32_aa(float * restrict dst, const float * restrict if (nloe > 0) { if (nloe <= 32) { HVX_Vector v0 = hvx_vmemu(src0 + nvec * 64); - HVX_Vector v2 = hvx_vmemu(theta_cache + nvec * 64); + HVX_Vector vcos = hvx_vmemu(theta_cache + nvec * 64); + HVX_Vector vsin = hvx_vmemu(theta_cache + nvec * 64 + 32); - HVX_VectorPair vx0_x1 = Q6_W_vdeal_VVR(Q6_V_vzero(), v0, -4); - HVX_VectorPair vcos_sin = Q6_W_vdeal_VVR(Q6_V_vzero(), v2, -4); + HVX_VectorPair vx0_x1 = Q6_W_vdeal_VVR(Q6_V_vzero(), v0, -4); - HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), Q6_V_lo_W(vcos_sin)); - HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), Q6_V_hi_W(vcos_sin)); - HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), Q6_V_lo_W(vcos_sin)); - HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), Q6_V_hi_W(vcos_sin)); + HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), vcos); + HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), vsin); + HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), vcos); + HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), vsin); HVX_Vector v4 = Q6_Vqf32_vsub_Vqf32Vqf32(vx0_c, vx1_s); HVX_Vector v5 = Q6_Vqf32_vadd_Vqf32Vqf32(vx0_s, vx1_c); @@ -504,16 +496,15 @@ static inline void hvx_rope_f32_aa(float * restrict dst, const float * restrict HVX_Vector v0 = hvx_vmemu(src0 + nvec * 64); HVX_Vector v1 = hvx_vmemu(src0 + nvec * 64 + 32); - HVX_Vector v2 = hvx_vmemu(theta_cache + nvec * 64); - HVX_Vector v3 = hvx_vmemu(theta_cache + nvec * 64 + 32); + HVX_Vector vcos = hvx_vmemu(theta_cache + nvec * 64); + HVX_Vector vsin = hvx_vmemu(theta_cache + nvec * 64 + 32); - HVX_VectorPair vx0_x1 = Q6_W_vdeal_VVR(v1, v0, -4); - HVX_VectorPair vcos_sin = Q6_W_vdeal_VVR(v3, v2, -4); + HVX_VectorPair vx0_x1 = Q6_W_vdeal_VVR(v1, v0, -4); - HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), Q6_V_lo_W(vcos_sin)); - HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), Q6_V_hi_W(vcos_sin)); - HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), Q6_V_lo_W(vcos_sin)); - HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), Q6_V_hi_W(vcos_sin)); + HVX_Vector vx0_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), vcos); + HVX_Vector vx0_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_lo_W(vx0_x1), vsin); + HVX_Vector vx1_c = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), vcos); + HVX_Vector vx1_s = Q6_Vqf32_vmpy_VsfVsf(Q6_V_hi_W(vx0_x1), vsin); HVX_Vector v4 = Q6_Vqf32_vsub_Vqf32Vqf32(vx0_c, vx1_s); HVX_Vector v5 = Q6_Vqf32_vadd_Vqf32Vqf32(vx0_s, vx1_c); @@ -526,58 +517,23 @@ static inline void hvx_rope_f32_aa(float * restrict dst, const float * restrict } } -static void inline rope_basic_f32(struct htp_rope_context * rctx, uint8_t * restrict dst, uint8_t * restrict src, - uint32_t nr, const float * restrict theta_cache) { - // Only reached when there is no pass-through region (n_dims == ne0, hence n_offs == 0); - // the n_offs>0 / partial-n_dims cases go through the *_inplace path (see rope_job_f32). - #pragma unroll(4) - for (uint32_t i = 0; i < nr; i++) { - float * d = (float *) (dst + i * rctx->dst_row_size_aligned); - float * s = (float *) (src + i * rctx->src0_row_size_aligned); - hvx_rope_f32_aa(d, s, rctx->n_dims, theta_cache); - } -} - -static void inline rope_neox_f32(struct htp_rope_context * rctx, uint8_t * restrict dst, uint8_t * restrict src, - uint32_t nr, const float * restrict theta_cache) { - // See rope_basic_f32: only the full-row (n_offs == 0) case reaches here. - #pragma unroll(4) - for (uint32_t i = 0; i < nr; i++) { - float * d = (float *) (dst + i * rctx->dst_row_size_aligned); - float * s = (float *) (src + i * rctx->src0_row_size_aligned); - hvx_rope_neox_f32_aa(d, s, rctx->n_dims, theta_cache); - } -} - static void inline rope_basic_f32_inplace(struct htp_rope_context * rctx, uint8_t * src, uint32_t nr, const float * restrict theta_cache) { - const uint32_t n_offs = rctx->n_offs; // VLEN-aligned (enforced by supports_op) + const uint32_t n_offs = rctx->n_offs; #pragma unroll(4) for (uint32_t i = 0; i < nr; i++) { float * s = (float *) (src + i * rctx->src0_row_size_aligned); - // Rotate the n_dims window at offset n_offs in place; the pass-through prefix [0,n_offs) - // and tail [n_offs+n_dims,ne0) stay untouched in src_spad and survive the whole-row writeback. hvx_rope_f32_aa(s + n_offs, s + n_offs, rctx->n_dims, theta_cache); } } static void inline rope_neox_f32_inplace(struct htp_rope_context * rctx, uint8_t * src, - uint32_t nr, const float * restrict theta_cache) { - const uint32_t n_offs = rctx->n_offs; // VLEN-aligned (enforced by supports_op) + uint32_t nr, uint32_t ne, const float * restrict theta_cache) { + const uint32_t n_offs = rctx->n_offs; #pragma unroll(4) for (uint32_t i = 0; i < nr; i++) { float * s = (float *) (src + i * rctx->src0_row_size_aligned); - hvx_rope_neox_f32_aa(s + n_offs, s + n_offs, rctx->n_dims, theta_cache); - } -} - -static void inline rope_vision_f32(struct htp_rope_context * rctx, uint8_t * restrict dst, uint8_t * restrict src, - uint32_t nr, uint32_t ne0, const float * restrict theta_cache) { - #pragma unroll(4) - for (uint32_t i = 0; i < nr; i++) { - float * d = (float *) (dst + i * rctx->dst_row_size_aligned); - float * s = (float *) (src + i * rctx->src0_row_size_aligned); - hvx_rope_neox_f32_aa(d, s, ne0, theta_cache); + hvx_rope_neox_f32_aa(s + n_offs, s + n_offs, ne, theta_cache); } } @@ -609,13 +565,11 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { // MROPE, IMROPE and VISION use NEOX-style pairing for the rotation const bool is_neox = (mode & HTP_ROPE_TYPE_NEOX) || (mode & HTP_ROPE_TYPE_MROPE); const bool is_vision = (mode == HTP_ROPE_TYPE_VISION); - const bool inplace_tail = !is_vision && rctx->n_dims < (int32_t) ne0; // VTCM setup uint8_t * src0_spad_base = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread); float * theta_cache = (float *) (src0_spad_base); src0_spad_base = src0_spad_base + rctx->theta_cache_offset; - uint8_t * dst_spad_base = octx->dst_spad.data + (ith * octx->dst_spad.size_per_thread); dma_queue * dma_queue = octx->ctx->dma[ith]; struct htp_thread_trace * tr = &octx->ctx->trace[ith]; @@ -629,6 +583,7 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { uint32_t ir = src0_start_row; uint32_t prev_i2 = (uint32_t) -1; + uint32_t cur_slot = 0; for (uint32_t i3 = i3_start; i3 < ne3; i3++) { // batch const uint32_t i2_init = (i3 == i3_start) ? i2_start : 0; @@ -641,29 +596,21 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { const uint32_t nrows = MIN(src0_end_row - ir, ne1 - i1); // Depth before prefetch - uint32_t dma_depth = dma_queue_depth(dma_queue); - - // FARF(HIGH, "rope-block %u: ir %u n-rows %u dma-depth %u : usec %u", ith, ir, nrows, dma_depth, - // (unsigned) HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - rctx->t_start)); + const uint32_t dma_depth = dma_queue_depth(dma_queue); - // Prefetch loop - for (uint32_t pnr = 0, pr = 0; pr < nrows && pr < HTP_ROPE_SPAD_NROWS; pr += pnr) { - pnr = MIN(nrows - pr, HTP_ROPE_SPAD_BLOCK); + // Prefetch up to 2 blocks + const uint32_t p_nrows = MIN(nrows, 2 * HTP_ROPE_SPAD_BLOCK); + for (uint32_t pr = 0; pr < p_nrows; pr += HTP_ROPE_SPAD_BLOCK) { + const uint32_t pnr = MIN(nrows - pr, HTP_ROPE_SPAD_BLOCK); + const uint32_t slot = (cur_slot + pr / HTP_ROPE_SPAD_BLOCK) % HTP_ROPE_SPAD_NSLOTS; + uint8_t * spad_slot = rope_spad_slot(src0_spad_base, slot, rctx->src0_row_size_aligned); + const uint8_t * src_addr = (const uint8_t *) src0->data + i3 * nb03 + i2 * nb02 + (i1 + pr) * nb01; - uint32_t pi1 = i1 + pr; - uint32_t pir = ir + pr; + // Dummy DMA transaction for sequencing (interleaving wr, rd, wr, rd, ...) + dma_queue_push(dma_queue, dma_make_ptr((void *) dst->data, spad_slot), 0, 0, 0, 0); - // Dummy DMA transaction for sequencing (interleaving dst,src,dst,...) - dma_queue_push_vtcm_to_ddr(dma_queue, dma_make_ptr((void *) dst->data, dst_spad_base + pr * rctx->dst_row_size_aligned), 0, 0, 0); - - const uint8_t * src_addr = (const uint8_t *) src0->data + i3 * nb03 + i2 * nb02 + pi1 * nb01; - uint8_t * src_spad = src0_spad_base + pr * rctx->src0_row_size_aligned; - - // Copy only the row payload while striding the DDR source - dma_queue_push(dma_queue, dma_make_ptr(src_spad, src_addr), + dma_queue_push(dma_queue, dma_make_ptr(spad_slot, src_addr), rctx->src0_row_size_aligned, rctx->src0_row_stride, rctx->src0_row_size, pnr); - - // FARF(HIGH, "rope-prefetch %u: pr %u i1 %u i2 %u i3 %u src-spad %p src-addr %p pnr %u", ith, pir, pi1, i2, i3, src_spad, src_addr, pnr); } // Update theta cache @@ -692,69 +639,56 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { n_cache, rctx->ext_factor, rctx->attn_factor, theta_cache, rctx->theta_scale, rctx->theta_powers, rctx->theta_scale_32); } - if (is_neox) { - rope_cache_deinterleave(theta_cache, n_cache); - } htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_A_PREP, i2); } // Skip output DMA transactions from prev block (if any) - // No need to wait for those here since we're explicitly waiting for the latest prefecthes below. - for (uint32_t d=0; d < dma_depth; d++) { dma_queue_pop_nowait(dma_queue); } + for (uint32_t d = 0; d < dma_depth; d++) { dma_queue_pop_nowait(dma_queue); } // Compute loop - for (uint32_t cnr = 0, cr = 0; cr < nrows; cr += cnr, ir += cnr, i1 += cnr) { - // Number of rows to compute + const uint32_t ne = is_vision ? ne0 : rctx->n_dims; + const uint32_t base_i1 = i1; + const uint32_t base_ir = ir; + + for (uint32_t cnr = 0, cr = 0; cr < nrows; cr += cnr) { cnr = MIN(nrows - cr, HTP_ROPE_SPAD_BLOCK); + const uint32_t slot = (cur_slot + cr / HTP_ROPE_SPAD_BLOCK) % HTP_ROPE_SPAD_NSLOTS; + const uint32_t cur_ir = base_ir + cr; + const uint32_t cur_i1 = base_i1 + cr; + + dma_queue_pop(dma_queue); + uint8_t * cur_spad = (uint8_t *) dma_queue_pop(dma_queue).dst; - uint8_t * dst_spad = (uint8_t *) dma_queue_pop(dma_queue).src; - uint8_t * src_spad = (uint8_t *) dma_queue_pop(dma_queue).dst; - - // FARF(HIGH, "rope-compute %u: ir %u i1 %u i2 %u i3 %u src-spad %p cnr %u : usec %u", ith, ir, i1, i2, i3, src_spad, cnr, - // (unsigned) HAP_perf_qtimer_count_to_us(HAP_perf_get_qtimer_count() - rctx->t_start)); - - htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, ir); - if (is_vision) { - rope_vision_f32(rctx, dst_spad, src_spad, cnr, ne0, theta_cache); - } else if (inplace_tail) { - if (is_neox) { - rope_neox_f32_inplace(rctx, src_spad, cnr, theta_cache); - } else { - rope_basic_f32_inplace(rctx, src_spad, cnr, theta_cache); - } - } else if (is_neox) { - rope_neox_f32(rctx, dst_spad, src_spad, cnr, theta_cache); + htp_trace_event_start(tr, HTP_TRACE_EVT_HVX_COMP, cur_ir); + if (is_neox || is_vision) { + rope_neox_f32_inplace(rctx, cur_spad, cnr, ne, theta_cache); } else { - rope_basic_f32(rctx, dst_spad, src_spad, cnr, theta_cache); + rope_basic_f32_inplace(rctx, cur_spad, cnr, theta_cache); } - htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, ir); - - uint8_t * dst_addr = (uint8_t *) dst->data + i3 * nb3 + i2 * nb2 + i1 * nb1; - uint8_t * wr_spad = inplace_tail ? src_spad : dst_spad; - const size_t wr_pitch = inplace_tail ? rctx->src0_row_size_aligned : rctx->dst_row_size_aligned; - - dma_queue_push(dma_queue, dma_make_ptr(dst_addr, wr_spad), - rctx->dst_row_stride, wr_pitch, rctx->dst_row_size, cnr); - - // Prefetch more rows (if any) - if ((cr + HTP_ROPE_SPAD_NROWS) < nrows) { - uint32_t pnr = MIN(nrows - (cr + HTP_ROPE_SPAD_NROWS), HTP_ROPE_SPAD_BLOCK); - uint32_t pi1 = i1 + HTP_ROPE_SPAD_NROWS; - uint32_t pir = ir + HTP_ROPE_SPAD_NROWS; - - const uint8_t * src_addr = (const uint8_t *) src0->data + i3 * nb03 + i2 * nb02 + pi1 * nb01; - if (inplace_tail) { - // Write and this fetch share src_spad, so the fetch waits for the write. - dma_queue_push_single_2d_ordered(dma_queue, dma_make_ptr(src_spad, src_addr), - rctx->src0_row_size_aligned, rctx->src0_row_stride, rctx->src0_row_size, pnr); - } else { - dma_queue_push(dma_queue, dma_make_ptr(src_spad, src_addr), - rctx->src0_row_size_aligned, rctx->src0_row_stride, rctx->src0_row_size, pnr); - } - - // FARF(HIGH, "rope-prefetch %u: pr %u i1 %u i2 %u i3 %u src-spad %p src-addr %p pnr %u", ith, pir, pi1, i2, i3, src_spad, src_addr, pnr); + htp_trace_event_stop(tr, HTP_TRACE_EVT_HVX_COMP, cur_ir); + + uint8_t * dst_addr = (uint8_t *) dst->data + i3 * nb3 + i2 * nb2 + cur_i1 * nb1; + dma_queue_push(dma_queue, dma_make_ptr(dst_addr, cur_spad), + rctx->dst_row_stride, rctx->src0_row_size_aligned, rctx->dst_row_size, cnr); + + // Prefetch 2 blocks ahead into the slot just freed + if ((cr + 2 * HTP_ROPE_SPAD_BLOCK) < nrows) { + const uint32_t p_cr = cr + 2 * HTP_ROPE_SPAD_BLOCK; + const uint32_t pnr = MIN(nrows - p_cr, HTP_ROPE_SPAD_BLOCK); + const uint32_t p_slot = (cur_slot + p_cr / HTP_ROPE_SPAD_BLOCK) % HTP_ROPE_SPAD_NSLOTS; + uint8_t * p_spad = rope_spad_slot(src0_spad_base, p_slot, rctx->src0_row_size_aligned); + const uint8_t * src_addr = (const uint8_t *) src0->data + i3 * nb03 + i2 * nb02 + (base_i1 + p_cr) * nb01; + + dma_queue_push(dma_queue, dma_make_ptr(p_spad, src_addr), + rctx->src0_row_size_aligned, rctx->src0_row_stride, rctx->src0_row_size, pnr); } } + + const uint32_t n_chunks = (nrows + HTP_ROPE_SPAD_BLOCK - 1) / HTP_ROPE_SPAD_BLOCK; + cur_slot = (cur_slot + n_chunks) % HTP_ROPE_SPAD_NSLOTS; + + ir += nrows; + i1 += nrows; } } } @@ -801,8 +735,7 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) { // Calculate spad sizes per thread size_t src0_spad_per_thread = theta_cache_size_aligned + HTP_ROPE_SPAD_NROWS * src0_row_size_aligned; - size_t dst_spad_per_thread = HTP_ROPE_SPAD_NROWS * dst_row_size_aligned; - size_t spad_per_thread = src0_spad_per_thread + dst_spad_per_thread; + size_t spad_per_thread = src0_spad_per_thread; // Check if we fit in VTCM size_t total_vtcm_needed = spad_per_thread * n_threads; @@ -812,14 +745,14 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) { } octx->src0_spad.size_per_thread = src0_spad_per_thread; - octx->dst_spad.size_per_thread = dst_spad_per_thread; + octx->dst_spad.size_per_thread = 0; octx->src0_spad.size = n_threads * src0_spad_per_thread; - octx->dst_spad.size = n_threads * dst_spad_per_thread; + octx->dst_spad.size = 0; octx->src1_spad.size = 0; octx->src0_spad.data = octx->ctx->vtcm_base; octx->src0_spad.src = NULL; octx->src1_spad.data = NULL; octx->src1_spad.src = NULL; - octx->dst_spad.data = octx->src0_spad.data + octx->src0_spad.size; octx->dst_spad.src = NULL; + octx->dst_spad.data = NULL; octx->dst_spad.src = NULL; struct htp_rope_context rctx; memset(&rctx, 0, sizeof(struct htp_rope_context)); From ad656502dad1aec66f6b5a49b9b07800cb8bd5c7 Mon Sep 17 00:00:00 2001 From: Max Krasnyansky Date: Mon, 7 Sep 2026 15:41:38 -0700 Subject: [PATCH 5/8] hex-rope: remove ctx->spad usage and legacy timers --- ggml/src/ggml-hexagon/htp/rope-ops.c | 37 +++++++++------------------- 1 file changed, 11 insertions(+), 26 deletions(-) diff --git a/ggml/src/ggml-hexagon/htp/rope-ops.c b/ggml/src/ggml-hexagon/htp/rope-ops.c index 3fa8632dc2b4..d196d2e503e6 100644 --- a/ggml/src/ggml-hexagon/htp/rope-ops.c +++ b/ggml/src/ggml-hexagon/htp/rope-ops.c @@ -75,23 +75,23 @@ struct htp_rope_context { float corr_dims[2]; uint32_t src0_nrows_per_thread; - size_t spad_stride; struct htp_ops_context * octx; + uint8_t * vtcm_base; + size_t spad_per_thread; + size_t theta_cache_offset; + size_t src0_row_size; size_t src0_row_stride; size_t dst_row_size; size_t dst_row_stride; size_t src0_row_size_aligned; size_t dst_row_size_aligned; - size_t theta_cache_offset; uint32_t src0_nrows; struct fastdiv_values div_ne2_ne1; struct fastdiv_values div_ne1; - - uint64_t t_start; }; static float rope_yarn_ramp(const float low, const float high, const int i0) { @@ -559,15 +559,13 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { return; } - uint64_t tt = HAP_perf_get_qtimer_count(); - const int32_t mode = rctx->mode; // MROPE, IMROPE and VISION use NEOX-style pairing for the rotation const bool is_neox = (mode & HTP_ROPE_TYPE_NEOX) || (mode & HTP_ROPE_TYPE_MROPE); const bool is_vision = (mode == HTP_ROPE_TYPE_VISION); // VTCM setup - uint8_t * src0_spad_base = octx->src0_spad.data + (ith * octx->src0_spad.size_per_thread); + uint8_t * src0_spad_base = rctx->vtcm_base + (ith * rctx->spad_per_thread); float * theta_cache = (float *) (src0_spad_base); src0_spad_base = src0_spad_base + rctx->theta_cache_offset; @@ -695,9 +693,8 @@ static void rope_job_f32(unsigned int nth, unsigned int ith, void * data) { done: dma_queue_flush(dma_queue); - tt = HAP_perf_get_qtimer_count() - tt; - FARF(HIGH, "rope-f32: %d/%d: (%u:%u) usec %u\n", ith, nth, src0_start_row, src0_end_row, (unsigned) HAP_perf_qtimer_count_to_us(tt)); + FARF(HIGH, "rope-f32: %d/%d: (%u:%u)\n", ith, nth, src0_start_row, src0_end_row); } static int execute_op_rope_f32(struct htp_ops_context * octx) { @@ -744,22 +741,13 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) { return HTP_STATUS_VTCM_TOO_SMALL; } - octx->src0_spad.size_per_thread = src0_spad_per_thread; - octx->dst_spad.size_per_thread = 0; - octx->src0_spad.size = n_threads * src0_spad_per_thread; - octx->dst_spad.size = 0; - octx->src1_spad.size = 0; - - octx->src0_spad.data = octx->ctx->vtcm_base; octx->src0_spad.src = NULL; - octx->src1_spad.data = NULL; octx->src1_spad.src = NULL; - octx->dst_spad.data = NULL; octx->dst_spad.src = NULL; - struct htp_rope_context rctx; memset(&rctx, 0, sizeof(struct htp_rope_context)); - rctx.t_start = HAP_perf_get_qtimer_count(); - - rctx.octx = octx; + rctx.octx = octx; + rctx.vtcm_base = (uint8_t *) octx->ctx->vtcm_base; + rctx.spad_per_thread = spad_per_thread; + rctx.theta_cache_offset = theta_cache_size_aligned; const int32_t * op_params = &octx->op_params[0]; rctx.n_dims = ((const int32_t *) op_params)[1]; @@ -790,7 +778,6 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) { rctx.dst_row_stride = dst_row_stride; rctx.src0_row_size_aligned = src0_row_size_aligned; rctx.dst_row_size_aligned = dst_row_size_aligned; - rctx.theta_cache_offset = theta_cache_size_aligned; rctx.src0_nrows = src0_nrows; rctx.src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads; @@ -803,9 +790,7 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) { FARF(HIGH, "rope-f32 n-rows %u n-dims %d ne0 %u ext-factor %.6f theta-scale %.6f attn-factor %.6f\n", rctx.src0_nrows, rctx.n_dims, ne0, rctx.ext_factor, rctx.theta_scale, rctx.attn_factor); - if (!(octx->flags & HTP_OPFLAGS_SKIP_COMPUTE)) { - worker_pool_run_func(octx->ctx->worker_pool, rope_job_f32, &rctx, n_threads); - } + work_queue_run(octx->ctx->work_queue, rope_job_f32, &rctx, n_threads); return err; } From 132f1724dad8beb8f66231717afa57249f7857bf Mon Sep 17 00:00:00 2001 From: Max Krasnyansky Date: Mon, 7 Sep 2026 16:22:30 -0700 Subject: [PATCH 6/8] hex-rope: add kernel params and enforce vtcm reqs at the host --- ggml/src/ggml-hexagon/ggml-hexagon.cpp | 54 +++++++++++++++++++++- ggml/src/ggml-hexagon/htp/rope-ops.c | 64 ++++++++------------------ 2 files changed, 70 insertions(+), 48 deletions(-) diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp index 104201daff51..7b9267b9ddd5 100644 --- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp +++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp @@ -56,6 +56,7 @@ #include "htp/unary-ops.h" #include "htp/get-rows-ops.h" #include "htp/set-rows-ops.h" +#include "htp/rope-ops.h" #include "htp_iface.h" #include "htp-drv.h" @@ -299,6 +300,12 @@ static void ggml_hexagon_precompute_set_rows_params( struct htp_set_rows_kernel_params * kparams ); +static void ggml_hexagon_precompute_rope_params( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_rope_kernel_params * kparams +); + static void ggml_hexagon_precompute_fused_mmnx_params( const struct ggml_hexagon_session * sess, const struct ggml_tensor * src0, @@ -4148,6 +4155,37 @@ static void ggml_hexagon_precompute_set_rows_params( kparams->vtcm_size = vtcm_layout.total_bytes; } +static void ggml_hexagon_precompute_rope_params( + const struct ggml_hexagon_session * sess, + const struct ggml_tensor * op, + struct htp_rope_kernel_params * kparams +) { + memset(kparams, 0, sizeof(*kparams)); + + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * dst = op; + + const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, src0_nrows); + + struct htp_rope_vtcm_layout layout; + htp_rope_vtcm_layout_build(&layout, src0->ne[0], dst->nb[1], n_threads); + + kparams->n_threads = n_threads; + kparams->src0_nrows = src0_nrows; + kparams->src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads; + kparams->vtcm_size = (uint32_t) layout.total_bytes; + kparams->spad_per_thread = (uint32_t) layout.bytes_per_thread; + kparams->theta_cache_offset = (uint32_t) layout.theta_cache_size_aligned; + kparams->src0_row_size_aligned = (uint32_t) layout.src0_row_size_aligned; + kparams->dst_row_size_aligned = (uint32_t) layout.dst_row_size_aligned; + + if (src0_nrows > 0) { + kparams->div_ne2_ne1 = init_fastdiv_values(dst->ne[2] * dst->ne[1]); + kparams->div_ne1 = init_fastdiv_values(dst->ne[1]); + } +} + static void ggml_hexagon_precompute_fused_mmnx_params( const struct ggml_hexagon_session * sess, const struct ggml_tensor * src0, // W0 @@ -4768,9 +4806,16 @@ static bool ggml_hexagon_supported_rope(const struct ggml_hexagon_session * sess if (src0->nb[1] < src0->ne[0] * sizeof(float) || dst->nb[1] < dst->ne[0] * sizeof(float)) { return false; } - return true; + const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, src0_nrows); - GGML_UNUSED(sess); + struct htp_rope_vtcm_layout layout; + htp_rope_vtcm_layout_build(&layout, src0->ne[0], dst->nb[1], n_threads); + if (layout.total_bytes > sess->vtcm_size) { + return false; + } + + return true; } static bool ggml_hexagon_supported_ssm_conv(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { @@ -5204,6 +5249,11 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg node.node->src[0], node.node->src[1], node.dst(), (struct htp_set_rows_kernel_params *)node.kernel_params ); + } else if (node.opcode == HTP_OP_ROPE) { + ggml_hexagon_precompute_rope_params(sess, + node.node, + (struct htp_rope_kernel_params *)node.kernel_params + ); } computed_nodes.push_back(std::move(node)); } diff --git a/ggml/src/ggml-hexagon/htp/rope-ops.c b/ggml/src/ggml-hexagon/htp/rope-ops.c index d196d2e503e6..22c4b82a6484 100644 --- a/ggml/src/ggml-hexagon/htp/rope-ops.c +++ b/ggml/src/ggml-hexagon/htp/rope-ops.c @@ -17,8 +17,8 @@ #include "ggml-common.h" #include "htp-ctx.h" #include "htp-ops.h" -#include "htp-ops.h" #include "htp-tensor.h" +#include "rope-ops.h" // Redefined the rope type constants as we can't include ggml.h #define HTP_ROPE_TYPE_NORMAL 0 @@ -27,14 +27,6 @@ #define HTP_ROPE_TYPE_VISION 24 #define HTP_ROPE_TYPE_IMROPE 40 -#define HTP_ROPE_SPAD_BLOCK 8 -#define HTP_ROPE_SPAD_NSLOTS 4 -#define HTP_ROPE_SPAD_NROWS (HTP_ROPE_SPAD_BLOCK * HTP_ROPE_SPAD_NSLOTS) - -static inline uint8_t * rope_spad_slot(uint8_t * base, uint32_t slot, size_t row_size_aligned) { - return base + (slot * HTP_ROPE_SPAD_BLOCK) * row_size_aligned; -} - #define htp_rope_preamble \ const uint32_t ne00 = src0->ne[0]; \ const uint32_t ne01 = src0->ne[1]; \ @@ -705,8 +697,6 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) { const struct htp_tensor * src2 = octx->src[2]; const struct htp_tensor * dst = octx->dst; - const char * op_type = "rope-f32"; - switch (octx->op) { case HTP_OP_ROPE: break; @@ -716,38 +706,23 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) { return HTP_STATUS_NO_SUPPORT; } - const uint32_t ne0 = dst->ne[0]; - const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t n_threads = MIN(octx->n_threads, src0_nrows); + const struct htp_rope_kernel_params * kparams = (const struct htp_rope_kernel_params *) octx->kernel_params; + assert(kparams->n_threads > 0); + assert(octx->ctx->vtcm_size >= kparams->vtcm_size); + const uint32_t ne0 = dst->ne[0]; const size_t src0_row_size = src0->ne[0] * sizeof(float); const size_t src0_row_stride = src0->nb[1]; const size_t dst_row_size = dst->ne[0] * sizeof(float); const size_t dst_row_stride = dst->nb[1]; - // Aligned row sizes for VTCM - const size_t src0_row_size_aligned = hex_round_up(src0_row_size, VLEN); - const size_t dst_row_size_aligned = hex_round_up(dst_row_stride, VLEN); - const size_t theta_cache_size_aligned = hex_round_up(src0->ne[0] * sizeof(float), 256); - - // Calculate spad sizes per thread - size_t src0_spad_per_thread = theta_cache_size_aligned + HTP_ROPE_SPAD_NROWS * src0_row_size_aligned; - size_t spad_per_thread = src0_spad_per_thread; - - // Check if we fit in VTCM - size_t total_vtcm_needed = spad_per_thread * n_threads; - if (octx->ctx->vtcm_size < total_vtcm_needed) { - FARF(ERROR, "%s : current VTCM reservation %zu is too small, needed %zu\n", op_type, octx->ctx->vtcm_size, total_vtcm_needed); - return HTP_STATUS_VTCM_TOO_SMALL; - } - struct htp_rope_context rctx; memset(&rctx, 0, sizeof(struct htp_rope_context)); rctx.octx = octx; rctx.vtcm_base = (uint8_t *) octx->ctx->vtcm_base; - rctx.spad_per_thread = spad_per_thread; - rctx.theta_cache_offset = theta_cache_size_aligned; + rctx.spad_per_thread = kparams->spad_per_thread; + rctx.theta_cache_offset = kparams->theta_cache_offset; const int32_t * op_params = &octx->op_params[0]; rctx.n_dims = ((const int32_t *) op_params)[1]; @@ -772,25 +747,22 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) { rope_corr_dims(rctx.n_dims, rctx.n_ctx_orig, rctx.freq_base, rctx.beta_fast, rctx.beta_slow, rctx.corr_dims); - rctx.src0_row_size = src0_row_size; - rctx.src0_row_stride = src0_row_stride; - rctx.dst_row_size = dst_row_size; - rctx.dst_row_stride = dst_row_stride; - rctx.src0_row_size_aligned = src0_row_size_aligned; - rctx.dst_row_size_aligned = dst_row_size_aligned; - - rctx.src0_nrows = src0_nrows; - rctx.src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads; + rctx.src0_row_size = src0_row_size; + rctx.src0_row_stride = src0_row_stride; + rctx.dst_row_size = dst_row_size; + rctx.dst_row_stride = dst_row_stride; + rctx.src0_row_size_aligned = kparams->src0_row_size_aligned; + rctx.dst_row_size_aligned = kparams->dst_row_size_aligned; - if (src0_nrows > 0) { - rctx.div_ne2_ne1 = init_fastdiv_values(dst->ne[2] * dst->ne[1]); - rctx.div_ne1 = init_fastdiv_values(dst->ne[1]); - } + rctx.src0_nrows = kparams->src0_nrows; + rctx.src0_nrows_per_thread = kparams->src0_nrows_per_thread; + rctx.div_ne2_ne1 = kparams->div_ne2_ne1; + rctx.div_ne1 = kparams->div_ne1; FARF(HIGH, "rope-f32 n-rows %u n-dims %d ne0 %u ext-factor %.6f theta-scale %.6f attn-factor %.6f\n", rctx.src0_nrows, rctx.n_dims, ne0, rctx.ext_factor, rctx.theta_scale, rctx.attn_factor); - work_queue_run(octx->ctx->work_queue, rope_job_f32, &rctx, n_threads); + work_queue_run(octx->ctx->work_queue, rope_job_f32, &rctx, kparams->n_threads); return err; } From 8d25744d4b1559bed2c172b920f2f6a81a7af6ac Mon Sep 17 00:00:00 2001 From: Max Krasnyansky Date: Mon, 7 Sep 2026 17:00:08 -0700 Subject: [PATCH 7/8] hex-rope: cleanup unused params and tighten the mode checks --- ggml/src/ggml-hexagon/ggml-hexagon.cpp | 87 +++++++++++++++++--------- ggml/src/ggml-hexagon/htp/rope-ops.c | 2 - 2 files changed, 56 insertions(+), 33 deletions(-) diff --git a/ggml/src/ggml-hexagon/ggml-hexagon.cpp b/ggml/src/ggml-hexagon/ggml-hexagon.cpp index 7b9267b9ddd5..effebfb5d625 100644 --- a/ggml/src/ggml-hexagon/ggml-hexagon.cpp +++ b/ggml/src/ggml-hexagon/ggml-hexagon.cpp @@ -4169,7 +4169,7 @@ static void ggml_hexagon_precompute_rope_params( const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, src0_nrows); struct htp_rope_vtcm_layout layout; - htp_rope_vtcm_layout_build(&layout, src0->ne[0], dst->nb[1], n_threads); + htp_rope_vtcm_layout_build(&layout, src0->ne[0], n_threads); kparams->n_threads = n_threads; kparams->src0_nrows = src0_nrows; @@ -4178,7 +4178,6 @@ static void ggml_hexagon_precompute_rope_params( kparams->spad_per_thread = (uint32_t) layout.bytes_per_thread; kparams->theta_cache_offset = (uint32_t) layout.theta_cache_size_aligned; kparams->src0_row_size_aligned = (uint32_t) layout.src0_row_size_aligned; - kparams->dst_row_size_aligned = (uint32_t) layout.dst_row_size_aligned; if (src0_nrows > 0) { kparams->div_ne2_ne1 = init_fastdiv_values(dst->ne[2] * dst->ne[1]); @@ -4744,56 +4743,82 @@ static bool ggml_hexagon_supported_argsort(const struct ggml_hexagon_session * s } static bool ggml_hexagon_supported_rope(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) { - const int32_t * op_params = &op->op_params[0]; + const struct ggml_tensor * src0 = op->src[0]; + const struct ggml_tensor * src1 = op->src[1]; + const struct ggml_tensor * src2 = op->src[2]; + const struct ggml_tensor * dst = op; - // ggml_rope_set_offset: HVX kernels need a VLEN-aligned window start (32 f32 elems) - if (op_params[15] % 32 != 0) { + if (!ggml_are_same_shape(src0, dst)) { return false; } - int mode = op_params[2]; + if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_I32) { + return false; + } - // n_dims == ne0/2, so the rotation spans the full row - if (mode == GGML_ROPE_TYPE_VISION) { - const int n_dims = op_params[1]; - if (n_dims != (int) (op->src[0]->ne[0] / 2)) { - return false; - } + if (src0->ne[0] <= 0) { + return false; } - if (mode & 1) { + + const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; + if (src0_nrows == 0) { return false; } - const struct ggml_tensor * src0 = op->src[0]; - const struct ggml_tensor * src1 = op->src[1]; - const struct ggml_tensor * src2 = op->src[2]; - const struct ggml_tensor * dst = op; + const int32_t * op_params = &op->op_params[0]; + const int n_dims = op_params[1]; + const int mode = op_params[2]; + const int n_offs = op_params[15]; - if (src0->type != GGML_TYPE_F32) { - return false; // FIXME: add support for GGML_TYPE_F16 for src0 + if (n_dims <= 0 || n_dims % 2 != 0) { + return false; } - if (dst->type != GGML_TYPE_F32) { + + // ggml_rope_set_offset: HVX kernels need a VLEN-aligned window start (32 f32 elems) + if (n_offs < 0 || (n_offs % 32 != 0) || (n_offs + n_dims > src0->ne[0])) { return false; } - if (src1->type != GGML_TYPE_I32) { + + float freq_base; + memcpy(&freq_base, op_params + 5, sizeof(float)); + if (freq_base <= 0.0f) { return false; } - if (src2) { - if (src2->type != GGML_TYPE_F32) { + + if (mode != GGML_ROPE_TYPE_NORMAL && + mode != GGML_ROPE_TYPE_NEOX && + mode != GGML_ROPE_TYPE_MROPE && + mode != GGML_ROPE_TYPE_VISION && + mode != GGML_ROPE_TYPE_IMROPE) { + return false; + } + + const bool is_mrope = (mode & GGML_ROPE_TYPE_MROPE) != 0; + + // n_dims == ne0/2, so the rotation spans the full row + if (mode == GGML_ROPE_TYPE_VISION) { + if (n_dims != (int) (src0->ne[0] / 2) || n_offs != 0) { return false; } - int n_dims = op_params[1]; - if (src2->ne[0] < (n_dims / 2)) { + } + + if (is_mrope) { + const int32_t * sections = op_params + 11; + if (sections[0] <= 0 && sections[1] <= 0 && sections[2] <= 0) { return false; } } + const int64_t min_pos_len = (is_mrope || mode == GGML_ROPE_TYPE_VISION) ? src0->ne[2] * 4 : src0->ne[2]; + if (src1->ne[0] < min_pos_len || !ggml_is_contiguous(src1)) { + return false; + } + if (src2) { - if (!ggml_is_contiguous(src1) || !ggml_is_contiguous(src2)) { + if (src2->type != GGML_TYPE_F32 || !ggml_is_contiguous(src2)) { return false; } - } else { - if (!ggml_is_contiguous(src1)) { + if (src2->ne[0] < (n_dims / 2)) { return false; } } @@ -4806,11 +4831,11 @@ static bool ggml_hexagon_supported_rope(const struct ggml_hexagon_session * sess if (src0->nb[1] < src0->ne[0] * sizeof(float) || dst->nb[1] < dst->ne[0] * sizeof(float)) { return false; } - const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3]; - const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, src0_nrows); + + const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, src0_nrows); struct htp_rope_vtcm_layout layout; - htp_rope_vtcm_layout_build(&layout, src0->ne[0], dst->nb[1], n_threads); + htp_rope_vtcm_layout_build(&layout, src0->ne[0], n_threads); if (layout.total_bytes > sess->vtcm_size) { return false; } diff --git a/ggml/src/ggml-hexagon/htp/rope-ops.c b/ggml/src/ggml-hexagon/htp/rope-ops.c index 22c4b82a6484..0a4b31ccb1d8 100644 --- a/ggml/src/ggml-hexagon/htp/rope-ops.c +++ b/ggml/src/ggml-hexagon/htp/rope-ops.c @@ -79,7 +79,6 @@ struct htp_rope_context { size_t dst_row_size; size_t dst_row_stride; size_t src0_row_size_aligned; - size_t dst_row_size_aligned; uint32_t src0_nrows; struct fastdiv_values div_ne2_ne1; @@ -752,7 +751,6 @@ static int execute_op_rope_f32(struct htp_ops_context * octx) { rctx.dst_row_size = dst_row_size; rctx.dst_row_stride = dst_row_stride; rctx.src0_row_size_aligned = kparams->src0_row_size_aligned; - rctx.dst_row_size_aligned = kparams->dst_row_size_aligned; rctx.src0_nrows = kparams->src0_nrows; rctx.src0_nrows_per_thread = kparams->src0_nrows_per_thread; From 7c56eff39174f9b3c806124dc7303cfd25f015fa Mon Sep 17 00:00:00 2001 From: Max Krasnyansky Date: Mon, 7 Sep 2026 17:50:34 -0700 Subject: [PATCH 8/8] hex-rope: add missing ops header --- ggml/src/ggml-hexagon/htp/rope-ops.h | 56 ++++++++++++++++++++++++++++ tests/test-backend-ops.cpp | 1 - 2 files changed, 56 insertions(+), 1 deletion(-) create mode 100644 ggml/src/ggml-hexagon/htp/rope-ops.h diff --git a/ggml/src/ggml-hexagon/htp/rope-ops.h b/ggml/src/ggml-hexagon/htp/rope-ops.h new file mode 100644 index 000000000000..476653d05d2b --- /dev/null +++ b/ggml/src/ggml-hexagon/htp/rope-ops.h @@ -0,0 +1,56 @@ +#ifndef HTP_ROPE_OPS_H +#define HTP_ROPE_OPS_H + +#include "hex-common.h" +#include "hex-fastdiv.h" + +#define HTP_ROPE_SPAD_BLOCK 8 +#define HTP_ROPE_SPAD_NSLOTS 4 +#define HTP_ROPE_SPAD_NROWS (HTP_ROPE_SPAD_BLOCK * HTP_ROPE_SPAD_NSLOTS) + +struct htp_rope_kernel_params { + uint32_t n_threads; + uint32_t src0_nrows; + uint32_t src0_nrows_per_thread; + uint32_t vtcm_size; + uint32_t spad_per_thread; + uint32_t theta_cache_offset; + uint32_t src0_row_size_aligned; + + struct fastdiv_values div_ne2_ne1; + struct fastdiv_values div_ne1; +}; + +#if defined(__cplusplus) +static_assert(sizeof(struct htp_rope_kernel_params) <= 128, "htp_rope_kernel_params is too large for kernel_params blob"); +#else +_Static_assert(sizeof(struct htp_rope_kernel_params) <= 128, "htp_rope_kernel_params is too large for kernel_params blob"); +#endif + +struct htp_rope_vtcm_layout { + size_t total_bytes; + size_t bytes_per_thread; + size_t theta_cache_size_aligned; + size_t src0_row_size_aligned; +}; + +static inline void htp_rope_vtcm_layout_build( + struct htp_rope_vtcm_layout * layout, + uint32_t ne00, + uint32_t n_threads +) { + const size_t src0_row_size = ne00 * sizeof(float); + const size_t src0_row_size_aligned = hex_round_up((uint32_t) src0_row_size, 128); + const size_t theta_cache_size_aligned = hex_round_up((uint32_t) src0_row_size, 256); + + layout->src0_row_size_aligned = src0_row_size_aligned; + layout->theta_cache_size_aligned = theta_cache_size_aligned; + layout->bytes_per_thread = theta_cache_size_aligned + HTP_ROPE_SPAD_NROWS * src0_row_size_aligned; + layout->total_bytes = layout->bytes_per_thread * n_threads; +} + +static inline uint8_t * rope_spad_slot(uint8_t * base, uint32_t slot, size_t row_size_aligned) { + return base + (slot * HTP_ROPE_SPAD_BLOCK) * row_size_aligned; +} + +#endif // HTP_ROPE_OPS_H diff --git a/tests/test-backend-ops.cpp b/tests/test-backend-ops.cpp index a244c84b4f01..0ea54e1563d4 100644 --- a/tests/test-backend-ops.cpp +++ b/tests/test-backend-ops.cpp @@ -10256,7 +10256,6 @@ static std::vector> make_test_cases_eval() { test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {256, 16, 512, 1}, 64, GGML_ROPE_TYPE_IMROPE, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // qwen3.5 4B test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {256, 8, 512, 1}, 256, GGML_ROPE_TYPE_NEOX, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // gemma4 E2B sliding test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {512, 8, 512, 1}, 128, GGML_ROPE_TYPE_NEOX, 512, 1.0f, 0.0f, 1.0f, true, 0, true)); // gemma4 E4B global - test_cases.emplace_back(new test_rope(GGML_TYPE_F32, {128, 32, 512, 1}, 128, GGML_ROPE_TYPE_NORMAL, 512, 1.0f, 0.0f, 1.0f, false, 0, true)); // llama 7B for (int v : { 0, 1, 2, 3 }) { for (int dim : { 0, 1, 2, 3, }) {