Skip to content

hexagon: rope updates #28628

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Merged
max-krasnyansky merged 8 commits into from
Sep 9, 2026
Merged

hexagon: rope updates #28628

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
f3533d2
hexagon: vectorize RoPE theta cache on v75
tboinovski1 Aug 16, 2026
9732ab3
hexagon: vectorize MROPE/IMROPE theta pick
tboinovski1 Aug 16, 2026
ee634ee
hexagon: tighten NEOX RoPE rotate and aligned tail copy
tboinovski1 Aug 28, 2026
900ad6d
hex-rope: use inplace rope for all scenarios
max-krasnyansky Sep 7, 2026
ad65650
hex-rope: remove ctx->spad usage and legacy timers
max-krasnyansky Sep 7, 2026
132f172
hex-rope: add kernel params and enforce vtcm reqs at the host
max-krasnyansky Sep 7, 2026
8d25744
hex-rope: cleanup unused params and tighten the mode checks
max-krasnyansky Sep 8, 2026
7c56eff
hex-rope: add missing ops header
max-krasnyansky Sep 8, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
131 changes: 103 additions & 28 deletions ggml/src/ggml-hexagon/ggml-hexagon.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -56,6 +56,7 @@
#include "htp/unary-ops.h"
#include "htp/get-rows-ops.h"
#include "htp/set-rows-ops.h"
#include "htp/rope-ops.h"
#include "htp_iface.h"
#include "htp-drv.h"

Expand Down Expand Up @@ -299,6 +300,12 @@ static void ggml_hexagon_precompute_set_rows_params(
struct htp_set_rows_kernel_params * kparams
);

static void ggml_hexagon_precompute_rope_params(
const struct ggml_hexagon_session * sess,
const struct ggml_tensor * op,
struct htp_rope_kernel_params * kparams
);

static void ggml_hexagon_precompute_fused_mmnx_params(
const struct ggml_hexagon_session * sess,
const struct ggml_tensor * src0,
Expand Down Expand Up @@ -4148,6 +4155,36 @@ static void ggml_hexagon_precompute_set_rows_params(
kparams->vtcm_size = vtcm_layout.total_bytes;
}

static void ggml_hexagon_precompute_rope_params(
const struct ggml_hexagon_session * sess,
const struct ggml_tensor * op,
struct htp_rope_kernel_params * kparams
) {
memset(kparams, 0, sizeof(*kparams));

const struct ggml_tensor * src0 = op->src[0];
const struct ggml_tensor * dst = op;

const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, src0_nrows);

struct htp_rope_vtcm_layout layout;
htp_rope_vtcm_layout_build(&layout, src0->ne[0], n_threads);

kparams->n_threads = n_threads;
kparams->src0_nrows = src0_nrows;
kparams->src0_nrows_per_thread = (src0_nrows + n_threads - 1) / n_threads;
kparams->vtcm_size = (uint32_t) layout.total_bytes;
kparams->spad_per_thread = (uint32_t) layout.bytes_per_thread;
kparams->theta_cache_offset = (uint32_t) layout.theta_cache_size_aligned;
kparams->src0_row_size_aligned = (uint32_t) layout.src0_row_size_aligned;

if (src0_nrows > 0) {
kparams->div_ne2_ne1 = init_fastdiv_values(dst->ne[2] * dst->ne[1]);
kparams->div_ne1 = init_fastdiv_values(dst->ne[1]);
}
}

static void ggml_hexagon_precompute_fused_mmnx_params(
const struct ggml_hexagon_session * sess,
const struct ggml_tensor * src0, // W0
Expand Down Expand Up @@ -4706,56 +4743,82 @@ static bool ggml_hexagon_supported_argsort(const struct ggml_hexagon_session * s
}

static bool ggml_hexagon_supported_rope(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
const int32_t * op_params = &op->op_params[0];
const struct ggml_tensor * src0 = op->src[0];
const struct ggml_tensor * src1 = op->src[1];
const struct ggml_tensor * src2 = op->src[2];
const struct ggml_tensor * dst = op;

// ggml_rope_set_offset: HVX kernels need a VLEN-aligned window start (32 f32 elems)
if (op_params[15] % 32 != 0) {
if (!ggml_are_same_shape(src0, dst)) {
return false;
}

int mode = op_params[2];
if (src0->type != GGML_TYPE_F32 || dst->type != GGML_TYPE_F32 || src1->type != GGML_TYPE_I32) {
return false;
}

// n_dims == ne0/2, so the rotation spans the full row
if (mode == GGML_ROPE_TYPE_VISION) {
const int n_dims = op_params[1];
if (n_dims != (int) (op->src[0]->ne[0] / 2)) {
return false;
}
if (src0->ne[0] <= 0) {
return false;
}
if (mode & 1) {

const uint32_t src0_nrows = src0->ne[1] * src0->ne[2] * src0->ne[3];
if (src0_nrows == 0) {
return false;
}

const struct ggml_tensor * src0 = op->src[0];
const struct ggml_tensor * src1 = op->src[1];
const struct ggml_tensor * src2 = op->src[2];
const struct ggml_tensor * dst = op;
const int32_t * op_params = &op->op_params[0];
const int n_dims = op_params[1];
const int mode = op_params[2];
const int n_offs = op_params[15];

if (src0->type != GGML_TYPE_F32) {
return false; // FIXME: add support for GGML_TYPE_F16 for src0
if (n_dims <= 0 || n_dims % 2 != 0) {
return false;
}
if (dst->type != GGML_TYPE_F32) {

// ggml_rope_set_offset: HVX kernels need a VLEN-aligned window start (32 f32 elems)
if (n_offs < 0 || (n_offs % 32 != 0) || (n_offs + n_dims > src0->ne[0])) {
return false;
}
if (src1->type != GGML_TYPE_I32) {

float freq_base;
memcpy(&freq_base, op_params + 5, sizeof(float));
if (freq_base <= 0.0f) {
return false;
}
if (src2) {
if (src2->type != GGML_TYPE_F32) {

if (mode != GGML_ROPE_TYPE_NORMAL &&
mode != GGML_ROPE_TYPE_NEOX &&
mode != GGML_ROPE_TYPE_MROPE &&
mode != GGML_ROPE_TYPE_VISION &&
mode != GGML_ROPE_TYPE_IMROPE) {
return false;
}

const bool is_mrope = (mode & GGML_ROPE_TYPE_MROPE) != 0;

// n_dims == ne0/2, so the rotation spans the full row
if (mode == GGML_ROPE_TYPE_VISION) {
if (n_dims != (int) (src0->ne[0] / 2) || n_offs != 0) {
return false;
}
int n_dims = op_params[1];
if (src2->ne[0] < (n_dims / 2)) {
}

if (is_mrope) {
const int32_t * sections = op_params + 11;
if (sections[0] <= 0 && sections[1] <= 0 && sections[2] <= 0) {
return false;
}
}

const int64_t min_pos_len = (is_mrope || mode == GGML_ROPE_TYPE_VISION) ? src0->ne[2] * 4 : src0->ne[2];
if (src1->ne[0] < min_pos_len || !ggml_is_contiguous(src1)) {
return false;
}

if (src2) {
if (!ggml_is_contiguous(src1) || !ggml_is_contiguous(src2)) {
if (src2->type != GGML_TYPE_F32 || !ggml_is_contiguous(src2)) {
return false;
}
} else {
if (!ggml_is_contiguous(src1)) {
if (src2->ne[0] < (n_dims / 2)) {
return false;
}
}
Expand All @@ -4768,9 +4831,16 @@ static bool ggml_hexagon_supported_rope(const struct ggml_hexagon_session * sess
if (src0->nb[1] < src0->ne[0] * sizeof(float) || dst->nb[1] < dst->ne[0] * sizeof(float)) {
return false;
}
return true;

GGML_UNUSED(sess);
const uint32_t n_threads = (std::min)((uint32_t) sess->n_threads, src0_nrows);

struct htp_rope_vtcm_layout layout;
htp_rope_vtcm_layout_build(&layout, src0->ne[0], n_threads);
if (layout.total_bytes > sess->vtcm_size) {
return false;
}

return true;
}

static bool ggml_hexagon_supported_ssm_conv(const struct ggml_hexagon_session * sess, const struct ggml_tensor * op) {
Expand Down Expand Up @@ -5204,6 +5274,11 @@ static ggml_status ggml_backend_hexagon_graph_compute(ggml_backend_t backend, gg
node.node->src[0], node.node->src[1], node.dst(),
(struct htp_set_rows_kernel_params *)node.kernel_params
);
} else if (node.opcode == HTP_OP_ROPE) {
ggml_hexagon_precompute_rope_params(sess,
node.node,
(struct htp_rope_kernel_params *)node.kernel_params
);
}
computed_nodes.push_back(std::move(node));
}
Expand Down
110 changes: 49 additions & 61 deletions ggml/src/ggml-hexagon/htp/hvx-sin-cos.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,87 +4,75 @@
#include "hvx-base.h"
#include "hvx-floor.h"

static inline HVX_Vector hvx_vec_cos_f32(HVX_Vector x) {
HVX_Vector const_inv_pi = hvx_vec_splat_f32(0.3183098861837907f);
HVX_Vector const_half = hvx_vec_splat_f32(0.5f);
HVX_Vector const_pi = hvx_vec_splat_f32(3.141592653589793f);
HVX_Vector const_one = hvx_vec_splat_f32(1.0f);
// Range-reduce x to y in [-pi/2, pi/2] and the quadrant sign (-1)^n.
// Floor/truncate need IEEE bits, so convert qf32 back to sf before them.
static inline void hvx_vec_sincos_reduce_f32(HVX_Vector x, HVX_Vector * y, HVX_Vector * sign) {
HVX_Vector const_inv_pi = hvx_vec_splat_f32(0.3183098861837907f);
HVX_Vector const_half = hvx_vec_splat_f32(0.5f);
HVX_Vector const_pi = hvx_vec_splat_f32(3.141592653589793f);
HVX_Vector const_one = hvx_vec_splat_f32(1.0f);
HVX_Vector const_neg_one = hvx_vec_splat_f32(-1.0f);
HVX_Vector const_one_i = Q6_V_vsplat_R(1);

HVX_Vector x_over_pi = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(x, const_inv_pi));
x_over_pi = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(x_over_pi, const_half));

// n = floor(x * (1/pi) + 0.5)
HVX_Vector n_float = hvx_vec_floor_f32(hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(x, const_inv_pi), const_half));
HVX_Vector n_float = hvx_vec_floor_f32(x_over_pi);
HVX_Vector n_int = hvx_vec_truncate_f32(n_float);

// y = x - n * pi
HVX_Vector y = hvx_vec_sub_f32_f32(x, hvx_vec_mul_f32_f32(n_float, const_pi));
HVX_Vector n_pi = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(n_float, const_pi));
*y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vsub_VsfVsf(x, n_pi));

// Sign determination: if n is odd, sign is -1.0f, else 1.0f
// half_n = n * 0.5f
HVX_Vector half_n = hvx_vec_mul_f32_f32(n_float, const_half);
// floor_half_n = floor(half_n)
HVX_Vector floor_half_n = hvx_vec_floor_f32(half_n);
// is_odd = half_n > floor_half_n
HVX_VectorPred is_odd = Q6_Q_vcmp_gt_VsfVsf(half_n, floor_half_n);
// sign = vmux(is_odd, -1.0f, 1.0f)
HVX_Vector sign = Q6_V_vmux_QVV(is_odd, const_neg_one, const_one);
HVX_VectorPred is_odd = Q6_Q_vcmp_eq_VwVw(Q6_V_vand_VV(n_int, const_one_i), const_one_i);
*sign = Q6_V_vmux_QVV(is_odd, const_neg_one, const_one);
}

// z = y^2
HVX_Vector z = hvx_vec_mul_f32_f32(y, y);
static inline void hvx_vec_sincos_f32(HVX_Vector x, HVX_Vector * vcos, HVX_Vector * vsin) {
HVX_Vector y;
HVX_Vector sign;
hvx_vec_sincos_reduce_f32(x, &y, &sign);

HVX_Vector z = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(y, y));

// Chebyshev approximation for cos(y)
HVX_Vector c4 = hvx_vec_splat_f32(2.3557242013849433e-05f);
HVX_Vector c3 = hvx_vec_splat_f32(-0.0013871428263450528f);
HVX_Vector c2 = hvx_vec_splat_f32(0.041665895266688284f);
HVX_Vector c1 = hvx_vec_splat_f32(-0.4999999360426369f);
HVX_Vector c0 = hvx_vec_splat_f32(0.9999999999071725f);

HVX_Vector cos_y = hvx_vec_add_f32_f32(c3, hvx_vec_mul_f32_f32(z, c4));
cos_y = hvx_vec_add_f32_f32(c2, hvx_vec_mul_f32_f32(z, cos_y));
cos_y = hvx_vec_add_f32_f32(c1, hvx_vec_mul_f32_f32(z, cos_y));
cos_y = hvx_vec_add_f32_f32(c0, hvx_vec_mul_f32_f32(z, cos_y));

return hvx_vec_mul_f32_f32(cos_y, sign);
}

static inline HVX_Vector hvx_vec_sin_f32(HVX_Vector x) {
HVX_Vector const_inv_pi = hvx_vec_splat_f32(0.3183098861837907f);
HVX_Vector const_half = hvx_vec_splat_f32(0.5f);
HVX_Vector const_pi = hvx_vec_splat_f32(3.141592653589793f);
HVX_Vector const_one = hvx_vec_splat_f32(1.0f);
HVX_Vector const_neg_one = hvx_vec_splat_f32(-1.0f);

// n = floor(x * (1/pi) + 0.5)
HVX_Vector n_float = hvx_vec_floor_f32(hvx_vec_add_f32_f32(hvx_vec_mul_f32_f32(x, const_inv_pi), const_half));
HVX_Vector cos_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(c3, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, c4))));
cos_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(c2, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, cos_y))));
cos_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(c1, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, cos_y))));
cos_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(c0, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, cos_y))));

// y = x - n * pi
HVX_Vector y = hvx_vec_sub_f32_f32(x, hvx_vec_mul_f32_f32(n_float, const_pi));

// Sign determination: if n is odd, sign is -1.0f, else 1.0f
// half_n = n * 0.5f
HVX_Vector half_n = hvx_vec_mul_f32_f32(n_float, const_half);
// floor_half_n = floor(half_n)
HVX_Vector floor_half_n = hvx_vec_floor_f32(half_n);
// is_odd = half_n > floor_half_n
HVX_VectorPred is_odd = Q6_Q_vcmp_gt_VsfVsf(half_n, floor_half_n);
// sign = vmux(is_odd, -1.0f, 1.0f)
HVX_Vector sign = Q6_V_vmux_QVV(is_odd, const_neg_one, const_one);

// z = y^2
HVX_Vector z = hvx_vec_mul_f32_f32(y, y);

// Chebyshev approximation for sin(y)
HVX_Vector s4 = hvx_vec_splat_f32(2.642186986152672e-06f);
HVX_Vector s3 = hvx_vec_splat_f32(-0.00019825318964070864f);
HVX_Vector s2 = hvx_vec_splat_f32(0.00833326283319605f);
HVX_Vector s1 = hvx_vec_splat_f32(-0.16666666082087775f);
HVX_Vector s0 = hvx_vec_splat_f32(0.999999999915155f);

HVX_Vector sin_y = hvx_vec_add_f32_f32(s3, hvx_vec_mul_f32_f32(z, s4));
sin_y = hvx_vec_add_f32_f32(s2, hvx_vec_mul_f32_f32(z, sin_y));
sin_y = hvx_vec_add_f32_f32(s1, hvx_vec_mul_f32_f32(z, sin_y));
sin_y = hvx_vec_add_f32_f32(s0, hvx_vec_mul_f32_f32(z, sin_y));
sin_y = hvx_vec_mul_f32_f32(y, sin_y);
HVX_Vector sin_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(s3, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, s4))));
sin_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(s2, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, sin_y))));
sin_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(s1, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, sin_y))));
sin_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vadd_VsfVsf(s0, Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(z, sin_y))));
sin_y = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(y, sin_y));

*vcos = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(cos_y, sign));
*vsin = Q6_Vsf_equals_Vqf32(Q6_Vqf32_vmpy_VsfVsf(sin_y, sign));
}

return hvx_vec_mul_f32_f32(sin_y, sign);
static inline HVX_Vector hvx_vec_cos_f32(HVX_Vector x) {
HVX_Vector vcos;
HVX_Vector vsin;
hvx_vec_sincos_f32(x, &vcos, &vsin);
return vcos;
}

static inline HVX_Vector hvx_vec_sin_f32(HVX_Vector x) {
HVX_Vector vcos;
HVX_Vector vsin;
hvx_vec_sincos_f32(x, &vcos, &vsin);
return vsin;
}

#endif /* HVX_SIN_COS_H */
Loading
Loading