Skip to content

ggml : add native INT8 convrot support #26914

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Open
leejet wants to merge 2 commits into
base: master
Choose a base branch
from
+1,555 −6
Open

ggml : add native INT8 convrot support #26914

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
2 commits
Select commit Hold shift + click to select a range
1679f8d
ggml : add native INT8 convrot support
leejet Aug 11, 2026
1a41cef
ggml : remove standalone regular Hadamard op
leejet Aug 12, 2026
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
4 changes: 2 additions & 2 deletions ggml/include/ggml-rpc.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -8,10 +8,10 @@ extern "C" {

#define RPC_PROTO_MAJOR_VERSION 5
#define RPC_PROTO_MINOR_VERSION 0
#define RPC_PROTO_PATCH_VERSION 0
#define RPC_PROTO_PATCH_VERSION 1

#ifdef __cplusplus
static_assert(GGML_OP_COUNT == 101, "GGML_OP_COUNT has changed - update RPC_PROTO_PATCH_VERSION");
static_assert(GGML_OP_COUNT == 102, "GGML_OP_COUNT has changed - update RPC_PROTO_PATCH_VERSION");
#endif

#define GGML_RPC_MAX_SERVERS 16
Expand Down
16 changes: 16 additions & 0 deletions ggml/include/ggml.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -590,6 +590,8 @@ extern "C" {

GGML_OP_GLU,

GGML_OP_QUANTIZE_I8_CONVROT,

GGML_OP_COUNT,
};

Expand Down Expand Up @@ -1430,6 +1432,14 @@ extern "C" {
struct ggml_tensor * a,
struct ggml_tensor * b);

GGML_API struct ggml_tensor * ggml_mul_mat_i8_tensorwise(
struct ggml_context * ctx,
struct ggml_tensor * weight,
struct ggml_tensor * input,
struct ggml_tensor * weight_scale,
struct ggml_tensor * bias,
int convrot_group_size);

// change the precision of a matrix multiplication
// set to GGML_PREC_F32 for higher precision (useful for phi-2)
GGML_API void ggml_mul_mat_set_prec(
Expand All @@ -1441,6 +1451,12 @@ extern "C" {
struct ggml_tensor * a,
enum ggml_op_hint hint);

// Packs row-wise I8 activations followed by one F32 scale per row.
GGML_API struct ggml_tensor * ggml_quantize_i8_convrot(
struct ggml_context * ctx,
struct ggml_tensor * a,
int group_size);

Comment on lines +1455 to +1459

@ggerganov ggerganov Sep 3, 2026

Copy link
Copy Markdown
Member

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I'm not 100% sure that the API is the best way. Can this be a ggml_cast + a convrot hint, without introducing GGML_OP_QUANTIZE_I8_CONVROT?

// indirect matrix multiplication
GGML_API struct ggml_tensor * ggml_mul_mat_id(
struct ggml_context * ctx,
Expand Down
3 changes: 3 additions & 0 deletions ggml/src/ggml-backend-meta.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1009,6 +1009,9 @@ static struct ggml_backend_meta_split_state ggml_backend_meta_get_split_state(
case GGML_OP_GLU: {
split_state = handle_generic(src_ss, /*scalar_only =*/ false);
} break;
case GGML_OP_QUANTIZE_I8_CONVROT: {
split_state = handle_generic(src_ss, /*scalar_only =*/ true);
} break;
default: {
GGML_ABORT("ggml op not implemented: %s", ggml_op_name(tensor->op));
split_state = {GGML_BACKEND_SPLIT_AXIS_UNKNOWN, {0}, {1}, 1};
Expand Down
158 changes: 158 additions & 0 deletions ggml/src/ggml-cpu/ggml-cpu.c
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -1161,6 +1161,147 @@ void ggml_set_f32_nd(const struct ggml_tensor * tensor, int i0, int i1, int i2,

// ggml_compute_forward_mul_mat

static void ggml_regular_hadamard_group_f32(float * values, int group_size) {
const float scale = 1.0f / sqrtf((float) group_size);
for (int i = 0; i < group_size; ++i) {
values[i] *= scale;
}

for (int stride = 1; stride < group_size; stride *= 4) {
for (int base = 0; base < group_size; base += 4 * stride) {
for (int j = 0; j < stride; ++j) {
const int i0 = base + j;
const int i1 = i0 + stride;
const int i2 = i1 + stride;
const int i3 = i2 + stride;
const float a = values[i0];
const float b = values[i1];
const float c = values[i2];
const float d = values[i3];
values[i0] = a + b + c - d;
values[i1] = a + b - c + d;
values[i2] = a - b + c + d;
values[i3] = -a + b + c + d;
}
}
}
}

static void ggml_compute_forward_mul_mat_i8_f32(
const struct ggml_compute_params * params,
struct ggml_tensor * dst) {
const struct ggml_tensor * src0 = dst->src[0];
const struct ggml_tensor * src1 = dst->src[1];

GGML_ASSERT(src0->type == GGML_TYPE_I8);
GGML_ASSERT(src1->type == GGML_TYPE_F32 || src1->type == GGML_TYPE_I8);
GGML_ASSERT(dst->type == GGML_TYPE_F32);
GGML_ASSERT(src0->ne[2] == 1 && src0->ne[3] == 1);
GGML_ASSERT(ggml_is_contiguous(src0));
GGML_ASSERT(ggml_is_contiguous(src1));
GGML_ASSERT(ggml_is_contiguous(dst));

const int64_t k = src0->ne[0];
const int64_t n = src0->ne[1];
const int64_t rows = ggml_nrows(dst);
const bool prequantized = src1->type == GGML_TYPE_I8;
const int64_t rows_padded = GGML_PAD(rows, 4);
const int64_t scale_rows = (rows * (int64_t) sizeof(float) + k - 1) / k;
const int convrot_group_size = ggml_get_op_params_i32(dst, 2);
const size_t qbytes = prequantized ? 0 : (size_t) ggml_nelements(src1) * sizeof(int8_t);
const size_t scale_offset = GGML_PAD(qbytes, sizeof(float));

GGML_ASSERT(src1->ne[0] == k);
GGML_ASSERT(!prequantized || (src1->op == GGML_OP_QUANTIZE_I8_CONVROT && src1->ne[1] == rows_padded + scale_rows));
GGML_ASSERT(convrot_group_size == 0 || (convrot_group_size <= 256 && k % convrot_group_size == 0));
GGML_ASSERT(prequantized || params->wsize >= scale_offset + (size_t) rows * sizeof(float));

int hadamard_size = convrot_group_size;
while (hadamard_size > 1 && hadamard_size % 4 == 0) {
hadamard_size /= 4;
}
GGML_ASSERT(hadamard_size == 0 || hadamard_size == 1);

int8_t * qdata = prequantized ? (int8_t *) src1->data : (int8_t *) params->wdata;
float * scales = prequantized ? NULL : (float *) ((char *) params->wdata + scale_offset);
const float * packed_scales = prequantized ? (const float *) ((const int8_t *) src1->data + k * rows_padded) : NULL;
const float * src1_data = prequantized ? NULL : (const float *) src1->data;

for (int64_t row = params->ith; !prequantized && row < rows; row += params->nth) {
const float * src_row = src1_data + row * k;
int8_t * qrow = qdata + row * k;
float amax = 0.0f;
if (convrot_group_size == 0) {
for (int64_t i = 0; i < k; ++i) {
amax = MAX(amax, fabsf(src_row[i]));
}
} else {
float values[256];
for (int64_t group = 0; group < k; group += convrot_group_size) {
memcpy(values, src_row + group, (size_t) convrot_group_size * sizeof(float));
ggml_regular_hadamard_group_f32(values, convrot_group_size);
for (int i = 0; i < convrot_group_size; ++i) {
amax = MAX(amax, fabsf(values[i]));
}
}
}

const float row_scale = amax / 127.0f;
scales[row] = row_scale;
if (row_scale == 0.0f) {
memset(qrow, 0, (size_t) k);
continue;
}

const float inv_scale = 1.0f / row_scale;
if (convrot_group_size == 0) {
for (int64_t i = 0; i < k; ++i) {
int value = (int) lrintf(src_row[i] * inv_scale);
value = MAX(-127, MIN(127, value));
qrow[i] = (int8_t) value;
}
} else {
float values[256];
for (int64_t group = 0; group < k; group += convrot_group_size) {
memcpy(values, src_row + group, (size_t) convrot_group_size * sizeof(float));
ggml_regular_hadamard_group_f32(values, convrot_group_size);
for (int i = 0; i < convrot_group_size; ++i) {
int value = (int) lrintf(values[i] * inv_scale);
value = MAX(-127, MIN(127, value));
qrow[group + i] = (int8_t) value;
}
}
}
}

if (!prequantized) {
ggml_barrier(params->threadpool);
}

float * dst_data = (float *) dst->data;
const int64_t output_elements = n * rows;
for (int64_t index = params->ith; index < output_elements; index += params->nth) {
const int64_t output_row = index / n;
const int64_t i01 = index - output_row * n;
const int8_t * weight_row = (const int8_t *) ((const char *) src0->data + i01 * src0->nb[1]);
const int8_t * activation_row = prequantized ? (const int8_t *) ((const char *) src1->data + output_row * src1->nb[1]) : qdata + output_row * k;

int32_t sum = 0;
for (int64_t i = 0; i < k; ++i) {
sum += (int32_t) weight_row[i] * (int32_t) activation_row[i];
}
const float activation_scale = prequantized ? packed_scales[output_row] : scales[output_row];
float value = (float) sum * activation_scale;
if (dst->src[2] != NULL) {
value *= ((const float *) dst->src[2]->data)[i01];
}
if (dst->src[3] != NULL) {
value += ((const float *) dst->src[3]->data)[i01];
}
dst_data[index] = value;
}
}

static void ggml_compute_forward_mul_mat_one_chunk(
const struct ggml_compute_params * params,
struct ggml_tensor * dst,
Expand Down Expand Up @@ -1258,6 +1399,13 @@ void ggml_compute_forward_mul_mat(
const struct ggml_tensor * src0 = dst->src[0];
const struct ggml_tensor * src1 = dst->src[1];

if (src0->type == GGML_TYPE_I8 &&
(src1->type == GGML_TYPE_F32 ||
(src1->type == GGML_TYPE_I8 && src1->op == GGML_OP_QUANTIZE_I8_CONVROT))) {
ggml_compute_forward_mul_mat_i8_f32(params, dst);
return;
}

const int32_t hint = ggml_get_op_params_i32(dst, 1);
if (hint == GGML_HINT_SRC0_IS_HADAMARD && !params->use_ref) {
ggml_compute_forward_fwht(params, dst);
Expand Down Expand Up @@ -2032,6 +2180,10 @@ static void ggml_compute_forward(struct ggml_compute_params * params, struct ggm
{
ggml_compute_forward_glu(params, tensor);
} break;
case GGML_OP_QUANTIZE_I8_CONVROT:
{
ggml_compute_forward_quantize_i8_convrot(params, tensor);
} break;
case GGML_OP_GET_REL_POS:
{
ggml_compute_forward_get_rel_pos(params, tensor);
Expand Down Expand Up @@ -2330,6 +2482,7 @@ static int ggml_get_n_tasks(struct ggml_tensor * node, int n_threads) {
case GGML_OP_MUL_MAT:
case GGML_OP_MUL_MAT_ID:
case GGML_OP_OUT_PROD:
case GGML_OP_QUANTIZE_I8_CONVROT:
{
n_tasks = n_threads;
} break;
Expand Down Expand Up @@ -2847,6 +3000,11 @@ struct ggml_cplan ggml_graph_plan(
} break;
case GGML_OP_MUL_MAT:
{
if (node->src[0]->type == GGML_TYPE_I8 && node->src[1]->type == GGML_TYPE_F32) {
const size_t qbytes = (size_t) ggml_nelements(node->src[1]) * sizeof(int8_t);
cur = GGML_PAD(qbytes, sizeof(float)) + (size_t) ggml_nrows(node->src[1]) * sizeof(float);
break;
}
const enum ggml_type vec_dot_type = type_traits_cpu[node->src[0]->type].vec_dot_type;

if (node->src[1]->type != vec_dot_type) {
Expand Down
38 changes: 38 additions & 0 deletions ggml/src/ggml-cpu/ggml-cpu.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -451,7 +451,45 @@ static bool ggml_backend_cpu_device_supports_op(ggml_backend_dev_t dev, const st
op->type != GGML_TYPE_IQ1_S &&
op->type != GGML_TYPE_IQ1_M; // missing type_traits.from_float
case GGML_OP_MUL_MAT:
if (src0->type == GGML_TYPE_I8) {
const ggml_tensor * weight_scale = op->src[2];
const ggml_tensor * bias = op->src[3];
const int convrot_group_size = ggml_get_op_params_i32(op, 2);
const ggml_tensor * packed_src = src1->src[0];
const int64_t packed_rows = packed_src != nullptr ? GGML_PAD(ggml_nrows(packed_src), 4) : 0;
const int64_t packed_scale_rows = packed_src != nullptr ? (ggml_nrows(packed_src) * (int64_t) sizeof(float) + src0->ne[0] - 1) / src0->ne[0] : 0;
const bool packed_input = src1->type == GGML_TYPE_I8 &&
src1->op == GGML_OP_QUANTIZE_I8_CONVROT && packed_src != nullptr &&
src1->ne[0] == src0->ne[0] && src1->ne[1] == packed_rows + packed_scale_rows;

int hadamard_size = convrot_group_size;
while (hadamard_size > 1 && hadamard_size % 4 == 0) {
hadamard_size /= 4;
}

return (src1->type == GGML_TYPE_F32 || packed_input) &&
op->type == GGML_TYPE_F32 && src0->ne[2] == 1 && src0->ne[3] == 1 &&
ggml_is_contiguous(src0) && ggml_is_contiguous(src1) && ggml_is_contiguous(op) &&
(weight_scale == nullptr ||
(weight_scale->type == GGML_TYPE_F32 && ggml_is_contiguous(weight_scale) && ggml_nelements(weight_scale) == src0->ne[1])) &&
(bias == nullptr ||
(bias->type == GGML_TYPE_F32 && ggml_is_contiguous(bias) && ggml_nelements(bias) == src0->ne[1])) &&
(convrot_group_size == 0 ||
(convrot_group_size <= 256 && src0->ne[0] % convrot_group_size == 0 && hadamard_size == 1));
}
return src1->type == GGML_TYPE_F32 || src1->type == ggml_get_type_traits_cpu(src0->type)->vec_dot_type;
case GGML_OP_QUANTIZE_I8_CONVROT: {
int group_size = ggml_get_op_params_i32(op, 0);
if (src0->type != GGML_TYPE_F32 || op->type != GGML_TYPE_I8 ||
!ggml_is_contiguous(src0) || !ggml_is_contiguous(op) ||
group_size <= 0 || group_size > 256 || src0->ne[0] % group_size != 0) {
return false;
}
while (group_size > 1 && group_size % 4 == 0) {
group_size /= 4;
}
return group_size == 1;
}
case GGML_OP_SOFT_MAX_BACK: {
if (op->src[0]->type != GGML_TYPE_F32 || op->src[1]->type != GGML_TYPE_F32) {
return false;
Expand Down
93 changes: 93 additions & 0 deletions ggml/src/ggml-cpu/ops.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -11919,6 +11919,99 @@ void ggml_compute_forward_fwht(const ggml_compute_params * params, ggml_tensor *
}
}

void ggml_compute_forward_quantize_i8_convrot(const ggml_compute_params * params, ggml_tensor * dst) {
const ggml_tensor * src = dst->src[0];

GGML_ASSERT(src->type == GGML_TYPE_F32);
GGML_ASSERT(dst->type == GGML_TYPE_I8);
GGML_ASSERT(ggml_is_contiguous(src));
GGML_ASSERT(ggml_is_contiguous(dst));

const int group_size = ggml_get_op_params_i32(dst, 0);
const int64_t k = src->ne[0];
const int64_t rows = ggml_nrows(src);
const int64_t rows_padded = GGML_PAD(rows, 4);
const int64_t scale_rows = (rows * (int64_t) sizeof(float) + k - 1) / k;
const float transform_scale = 1.0f / sqrtf((float) group_size);
GGML_ASSERT(group_size > 0 && group_size <= 256 && k % group_size == 0);
GGML_ASSERT(dst->ne[0] == k && dst->ne[1] == rows_padded + scale_rows);

const float * src_data = (const float *) src->data;
char * dst_data = (char *) dst->data;
float * dst_scales = (float *) (dst_data + k * rows_padded);

for (int64_t row = params->ith; row < rows; row += params->nth) {
const float * src_row = src_data + row * k;
int8_t * dst_row = (int8_t *) (dst_data + row * dst->nb[1]);
float amax = 0.0f;
float values[256];

for (int64_t group = 0; group < k; group += group_size) {
for (int i = 0; i < group_size; ++i) {
values[i] = src_row[group + i] * transform_scale;
}
for (int stride = 1; stride < group_size; stride *= 4) {
for (int base = 0; base < group_size; base += 4 * stride) {
for (int j = 0; j < stride; ++j) {
const int i0 = base + j;
const int i1 = i0 + stride;
const int i2 = i1 + stride;
const int i3 = i2 + stride;
const float a = values[i0];
const float b = values[i1];
const float c = values[i2];
const float d = values[i3];
values[i0] = a + b + c - d;
values[i1] = a + b - c + d;
values[i2] = a - b + c + d;
values[i3] = -a + b + c + d;
}
}
}
for (int i = 0; i < group_size; ++i) {
amax = MAX(amax, fabsf(values[i]));
}
}

const float row_scale = amax / 127.0f;
const float inv_scale = row_scale == 0.0f ? 0.0f : 1.0f / row_scale;
dst_scales[row] = row_scale;

for (int64_t group = 0; group < k; group += group_size) {
for (int i = 0; i < group_size; ++i) {
values[i] = src_row[group + i] * transform_scale;
}
for (int stride = 1; stride < group_size; stride *= 4) {
for (int base = 0; base < group_size; base += 4 * stride) {
for (int j = 0; j < stride; ++j) {
const int i0 = base + j;
const int i1 = i0 + stride;
const int i2 = i1 + stride;
const int i3 = i2 + stride;
const float a = values[i0];
const float b = values[i1];
const float c = values[i2];
const float d = values[i3];
values[i0] = a + b + c - d;
values[i1] = a + b - c + d;
values[i2] = a - b + c + d;
values[i3] = -a + b + c + d;
}
}
}
for (int i = 0; i < group_size; ++i) {
int value = row_scale == 0.0f ? 0 : (int) lrintf(values[i] * inv_scale);
value = MAX(-127, MIN(127, value));
dst_row[group + i] = (int8_t) value;
}
}
}

if (params->ith == 0 && rows_padded > rows) {
memset(dst_data + rows * dst->nb[1], 0, (size_t) (rows_padded - rows) * dst->nb[1]);
}
}

// ggml_compute_forward_lightning_indexer

void ggml_compute_forward_lightning_indexer(
Expand Down
Loading