diff --git a/docs/backend/OPENVINO.md b/docs/backend/OPENVINO.md index 2c0e06f3cd91..d41cec4c7c7a 100644 --- a/docs/backend/OPENVINO.md +++ b/docs/backend/OPENVINO.md @@ -734,6 +734,7 @@ Boolean flags follow a uniform convention: set to a **positive integer** (e.g. ` | `GGML_OPENVINO_PROFILING` | Integer | `0` | `1` logs execution timing; `2` or higher also enables OpenVINO and OpenCL profiling. | | `GGML_OPENVINO_DEBUG_NODE` | String | `not set` | Add the named graph nodes as compiled outputs for debugging. Separate multiple names with commas. | | `GGML_OPENVINO_MOE_OP` | Boolean | `1` | On GPU, set to `0` to keep the unfused GatherMatmul path. | +| `GGML_OPENVINO_EXACT_ZERO_POINT` | Boolean | `0` | Keep the exact zero point (`min / scale`, f16) of asymmetric weights that are not requantized (Q4_1, Q4_K, Q5_1, Q5_K) instead of rounding it to an integer. Improves accuracy. On GPU, use it only with an OpenVINO version whose oneDNN fully-connected path accepts f16 zero points; older versions run these weights on a slower kernel. MoE expert weights always use the exact zero point. | | `GGML_OPENVINO_DUMP_CGRAPH` | Boolean | `0` | Dump the GGML compute graph to `cgraph_ov.txt`. | | `GGML_OPENVINO_DUMP_IR` | Boolean | `0` | Serialize OpenVINO IR files with timestamps. | | `GGML_OPENVINO_DEBUG_INPUT` | Boolean | `0` | Enable input debugging and print input tensor info. | diff --git a/ggml/src/ggml-openvino/README.md b/ggml/src/ggml-openvino/README.md index a12233d2aa17..b9067e76c0ff 100644 --- a/ggml/src/ggml-openvino/README.md +++ b/ggml/src/ggml-openvino/README.md @@ -29,7 +29,7 @@ GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY=1 \ Cache-only mode allocates backend address space without filling weight pages. On Windows, this also uses system commit capacity. The model-buffer size in the loader log is this virtual size. Weight uploads only record source identity; they do not read or requantize the weights. Graph conversion and compilation are skipped. Runtime buffers are still allocated and populated normally. -Cache-only mode uses the settings provided by the current process. Keep these values exactly the same, including set versus unset: `GGML_OPENVINO_REQUANT_KQUANT`, `GGML_OPENVINO_NATIVE_SOFTPLUS`, `GGML_OPENVINO_DISABLE_KV_SLICE`, `GGML_OPENVINO_MANUAL_GQA_ATTN`, `GGML_OPENVINO_STATEFUL_EXECUTION`, `GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT`, `GGML_OPENVINO_DISABLE_REMOTE_OUTPUTS`, `GGML_OPENVINO_REDUCE_COMPILE_MEM`, `GGML_OPENVINO_MEMORY_OPTIMIZE`, `GGML_OPENVINO_PROFILING`, `GGML_OPENVINO_DISABLE_SHAPE_FROM_MASK`, and `GGML_OPENVINO_DEBUG_NODE`. On GPU, also repeat `GGML_OPENVINO_MOE_OP=0` if used. `GGML_OPENVINO_SPILL_DIR` is optional on the first run and ignored in cache-only mode; host-weight release is disabled in cache-only mode. +Cache-only mode uses the settings provided by the current process. Keep these values exactly the same, including set versus unset: `GGML_OPENVINO_REQUANT_KQUANT`, `GGML_OPENVINO_EXACT_ZERO_POINT`, `GGML_OPENVINO_NATIVE_SOFTPLUS`, `GGML_OPENVINO_DISABLE_KV_SLICE`, `GGML_OPENVINO_MANUAL_GQA_ATTN`, `GGML_OPENVINO_STATEFUL_EXECUTION`, `GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT`, `GGML_OPENVINO_DISABLE_REMOTE_OUTPUTS`, `GGML_OPENVINO_REDUCE_COMPILE_MEM`, `GGML_OPENVINO_MEMORY_OPTIMIZE`, `GGML_OPENVINO_PROFILING`, `GGML_OPENVINO_DISABLE_SHAPE_FROM_MASK`, and `GGML_OPENVINO_DEBUG_NODE`. On GPU, also repeat `GGML_OPENVINO_MOE_OP=0` if used. `GGML_OPENVINO_SPILL_DIR` is optional on the first run and ignored in cache-only mode; host-weight release is disabled in cache-only mode. A missing or incompatible graph fails with an error instead of compiling with absent weights. The fingerprint uses the dynamic graph's topology, ports, model parameters, weights, settings, and OpenVINO version; changing only dynamic token or KV sizes does not require a new entry. A different workload can still require another graph; populate it first without cache-only mode. diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp index 1c631ae5a5d7..3a30a546f5f0 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp @@ -134,6 +134,7 @@ void ggml_openvino_device_config::init() { "GGML_OPENVINO_MANUAL_GQA_ATTN", "GGML_OPENVINO_DISABLE_ELTWISE_RANK_ALIGN", "GGML_OPENVINO_MOE_OP", + "GGML_OPENVINO_EXACT_ZERO_POINT", "GGML_OPENVINO_MEMORY_OPTIMIZE", "GGML_OPENVINO_RELEASE_WEIGHTS", "GGML_OPENVINO_REDUCE_COMPILE_MEM", @@ -455,6 +456,10 @@ std::optional ggml_openvino_get_requant_type(const ggml_tensor * } } +bool ggml_openvino_use_exact_zero_point(const ggml_tensor * tensor) { + return tensor->ne[2] > 1 || ggml_openvino_getenv_int("GGML_OPENVINO_EXACT_ZERO_POINT") > 0; +} + // ===================================================== // Extracted Layout Calculation // ===================================================== @@ -474,11 +479,9 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten return layout; } - // 3D MoE expert weights that are not requantized (see below) always use the exact f16 - // zero-point extraction (see extract_quantized_weights), which needs a wider zp slot than - // the packed integer zero point -- must be kept in sync with that function so the buffer - // sizing here matches what process_weight_tensor actually writes. - const bool for_gather_matmul = tensor->ne[2] > 1; + // An exact zero point (see extract_quantized_weights) needs an f16 zp slot instead of the + // packed integer one. Keep in sync with process_weight_tensor, which fills the slot. + const bool exact_zp = ggml_openvino_use_exact_zero_point(tensor); int64_t n_elements = ggml_nelements(tensor); const size_t alignment = 64; // Good for SIMD @@ -618,13 +621,11 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten // Scales: F16 per block, except MXFP4 which stores one E8M0 byte per block. int64_t n_blocks = n_elements / layout.weights_per_block; layout.scales_size = n_blocks * (tensor->type == GGML_TYPE_MXFP4 ? sizeof(uint8_t) : sizeof(uint16_t)); - // For symmetric quantization, no zp needed (weights stored as signed). Asymmetric - // for_gather_matmul (3D MoE expert) weights use an exact f16 zero point (see - // extract_quantized_weights/make_int8_weights/make_int4_weights), which needs one f16 per - // block instead of a packed u4/u8 integer zero point. + // For symmetric quantization, no zp needed (weights stored as signed). An exact zero point + // needs one f16 per block instead of a packed u4/u8 integer zero point. if (layout.is_symmetric) { layout.zp_size = 0; - } else if (use_bias || for_gather_matmul) { + } else if (use_bias || exact_zp) { layout.zp_size = n_blocks * sizeof(uint16_t); } else { layout.zp_size = layout.is_u4 ? ((n_blocks + 1) / 2) : n_blocks; diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.h b/ggml/src/ggml-openvino/ggml-openvino-extra.h index 9b42ac721068..cb0766eb4b21 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.h +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.h @@ -137,6 +137,10 @@ bool ggml_openvino_weight_buffers_released(); // Get requantization type for a tensor type (returns nullopt if no requant needed) std::optional ggml_openvino_get_requant_type(const ggml_tensor * tensor, bool no_requant = false); +// True if an asymmetric weight keeps an exact f16 zero point instead of a rounded integer one. +// Always true for 3D MoE expert weights. Other weights need GGML_OPENVINO_EXACT_ZERO_POINT. +bool ggml_openvino_use_exact_zero_point(const ggml_tensor * tensor); + // ===================================================== // OpenVINO Tensor Extra Types // ===================================================== diff --git a/ggml/src/ggml-openvino/ggml-quants.cpp b/ggml/src/ggml-openvino/ggml-quants.cpp index f4e21eac5595..bb84b3a9e880 100644 --- a/ggml/src/ggml-openvino/ggml-quants.cpp +++ b/ggml/src/ggml-openvino/ggml-quants.cpp @@ -1091,12 +1091,11 @@ std::shared_ptr extract_quantized_weights(const ggml_tensor * tensor, std::string(ggml_type_name(tensor->type))); } - // 3D MoE expert weights (for_gather_matmul) always use the exact f16 zero-point extraction - // (see make_int8_weights/make_int4_weights) rather than the rounded integer zero point -- - // round(min/scale) error is what corrupts Q4_K/Q5_1 experts, and the f16-zp form still fuses - // into GatherMatmulCompressed since it stays a Subtract, not an Add. + // An exact f16 zero point avoids the round(min/scale) error of the integer one, which corrupts + // Q4_K/Q5_1 MoE experts and costs accuracy on dense weights. It stays a Subtract, so the + // weights still fuse into GatherMatmulCompressed / compressed FullyConnected. const bool for_gather_matmul = tensor->ne[2] > 1; - use_bias = use_bias || for_gather_matmul; + use_bias = use_bias || ggml_openvino_use_exact_zero_point(tensor); // Extract quantized data switch (tensor->type) { @@ -1294,13 +1293,9 @@ OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, vo OPENVINO_THROW("Unsupported quantized type: ", ggml_type_name(tensor->type)); } - // 3D MoE expert weights (for_gather_matmul) always use the exact f16 zero-point path (see - // extract_quantized_weights) -- must be kept in sync with the "use_bias || for_gather_matmul" - // check in ggml_openvino_get_extracted_layout, which sizes/offsets the zp slot accordingly. - // Requantized tensors (layout.is_requant) are handled by requantize_to_buffers instead, whose - // zp sizing/type is unaffected by for_gather_matmul, so they are excluded here. - const bool for_gather_matmul = tensor->ne[2] > 1; - const bool zp_is_f16 = !layout.is_requant && (use_bias || for_gather_matmul); + // Keep in sync with ggml_openvino_get_extracted_layout, which sizes the zp slot. Requantized + // tensors are handled by requantize_to_buffers and keep an integer zero point. + const bool zp_is_f16 = !layout.is_requant && (use_bias || ggml_openvino_use_exact_zero_point(tensor)); const bool is_3d_mxfp4_moe = tensor->type == GGML_TYPE_MXFP4 && (tensor->ne[2] > 1 || tensor->ne[3] > 1); if (is_3d_mxfp4_moe) { diff --git a/ggml/src/ggml-openvino/model-cache.cpp b/ggml/src/ggml-openvino/model-cache.cpp index 278f62a25a52..4ddd893eadb0 100644 --- a/ggml/src/ggml-openvino/model-cache.cpp +++ b/ggml/src/ggml-openvino/model-cache.cpp @@ -169,6 +169,7 @@ static const char * cache_settings[] = { "GGML_OPENVINO_NATIVE_SOFTPLUS", "GGML_OPENVINO_DISABLE_KV_SLICE", "GGML_OPENVINO_MANUAL_GQA_ATTN", + "GGML_OPENVINO_EXACT_ZERO_POINT", "GGML_OPENVINO_STATEFUL_EXECUTION", "GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT", "GGML_OPENVINO_DISABLE_REMOTE_OUTPUTS",