ggml-openvino: add opt-in exact zero points for asymmetric weights

docs/backend/OPENVINO.md

@@ -734,6 +734,7 @@ Boolean flags follow a uniform convention: set to a **positive integer** (e.g. `
 | `GGML_OPENVINO_PROFILING`         | Integer   | `0`        | `1` logs execution timing; `2` or higher also enables OpenVINO and OpenCL profiling.                        |
 | `GGML_OPENVINO_DEBUG_NODE`       | String    | `not set`  | Add the named graph nodes as compiled outputs for debugging. Separate multiple names with commas.          |
 | `GGML_OPENVINO_MOE_OP`           | Boolean   | `1`        | On GPU, set to `0` to keep the unfused GatherMatmul path.                                                  |
+| `GGML_OPENVINO_EXACT_ZERO_POINT`  | Boolean   | `0`        | Keep the exact zero point (`min / scale`, f16) of asymmetric weights that are not requantized (Q4_1, Q4_K, Q5_1, Q5_K) instead of rounding it to an integer. Improves accuracy. On GPU, use it only with an OpenVINO version whose oneDNN fully-connected path accepts f16 zero points; older versions run these weights on a slower kernel. MoE expert weights always use the exact zero point. |
 | `GGML_OPENVINO_DUMP_CGRAPH`       | Boolean   | `0`        | Dump the GGML compute graph to `cgraph_ov.txt`.                                                             |
 | `GGML_OPENVINO_DUMP_IR`           | Boolean   | `0`        | Serialize OpenVINO IR files with timestamps.                                                                |
 | `GGML_OPENVINO_DEBUG_INPUT`       | Boolean   | `0`        | Enable input debugging and print input tensor info.                                                         |

ggml/src/ggml-openvino/README.md

@@ -29,7 +29,7 @@ GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY=1 \
 
 Cache-only mode allocates backend address space without filling weight pages. On Windows, this also uses system commit capacity. The model-buffer size in the loader log is this virtual size. Weight uploads only record source identity; they do not read or requantize the weights. Graph conversion and compilation are skipped. Runtime buffers are still allocated and populated normally.
 
-Cache-only mode uses the settings provided by the current process. Keep these values exactly the same, including set versus unset: `GGML_OPENVINO_REQUANT_KQUANT`, `GGML_OPENVINO_NATIVE_SOFTPLUS`, `GGML_OPENVINO_DISABLE_KV_SLICE`, `GGML_OPENVINO_MANUAL_GQA_ATTN`, `GGML_OPENVINO_STATEFUL_EXECUTION`, `GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT`, `GGML_OPENVINO_DISABLE_REMOTE_OUTPUTS`, `GGML_OPENVINO_REDUCE_COMPILE_MEM`, `GGML_OPENVINO_MEMORY_OPTIMIZE`, `GGML_OPENVINO_PROFILING`, `GGML_OPENVINO_DISABLE_SHAPE_FROM_MASK`, and `GGML_OPENVINO_DEBUG_NODE`. On GPU, also repeat `GGML_OPENVINO_MOE_OP=0` if used. `GGML_OPENVINO_SPILL_DIR` is optional on the first run and ignored in cache-only mode; host-weight release is disabled in cache-only mode.
+Cache-only mode uses the settings provided by the current process. Keep these values exactly the same, including set versus unset: `GGML_OPENVINO_REQUANT_KQUANT`, `GGML_OPENVINO_EXACT_ZERO_POINT`, `GGML_OPENVINO_NATIVE_SOFTPLUS`, `GGML_OPENVINO_DISABLE_KV_SLICE`, `GGML_OPENVINO_MANUAL_GQA_ATTN`, `GGML_OPENVINO_STATEFUL_EXECUTION`, `GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT`, `GGML_OPENVINO_DISABLE_REMOTE_OUTPUTS`, `GGML_OPENVINO_REDUCE_COMPILE_MEM`, `GGML_OPENVINO_MEMORY_OPTIMIZE`, `GGML_OPENVINO_PROFILING`, `GGML_OPENVINO_DISABLE_SHAPE_FROM_MASK`, and `GGML_OPENVINO_DEBUG_NODE`. On GPU, also repeat `GGML_OPENVINO_MOE_OP=0` if used. `GGML_OPENVINO_SPILL_DIR` is optional on the first run and ignored in cache-only mode; host-weight release is disabled in cache-only mode.
 
 A missing or incompatible graph fails with an error instead of compiling with absent weights. The fingerprint uses the dynamic graph's topology, ports, model parameters, weights, settings, and OpenVINO version; changing only dynamic token or KV sizes does not require a new entry. A different workload can still require another graph; populate it first without cache-only mode.
 

ggml/src/ggml-openvino/ggml-openvino-extra.cpp

@@ -134,6 +134,7 @@ void ggml_openvino_device_config::init() {
         "GGML_OPENVINO_MANUAL_GQA_ATTN",
         "GGML_OPENVINO_DISABLE_ELTWISE_RANK_ALIGN",
         "GGML_OPENVINO_MOE_OP",
+        "GGML_OPENVINO_EXACT_ZERO_POINT",
         "GGML_OPENVINO_MEMORY_OPTIMIZE",
         "GGML_OPENVINO_RELEASE_WEIGHTS",
         "GGML_OPENVINO_REDUCE_COMPILE_MEM",
@@ -451,6 +452,10 @@ std::optional<ExtraQuantType> ggml_openvino_get_requant_type(const ggml_tensor *
     }
 }
 
+bool ggml_openvino_use_exact_zero_point(const ggml_tensor * tensor) {
+    return tensor->ne[2] > 1 || ggml_openvino_getenv_int("GGML_OPENVINO_EXACT_ZERO_POINT") > 0;
+}
+
 // =====================================================
 // Extracted Layout Calculation
 // =====================================================
@@ -470,11 +475,9 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten
         return layout;
     }
 
-    // 3D MoE expert weights that are not requantized (see below) always use the exact f16
-    // zero-point extraction (see extract_quantized_weights), which needs a wider zp slot than
-    // the packed integer zero point -- must be kept in sync with that function so the buffer
-    // sizing here matches what process_weight_tensor actually writes.
-    const bool for_gather_matmul = tensor->ne[2] > 1;
+    // An exact zero point (see extract_quantized_weights) needs an f16 zp slot instead of the
+    // packed integer one. Keep in sync with process_weight_tensor, which fills the slot.
+    const bool exact_zp = ggml_openvino_use_exact_zero_point(tensor);
 
     int64_t n_elements = ggml_nelements(tensor);
     const size_t alignment = 64;  // Good for SIMD
@@ -609,13 +612,11 @@ ggml_openvino_extracted_layout ggml_openvino_get_extracted_layout(const ggml_ten
     // Scales: F16 per block, except MXFP4 which stores one E8M0 byte per block.
     int64_t n_blocks = n_elements / layout.weights_per_block;
     layout.scales_size = n_blocks * (tensor->type == GGML_TYPE_MXFP4 ? sizeof(uint8_t) : sizeof(uint16_t));
-    // For symmetric quantization, no zp needed (weights stored as signed). Asymmetric
-    // for_gather_matmul (3D MoE expert) weights use an exact f16 zero point (see
-    // extract_quantized_weights/make_int8_weights/make_int4_weights), which needs one f16 per
-    // block instead of a packed u4/u8 integer zero point.
+    // For symmetric quantization, no zp needed (weights stored as signed). An exact zero point
+    // needs one f16 per block instead of a packed u4/u8 integer zero point.
     if (layout.is_symmetric) {
         layout.zp_size = 0;
-    } else if (use_bias || for_gather_matmul) {
+    } else if (use_bias || exact_zp) {
         layout.zp_size = n_blocks * sizeof(uint16_t);
     } else {
         layout.zp_size = layout.is_u4 ? ((n_blocks + 1) / 2) : n_blocks;

ggml/src/ggml-openvino/ggml-openvino-extra.h

@@ -137,6 +137,10 @@ bool ggml_openvino_weight_buffers_released();
 // Get requantization type for a tensor type (returns nullopt if no requant needed)
 std::optional<ExtraQuantType> ggml_openvino_get_requant_type(const ggml_tensor * tensor, bool no_requant = false);
 
+// True if an asymmetric weight keeps an exact f16 zero point instead of a rounded integer one.
+// Always true for 3D MoE expert weights. Other weights need GGML_OPENVINO_EXACT_ZERO_POINT.
+bool ggml_openvino_use_exact_zero_point(const ggml_tensor * tensor);
+
 // =====================================================
 // OpenVINO Tensor Extra Types
 // =====================================================

ggml/src/ggml-openvino/ggml-quants.cpp

@@ -1058,12 +1058,11 @@ std::shared_ptr<ov::Node> extract_quantized_weights(const ggml_tensor * tensor,
                                  std::string(ggml_type_name(tensor->type)));
     }
 
-    // 3D MoE expert weights (for_gather_matmul) always use the exact f16 zero-point extraction
-    // (see make_int8_weights/make_int4_weights) rather than the rounded integer zero point --
-    // round(min/scale) error is what corrupts Q4_K/Q5_1 experts, and the f16-zp form still fuses
-    // into GatherMatmulCompressed since it stays a Subtract, not an Add.
+    // An exact f16 zero point avoids the round(min/scale) error of the integer one, which corrupts
+    // Q4_K/Q5_1 MoE experts and costs accuracy on dense weights. It stays a Subtract, so the
+    // weights still fuse into GatherMatmulCompressed / compressed FullyConnected.
     const bool for_gather_matmul = tensor->ne[2] > 1;
-    use_bias = use_bias || for_gather_matmul;
+    use_bias = use_bias || ggml_openvino_use_exact_zero_point(tensor);
 
     // Extract quantized data
     switch (tensor->type) {
@@ -1258,13 +1257,9 @@ OvWeight process_weight_tensor(const ggml_tensor * tensor, const void * data, vo
         OPENVINO_THROW("Unsupported quantized type: ", ggml_type_name(tensor->type));
     }
 
-    // 3D MoE expert weights (for_gather_matmul) always use the exact f16 zero-point path (see
-    // extract_quantized_weights) -- must be kept in sync with the "use_bias || for_gather_matmul"
-    // check in ggml_openvino_get_extracted_layout, which sizes/offsets the zp slot accordingly.
-    // Requantized tensors (layout.is_requant) are handled by requantize_to_buffers instead, whose
-    // zp sizing/type is unaffected by for_gather_matmul, so they are excluded here.
-    const bool for_gather_matmul = tensor->ne[2] > 1;
-    const bool zp_is_f16 = !layout.is_requant && (use_bias || for_gather_matmul);
+    // Keep in sync with ggml_openvino_get_extracted_layout, which sizes the zp slot. Requantized
+    // tensors are handled by requantize_to_buffers and keep an integer zero point.
+    const bool zp_is_f16 = !layout.is_requant && (use_bias || ggml_openvino_use_exact_zero_point(tensor));
 
     const bool is_3d_mxfp4_moe = tensor->type == GGML_TYPE_MXFP4 && (tensor->ne[2] > 1 || tensor->ne[3] > 1);
     if (is_3d_mxfp4_moe) {

ggml/src/ggml-openvino/model-cache.cpp

@@ -169,6 +169,7 @@ static const char * cache_settings[] = {
     "GGML_OPENVINO_NATIVE_SOFTPLUS",
     "GGML_OPENVINO_DISABLE_KV_SLICE",
     "GGML_OPENVINO_MANUAL_GQA_ATTN",
+    "GGML_OPENVINO_EXACT_ZERO_POINT",
     "GGML_OPENVINO_STATEFUL_EXECUTION",
     "GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT",
     "GGML_OPENVINO_DISABLE_REMOTE_OUTPUTS",