diff --git a/docs/backend/OPENVINO.md b/docs/backend/OPENVINO.md index c9bbc9a9e38e..d1ec2bb74a18 100644 --- a/docs/backend/OPENVINO.md +++ b/docs/backend/OPENVINO.md @@ -113,13 +113,13 @@ Although, the validated models below were tested with `llama-cli` using the `Q4_ | [bartowski/Qwen_Qwen3-1.7B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3-1.7B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ | | [Qwen/Qwen3-4B-Q4_K_M](https://huggingface.co/Qwen/Qwen3-4B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ | | [lm-kit/Qwen3-8B-Q4_K_M](https://huggingface.co/lm-kit/qwen-3-8b-instruct-gguf) | ✓ / ✓ | ✓ / ✓ | ✓ | -| [bartowski/Qwen_Qwen3.5-0.8B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3.5-0.8B-GGUF) | ✓ / ✓ | ✓ / ~ | ✗ | -| [bartowski/Qwen_Qwen3.5-2B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3.5-2B-GGUF) | ✓ / ✓ | ✓ / ~ | ✗ | -| [bartowski/Qwen_Qwen3.5-4B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3.5-4B-GGUF) | ✓ / ✓ | ✓ / ~ | ✗ | -| [lmstudio-community/Qwen3.5-9B-Q4_K_M](https://huggingface.co/lmstudio-community/Qwen3.5-9B-GGUF) | ✓ / ✓ | ✓ / ~ | ✗ | +| [bartowski/Qwen_Qwen3.5-0.8B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3.5-0.8B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✗ | +| [bartowski/Qwen_Qwen3.5-2B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3.5-2B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✗ | +| [bartowski/Qwen_Qwen3.5-4B-Q4_K_M](https://huggingface.co/bartowski/Qwen_Qwen3.5-4B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✗ | +| [lmstudio-community/Qwen3.5-9B-Q4_K_M](https://huggingface.co/lmstudio-community/Qwen3.5-9B-GGUF) | ✓ / ✓ | ✓ / ✓ | ✗ | | | | | | | [unsloth/gemma-3-4b-it-Q4_K_M](https://huggingface.co/unsloth/gemma-3-4b-it-GGUF) | ✓ / ✓ | ✓ / ✓ | ✓ | -| [bartowski/google_gemma-4-E2B-it-Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-E2B-it-GGUF) | ✓ / ✓ | ✓ / ~ | ~ | +| [bartowski/google_gemma-4-E2B-it-Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-E2B-it-GGUF) | ✓ / ✓ | ✓ / ✓ | ~ | | [bartowski/google_gemma-4-E4B-it-Q4_K_M](https://huggingface.co/bartowski/google_gemma-4-E4B-it-GGUF) | ✓ / ✓ | ✗ / ✗ | ✓ | | [bartowski/gemma-4-12B-it-Q4_K_M](https://huggingface.co/bartowski/gemma-4-12B-it-GGUF) | ✓ / ✓ | ✓ / ✓ | ✗ | | | | | | @@ -722,7 +722,9 @@ Boolean flags follow a uniform convention: set to a **positive integer** (e.g. ` | `GGML_OPENVINO_STATEFUL_EXECUTION`| Boolean | `0` | Keep KV and supported recurrent caches inside the model. Single-slot CPU/GPU execution only. | | `GGML_OPENVINO_DISABLE_CACHE` | Boolean | `0` | Disable the in-process compiled-model / decoder cache (cache is on by default). Set to `1` to disable. | | `GGML_OPENVINO_DISABLE_KV_SLICE` | Boolean | `0` | Disable the KV-cache input-tensor slicing optimization (slicing is on by default on CPU/GPU). Set to `1` to disable. | +| `GGML_OPENVINO_DISABLE_KV_4D` | Boolean | `0` | GPU, stateless: keep the KV cache in the `[1, 1, ctx, n_heads_kv * head_size]` layout instead of `[1, ctx, n_heads_kv, head_size]`, which lets the GPU plugin update and read the cache in place. Set to `1` to disable. | | `GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT` | Boolean | `0` | Disable the stateful KV-state sequence-axis relayout (relayout is on by default). It moves the KV state sequence axis from dim 1 to dim 2, so the GPU plugin can append new tokens in place instead of copying the whole state every token, and the reader side no longer transposes the whole accumulated state. Set to `1` to disable. | +| `GGML_OPENVINO_DISABLE_SHAPE_FROM_MASK` | Boolean | `0` | Dynamic CPU/GPU graphs read the active sequence count and the attended KV length from the shape of the KQ mask inside the model, so the GPU plugin can infer the attention shapes without waiting for input values and selects its fused SDPA kernel. Set to `1` to pass them as separate inputs instead. | | `GGML_OPENVINO_MANUAL_GQA_ATTN` | Boolean | device-based | Tri-state. When **unset**, manual GQA attention is enabled by default on `GPU` and disabled on other devices. Set to a positive integer to force-enable, or `0` to force-disable. | | `GGML_OPENVINO_MEMORY_OPTIMIZE` | Boolean | `0` | Umbrella switch for compile-time memory reductions. Enables `GGML_OPENVINO_REDUCE_COMPILE_MEM` and, on GPU, `GGML_OPENVINO_RELEASE_WEIGHTS` unless those fine-grained variables are explicitly set. | | `GGML_OPENVINO_REDUCE_COMPILE_MEM`| Boolean | inherits from `GGML_OPENVINO_MEMORY_OPTIMIZE` | Reduce compile-time host memory use by streaming weight requantization and avoiding extra weight-node materialization where possible. Set explicitly to override the umbrella switch. | diff --git a/ggml/src/ggml-openvino/CMakeLists.txt b/ggml/src/ggml-openvino/CMakeLists.txt index 347c24f4f526..458215855d71 100644 --- a/ggml/src/ggml-openvino/CMakeLists.txt +++ b/ggml/src/ggml-openvino/CMakeLists.txt @@ -1,17 +1,21 @@ find_package(OpenVINO REQUIRED COMPONENTS Runtime Threading) find_package(OpenCL REQUIRED) +if(NOT TARGET openvino::frontend::gguf) + message(FATAL_ERROR "OpenVINO GGUF frontend target openvino::frontend::gguf is required") +endif() + message(STATUS "Found OpenVINO: ${OpenVINO_DIR} (found version \"${OpenVINO_VERSION}\")") -file(GLOB_RECURSE GGML_HEADERS_OPENVINO "*.h" "*.hpp") -file(GLOB_RECURSE GGML_SOURCES_OPENVINO "*.cpp") +file(GLOB GGML_HEADERS_OPENVINO "*.h" "*.hpp") +file(GLOB GGML_SOURCES_OPENVINO "*.cpp") ggml_add_backend_library(ggml-openvino ${GGML_SOURCES_OPENVINO} ${GGML_HEADERS_OPENVINO} ) -target_link_libraries(ggml-openvino PRIVATE openvino::runtime openvino::threading OpenCL::OpenCL) +target_link_libraries(ggml-openvino PRIVATE openvino::runtime openvino::threading openvino::frontend::gguf OpenCL::OpenCL) # the OpenVINO RTTI macros take one argument and leave __VA_ARGS__ empty, which -Wpedantic reports if (CMAKE_CXX_COMPILER_ID MATCHES "Clang" OR CMAKE_CXX_COMPILER_ID STREQUAL "IntelLLVM") diff --git a/ggml/src/ggml-openvino/README.md b/ggml/src/ggml-openvino/README.md index 6a23f62bf246..a12233d2aa17 100644 --- a/ggml/src/ggml-openvino/README.md +++ b/ggml/src/ggml-openvino/README.md @@ -29,7 +29,7 @@ GGML_OPENVINO_COMPILED_MODEL_CACHE_ONLY=1 \ Cache-only mode allocates backend address space without filling weight pages. On Windows, this also uses system commit capacity. The model-buffer size in the loader log is this virtual size. Weight uploads only record source identity; they do not read or requantize the weights. Graph conversion and compilation are skipped. Runtime buffers are still allocated and populated normally. -Cache-only mode uses the settings provided by the current process. Keep these values exactly the same, including set versus unset: `GGML_OPENVINO_REQUANT_KQUANT`, `GGML_OPENVINO_NATIVE_SOFTPLUS`, `GGML_OPENVINO_DISABLE_KV_SLICE`, `GGML_OPENVINO_MANUAL_GQA_ATTN`, `GGML_OPENVINO_STATEFUL_EXECUTION`, `GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT`, `GGML_OPENVINO_DISABLE_REMOTE_OUTPUTS`, `GGML_OPENVINO_REDUCE_COMPILE_MEM`, `GGML_OPENVINO_MEMORY_OPTIMIZE`, `GGML_OPENVINO_PROFILING`, and `GGML_OPENVINO_DEBUG_NODE`. On GPU, also repeat `GGML_OPENVINO_MOE_OP=0` if used. `GGML_OPENVINO_SPILL_DIR` is optional on the first run and ignored in cache-only mode; host-weight release is disabled in cache-only mode. +Cache-only mode uses the settings provided by the current process. Keep these values exactly the same, including set versus unset: `GGML_OPENVINO_REQUANT_KQUANT`, `GGML_OPENVINO_NATIVE_SOFTPLUS`, `GGML_OPENVINO_DISABLE_KV_SLICE`, `GGML_OPENVINO_MANUAL_GQA_ATTN`, `GGML_OPENVINO_STATEFUL_EXECUTION`, `GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT`, `GGML_OPENVINO_DISABLE_REMOTE_OUTPUTS`, `GGML_OPENVINO_REDUCE_COMPILE_MEM`, `GGML_OPENVINO_MEMORY_OPTIMIZE`, `GGML_OPENVINO_PROFILING`, `GGML_OPENVINO_DISABLE_SHAPE_FROM_MASK`, and `GGML_OPENVINO_DEBUG_NODE`. On GPU, also repeat `GGML_OPENVINO_MOE_OP=0` if used. `GGML_OPENVINO_SPILL_DIR` is optional on the first run and ignored in cache-only mode; host-weight release is disabled in cache-only mode. A missing or incompatible graph fails with an error instead of compiling with absent weights. The fingerprint uses the dynamic graph's topology, ports, model parameters, weights, settings, and OpenVINO version; changing only dynamic token or KV sizes does not require a new entry. A different workload can still require another graph; populate it first without cache-only mode. diff --git a/ggml/src/ggml-openvino/align-eltwise-ranks.cpp b/ggml/src/ggml-openvino/align-eltwise-ranks.cpp new file mode 100644 index 000000000000..7b376b48b195 --- /dev/null +++ b/ggml/src/ggml-openvino/align-eltwise-ranks.cpp @@ -0,0 +1,91 @@ +#include "align-eltwise-ranks.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ggml_openvino { + +namespace { + +// True for an RMS-norm output, x * (1 / sqrt(mean(x^2) + eps)) or x * pow(mean(x^2) + eps, -0.5), optionally scaled +// by the norm weight. +bool is_rms_norm_output(const ov::Output & value, int depth = 1) { + const auto * node = value.get_node(); + if (!ov::is_type(node)) { + return false; + } + for (const auto & input : node->input_values()) { + const auto * src = input.get_node(); + if (ov::is_type(src) && ov::is_type(src->get_input_node_ptr(1))) { + return true; + } + if (ov::is_type(src) && ov::is_type(src->get_input_node_ptr(1))) { + return true; + } + if (depth > 0 && is_rms_norm_output(input, depth - 1)) { + return true; + } + } + return false; +} + +} // namespace + +AlignEltwiseOperandRanks::AlignEltwiseOperandRanks() { + auto eltwise_m = ov::pass::pattern::wrap_type(); + + const auto callback = [this](ov::pass::pattern::Matcher & m) { + auto node = m.get_match_root(); + if (node->get_input_size() != 2) { + return false; + } + + auto lhs = node->input_value(0); + auto rhs = node->input_value(1); + + // A tensor-vs-Constant mismatch is the norm's own eps / 1-over-sqrt arithmetic, which folds into the `rms` + // primitive itself instead of becoming a fused eltwise post-op, so it is not affected. + if (ov::is_type(lhs.get_node()) || ov::is_type(rhs.get_node())) { + return false; + } + + const auto lhs_rank = lhs.get_partial_shape().rank(); + const auto rhs_rank = rhs.get_partial_shape().rank(); + if (lhs_rank.is_dynamic() || rhs_rank.is_dynamic() || lhs_rank == rhs_rank) { + return false; + } + + const size_t shorter_idx = lhs_rank.get_length() < rhs_rank.get_length() ? 0 : 1; + const auto & shorter = shorter_idx == 0 ? lhs : rhs; + + // The defect is with the norm output as the higher-rank operand. When the norm output is the lower-rank + // one, unsqueezing it puts the Unsqueeze between `rms` and its post-op, and the plugin then computes the + // layer wrongly. + if (is_rms_norm_output(shorter)) { + return false; + } + const int64_t diff = std::abs(lhs_rank.get_length() - rhs_rank.get_length()); + + std::vector axes(static_cast(diff)); + std::iota(axes.begin(), axes.end(), 0); + auto unsqueeze = std::make_shared( + shorter, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{ axes.size() }, axes)); + + node->input(shorter_idx).replace_source_output(unsqueeze->output(0)); + register_new_node(unsqueeze); + return true; + }; + + register_matcher(std::make_shared(eltwise_m, "ggml_openvino::AlignEltwiseOperandRanks"), + callback); +} + +} // namespace ggml_openvino diff --git a/ggml/src/ggml-openvino/align-eltwise-ranks.h b/ggml/src/ggml-openvino/align-eltwise-ranks.h new file mode 100644 index 000000000000..eca340bbb020 --- /dev/null +++ b/ggml/src/ggml-openvino/align-eltwise-ranks.h @@ -0,0 +1,25 @@ +#pragma once + +#include "openvino/pass/matcher_pass.hpp" + +namespace ggml_openvino { + +// Gives a binary eltwise op's two operands the same rank, by unsqueezing leading axes onto the shorter one. +// +// This is semantically a no-op (NUMPY broadcasting already left-pads the lower-rank operand with 1s, and the result +// rank is max(rank_a, rank_b) either way). It exists purely to work around an OpenVINO GPU-plugin defect: an eltwise +// op whose operands differ in rank is computed incorrectly once the plugin fuses it as a post-op into an `rms` +// primitive (e.g. a rank-3 residual added to a rank-4 RMS-norm output; the model degenerates into repeated tokens on +// GPU while CPU is correct). Equalising the ranks keeps the fusion and makes it compute the right answer. +// +// Only applies when both operands are real tensors: a tensor-vs-Constant mismatch is the RMS norm's own eps/rsqrt +// arithmetic, which folds into the `rms` primitive and is not affected. The norm output itself is never unsqueezed: +// when it is the lower-rank operand, that breaks the fused path instead. Registered for GPU only, disabled by +// GGML_OPENVINO_DISABLE_ELTWISE_RANK_ALIGN. Remove once the plugin guards that fusion. +class AlignEltwiseOperandRanks : public ov::pass::MatcherPass { +public: + OPENVINO_MATCHER_PASS_RTTI("ggml_openvino::AlignEltwiseOperandRanks") + AlignEltwiseOperandRanks(); +}; + +} // namespace ggml_openvino diff --git a/ggml/src/ggml-openvino/ggml-decoder.cpp b/ggml/src/ggml-openvino/ggml-decoder.cpp index ae36a09591da..82bcfdf4858f 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.cpp +++ b/ggml/src/ggml-openvino/ggml-decoder.cpp @@ -14,6 +14,7 @@ #include #include #include +#include #include #include #include @@ -26,6 +27,8 @@ #include #include #include +#include +#include #include #include #include @@ -73,13 +76,12 @@ GgmlOvDecoder::GgmlOvDecoder(ggml_cgraph * cgraph, m_node_info_list[node_n].node_op_case = compute_op_case(m_node_info_list[node_n].node); m_node_info_list[node_n].node_op_type = compute_op_type(m_node_info_list[node_n].node); } - - add_extra_inputs(); } void GgmlOvDecoder::update_io(ggml_cgraph * cgraph) { m_cgraph = cgraph; m_model_inputs.clear(); + m_model_input_nodes.clear(); m_model_outputs.clear(); m_node_info_list.clear(); set_input_output(); @@ -109,6 +111,48 @@ GgmlOvDecoder::GgmlOvDecoder(ggml_cgraph * cgraph, std::map create_parameter(const std::string & name, const BackendInputInfo & info) { + auto param = std::make_shared(info.type, info.shape); + param->set_friendly_name(name); + param->output(0).get_tensor().set_names({name}); + return param; +} + +float op_param_float(const ggml_tensor * node, size_t index) { + float value = 0.0f; + memcpy(&value, reinterpret_cast(node->op_params) + index, sizeof(float)); + return value; +} + +ov::frontend::gguf::RopeConfig rope_config_from_params(const int32_t * params, bool per_op) { + ov::frontend::gguf::RopeConfig config; + if (params == nullptr) { + return config; + } + config.n_dims = params[1]; + config.n_ctx_orig = params[4]; + memcpy(&config.freq_base, params + 5, sizeof(float)); + memcpy(&config.freq_scale, params + 6, sizeof(float)); + memcpy(&config.ext_factor, params + 7, sizeof(float)); + memcpy(&config.attn_factor, params + 8, sizeof(float)); + memcpy(&config.beta_fast, params + 9, sizeof(float)); + memcpy(&config.beta_slow, params + 10, sizeof(float)); + config.per_op = per_op; + config.is_imrope = params[2] == GGML_ROPE_TYPE_IMROPE; + return config; +} + +std::vector permute_order_from_params(const int32_t * params) { + std::vector perm_values{0, 2, 1, 3}; + if (params != nullptr) { + for (size_t input_axis = 0; input_axis < perm_values.size(); ++input_axis) { + const size_t output_axis = static_cast(params[input_axis]); + perm_values[perm_values.size() - 1 - output_axis] = static_cast(perm_values.size() - 1 - input_axis); + } + } + return perm_values; +} + bool is_inplace_op(const ggml_tensor * node) { return node->op == GGML_OP_SET_ROWS || node->op == GGML_OP_CPY || (node->op == GGML_OP_SCALE && node->view_src); } @@ -197,6 +241,12 @@ std::string GgmlOvDecoder::get_tensor_name(const ggml_cgraph * cgraph, const ggm return std::string(tensor->name) + "#l" + std::to_string(leaf - cgraph->leafs); } } + if (is_external_view(cgraph, tensor)) { + // A VIEW produced by an earlier split enters this graph as an input. Sibling views often share a + // name (e.g. the s_copy main/extra slices are both " (view)"), so key it by what it selects. + return std::string(tensor->name) + "#v" + tensor->view_src->name + "+" + std::to_string(tensor->view_offs) + + "x" + std::to_string(ggml_nelements(tensor)); + } return tensor->name; } @@ -229,6 +279,15 @@ static std::string get_tensor_graph_input_ov_name(const GgmlOvDecoder * decoder, return get_tensor_ov_name(cgraph, tensor) + "_swa"; } } + if (!decoder->get_model_params().swa_layers.empty() && GgmlOvDecoder::is_kv_idx(tensor, op) && + op->src[2] != nullptr) { + // same as the masks: the full and the sliding-window cache get KV write indices of the same name + const ggml_tensor * cache = op->src[2]->view_src != nullptr ? op->src[2]->view_src : op->src[2]; + auto layer = extract_layer_from_name(cache->name); + if (layer.has_value() && decoder->is_swa_layer(layer.value())) { + return get_tensor_ov_name(cgraph, tensor) + "_swa"; + } + } return get_tensor_ov_name(cgraph, tensor); } @@ -246,38 +305,22 @@ void GgmlOvDecoder::set_input_output() { for (int i = 0; i < GGML_MAX_SRC; i++) { auto * src = node->src[i]; - if (src == nullptr) { + // ggml_cast() records the result as its own CPY destination (src[1] == node); the cast is + // translated from src[0] and dst_type alone. + if (src == nullptr || src == node) { continue; } + // A full single-slot writeback overwrites the whole cache; name the cache itself as the destination + // (not a whole-storage VIEW of it) so the frontend rebinds the cache storage to the written value. + if (i == 1 && node->op == GGML_OP_CPY && is_full_single_slot_writeback(node)) { + src = node->view_src; + } auto src_name = get_tensor_ov_name(m_cgraph, src); if (src->flags & GGML_TENSOR_FLAG_INPUT) { src_name = get_tensor_graph_input_ov_name(this, m_cgraph, src, node); } current_node_info.node_inputs[src_name] = src; current_node_info.node_inputs_names.push_back(src_name); - - if (src->op == GGML_OP_VIEW) { - // Traverse upward through nested VIEW operations - std::remove_reference_t view_chain; - auto * current = src; - - while (current != nullptr) { - auto current_name = get_tensor_ov_name(m_cgraph, current); - if (current->flags & GGML_TENSOR_FLAG_INPUT) { - current_name = get_tensor_graph_input_ov_name(this, m_cgraph, current, node); - } - view_chain.emplace_back(current_name, current); - // If current src is also a VIEW, continue traversing - if (current->src[0] != nullptr && current->src[0]->op == GGML_OP_VIEW) { - current = current->src[0]; - } else { - break; - } - } - - // Assign all collected view inputs to node_inputs_views - current_node_info.node_inputs_views[src_name] = view_chain; - } } m_node_info_list.push_back(current_node_info); @@ -291,7 +334,6 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const { if (m_naive) { break; } - auto name = std::string(node->name); auto * src = node->src[0]; // Identify recurrent sequence reshapes before size checks, which are ambiguous for one token. bool recurrent_sequence = false; @@ -327,185 +369,6 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const { } } else if (node->ne[0] == 1 && src->ne[0] * src->ne[1] * src->ne[2] == node->ne[1]) { op_case = 3; - } else if (name.find("linear_attn_qkv_mixed") == 0 || name.find("alpha") == 0) { - op_case = 6; - } else if (name.find("linear_attn_out") == 0) { - op_case = 7; - } else if (name.find("state_predelta") == 0) { - op_case = 8; - } - if (op_case == 1 && m_is_stateful) { - // Recurrent convolution and GDN gates retain their rank-4 layout. - bool recurrent = src->op == GGML_OP_GET_ROWS && is_recurrent_cache(src->src[0]); - for (int i = 0; i < m_cgraph->n_nodes && !recurrent; ++i) { - const auto * consumer = m_cgraph->nodes[i]; - if (consumer->op == GGML_OP_GATED_DELTA_NET) { - for (int j : {3, 4}) { - const auto * gate = consumer->src[j]; - if (gate->op == GGML_OP_UNARY) { - gate = gate->src[0]; - } - recurrent = recurrent || gate == node; - } - } else if (consumer->op == GGML_OP_MUL) { - for (int j = 0; j < 2; ++j) { - const auto * gate = consumer->src[j]; - const auto * norm = consumer->src[1 - j]; - if (gate->op != GGML_OP_UNARY || gate->src[0] != node) { - continue; - } - if (norm->op == GGML_OP_MUL) { - norm = norm->src[0]; - } - recurrent = recurrent || (norm->op == GGML_OP_RMS_NORM && norm->src[0]->op == GGML_OP_VIEW && - norm->src[0]->src[0]->op == GGML_OP_GATED_DELTA_NET); - } - } - } - if (recurrent) { - op_case = 9; - } - } - break; - } - case GGML_OP_PERMUTE: { - if (node->src[0]->op != GGML_OP_VIEW) { - op_case = 1; - } else if (node->src[0]->src[0]->op == GGML_OP_NONE) { - // kv cache tensor - std::string src_name(node->view_src->name); - int layer = extract_layer_from_name(src_name).value(); - if (ggml_is_contiguous(node->src[0])) { - // - 19: [ 64, 8, 256, 1] VIEW cache_k_l0 (view) [ 2, 128, 1024, 1048576] - // [ 512, 1024, 1, 1] 0: NONE cache_k_l0 [ 2, 1024, 1048576, 1048576] - // - 20: [ 64, 256, 8, 1] PERMUTE cache_k_l0 (view) (permuted) [ 2, 1024, 128, 1048576] - // [ 64, 8, 256, 1] 0: VIEW cache_k_l0 (view) [ 2, 128, 1024, 1048576] - if (!is_swa_layer(layer)) { - op_case = 3; - } else { - op_case = 4; - } - } else { - // special case of cache v when `-fa off` - // - 17: [ 256, 8, 64, 1] VIEW cache_v_l0 (view) [ 2, 131072, 2048, 1048576] - // [ 512, 1024, 1, 1] 0: NONE cache_v_l0 [ 2, 1024, 1048576, 1048576] - // - 18: [ 256, 64, 8, 1] PERMUTE cache_v_l0 (view) (permuted) [ 2, 2048, 131072, 1048576] - // [ 256, 8, 64, 1] 0: VIEW cache_v_l0 (view) [ 2, 131072, 2048, 1048576] - if (!is_swa_layer(layer)) { - op_case = 5; - } else { - op_case = 6; - } - } - } else { - // rope'ed query tensor - op_case = 2; - } - break; - } - case GGML_OP_MUL_MAT: { - if (node->src[1]->op == GGML_OP_SOFT_MAX) { - // In the case of `-fa off`, softmax is used, v_trans=true, the dynamic dim is ne[0] for cache_v - op_case = 2; - } - break; - } - case GGML_OP_GET_ROWS: { - if (node->src[1]->op == GGML_OP_VIEW) { - // GET_ROWS gathering recurrent state cache rows via the inp->s_copy index list: - // src[0] is a reshape of cache_r/cache_s, src[1] is a view of the s_copy leaf. - // op_case 1/2: active/extra rows of a multi-slot cache - // op_case 3/4: active/extra rows of a single-slot cache - if (node->src[0]->op == GGML_OP_RESHAPE && node->src[0]->src[0] != nullptr && - is_recurrent_cache(node->src[0]->src[0])) { - const bool single_slot = node->src[0]->src[0]->ne[1] == 1; - op_case = (node->src[1]->view_offs == 0 ? 1 : 2) + (single_slot ? 2 : 0); - } - } - break; - } - case GGML_OP_ROPE: { - const int mode = node->op_params[2]; - switch (mode) { - case GGML_ROPE_TYPE_NEOX: { - op_case = 1; - break; - } - case GGML_ROPE_TYPE_IMROPE: { - op_case = 2; - break; - } - case GGML_ROPE_TYPE_VISION: { - op_case = 3; - break; - } - case GGML_ROPE_TYPE_MROPE: { - op_case = 4; - break; - } - default: - op_case = 0; - break; - } - break; - } - case GGML_OP_VIEW: { - if (!m_model_params.has_rs_rollback && node->src[0] != nullptr && - node->src[0]->op == GGML_OP_GATED_DELTA_NET) { - // The GDN translator publishes native attention/state outputs under these VIEW names. - op_case = 2; - break; - } - if (m_is_static && node->src[0] != nullptr && - (node->src[0]->op == GGML_OP_GATED_DELTA_NET || node->src[0]->op == GGML_OP_CONCAT)) { - // VIEW slicing a GATED_DELTA_NET combined [attn|state] output, or the conv_input - // CONCAT. The consuming CPY/RMS_NORM op recovers the true window at runtime via - // ssm_state_size / the fixed conv kernel width, so this VIEW must stay an identity - // pass-through of the full source here too (it already is on the dynamic path); - // otherwise the generic static-mode Slice below would bake in the *captured* - // cgraph's token count, which is wrong once the compiled static model runs with a - // different token count (prefill chunk size or 1). - op_case = 1; - break; - } - if (node->src[0]->op == GGML_OP_VIEW) { - auto * src = node->src[0]; - if (ggml_nelements(node) != ggml_nelements(src)) { - // throw std::runtime_error("Unsupported VIEW case"); - } - op_case = 0; - if (m_model_is_splitted && m_model_inputs.find(get_tensor_ov_name(m_cgraph, src)) != m_model_inputs.end()) { - op_case = 0; - } - } - { - auto * src = node->src[0]; - if (ggml_nelements(node) != ggml_nelements(src)) { - // Case 4: select one slice on src dim1 (via view offset), keep src dim2 as output dim1. - // Typical pattern: - // src: ne=[N, M, K, 1], nb=[b0, b1, b2, b3] - // dst: ne=[N, K, 1, 1], nb=[b0, b2, b3, b3] - if (node->ne[0] == src->ne[0] && node->ne[1] == src->ne[2] && node->ne[2] == 1 && - node->nb[0] == src->nb[0] && node->nb[1] == src->nb[2] && src->ne[1] > 1) { - op_case = 0; - break; - } - - // General case 3: shape differs from source (one or more dims) and is handled as VIEW slicing. - int diff_count = 0; - for (int i = 0; i < GGML_MAX_DIMS; i++) { - if (node->ne[i] != src->ne[i]) { - diff_count++; - } - // if node ne[i] > src ne[i], case = 0 - if (node->ne[i] > src->ne[i]) { - return 0; - } - } - if (diff_count >= 1) { - op_case = 0; - } - } } break; } @@ -579,6 +442,12 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const { node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src == node->view_src) { op_case = 4; break; + } else if (node->src[0]->src[0]->op == GGML_OP_GET_ROWS && node->src[1] != nullptr && + node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src != nullptr && + is_recurrent_cache(node->src[1]->view_src) && node->src[1]->view_src->ne[1] == 1) { + // defrag remainder writeback of a single-slot cache, taken from a view of the one gather of + // all states (see build_rs) + op_case = 9; } } else if (node->src[0]->op == GGML_OP_GET_ROWS && node->src[1] != nullptr && node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src != nullptr && @@ -613,35 +482,12 @@ int GgmlOvDecoder::compute_op_case(const ggml_tensor * node) const { } break; } - case GGML_OP_ADD: { - if (is_moe_expert_sum_add(node)) { - // Outermost ADD of a MoE expert-plane sum chain: translated as a single - // ReduceSum over the base tensor instead of N-1 chained Adds over N Slices. - op_case = 1; - } - break; - } case GGML_OP_SCALE: { if (node->view_src && node->buffer->usage == GGML_BACKEND_BUFFER_USAGE_ANY) { op_case = node->view_src->ne[1] == 1 ? 2 : 1; } break; } - case GGML_OP_L2_NORM: { - if (std::string(node->name).find("predelta") != std::string::npos) { - op_case = 1; - } - break; - } - case GGML_OP_FLASH_ATTN_EXT: { - if (node->src[1] != nullptr && node->src[1]->op == GGML_OP_VIEW && node->src[1]->view_src != nullptr) { - const ggml_tensor * kv_buf = node->src[1]->view_src; - if (kv_buf->ne[1] == 1 && kv_buf->ne[2] == 1 && kv_buf->ne[3] == 1) { - op_case = (node->src[3] != nullptr) ? 1 : 2; - } - } - break; - } default: break; } @@ -791,10 +637,9 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr // from the first graph onwards, while the view grows with context depth and would invert the // comparison at shallow depth. // - // Layers whose leaf is smaller than the largest leaf are the windowed ones. When every layer - // reports the same extent there is no distinction to draw -- either the model has no windowed - // layers, or the window is at least as large as the context so the two caches coincide, in - // which case a windowed layer and a full-attention one compute the same thing. + // Layers whose leaf is smaller than the largest leaf are the windowed ones. Equal extents do not + // rule them out: the windowed cache is padded to n_swa + n_ubatch, which can equal the full + // context (gemma-4 at -c 1024 -ub 512). See the mask fallback below. // // Getting this wrong is silent and severe: with the windowed layers classified as // full-attention, permute's KV slicing uses attention_size instead of attention_size_swa. The @@ -839,6 +684,32 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr } } } + + // equal extents: tell the two masks apart by the layers that read them, the larger group gets the swa role + // (gemma-4: 4 of 5 layers). Groups of equal size can not be told apart, so leave the layers unclassified. + // Two masks do not imply swa (deepseek sparse attention builds two full masks), but the role does not need + // them to: it gives each group its own mask, KV write indices and attention size, which is right either way. + // The only use of the window itself, the stateful mask rebuild, reads it back from the mask, and on a full + // mask that gives back the causal mask. Decide from the first graph, before the window shows in the mask, so + // the graph does not change, and is not recompiled, when it does. + // TODO: "larger group is swa" is a model-level guess. Revisit when llama.cpp tells the backend which layers + // use a sliding window. Stateless graphs only need the two groups kept apart (separate mask, KV write index + // and attention size inputs), not the swa role itself, so the guess could be limited to stateful execution. + if (model_params.swa_layers.empty()) { + std::map> mask_layers; + for (const auto & [layer, mask] : layer_mask) { + mask_layers[mask].push_back(layer); + } + if (mask_layers.size() == 2 && + mask_layers.begin()->second.size() != std::next(mask_layers.begin())->second.size()) { + auto swa = mask_layers.begin(); + if (std::next(swa)->second.size() > swa->second.size()) { + swa = std::next(swa); + } + model_params.swa_layers = swa->second; + model_params.swa_mask = swa->first; + } + } std::sort(model_params.swa_layers.begin(), model_params.swa_layers.end()); if (ggml_openvino_getenv_int("GGML_OPENVINO_LOG_SWA_LAYERS")) { @@ -928,11 +799,17 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr memcpy(&offset, cache_k_view->op_params, sizeof(size_t)); compute_params.seq_active_start = offset / seq_size; + // a mask that is itself a graph input has the shape of its OV parameter + const std::string mask_name = mask->op != GGML_OP_NONE ? std::string() : + mask == model_params.swa_mask ? get_tensor_ov_name(cgraph, mask) + "_swa" : + get_tensor_ov_name(cgraph, mask); if (layer_is_swa) { compute_params.attention_size_swa = mask->ne[0]; compute_params.swa_window = get_swa_window_from_mask(mask); + compute_params.mask_swa_name = mask_name; } else { compute_params.attention_size = mask->ne[0]; + compute_params.mask_name = mask_name; } if (is_static) { compute_params.attention_size = model_params.ctx_per_seq; @@ -946,6 +823,7 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr if (node->src[1]->op == GGML_OP_PERMUTE && node->src[1]->src[0]->op == GGML_OP_VIEW && node->src[1]->src[0]->src[0]->op == GGML_OP_ROPE) { compute_params.attention_size = node->ne[0]; + compute_params.mask_name.clear(); } } @@ -953,6 +831,7 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr if (node->op == GGML_OP_TRANSPOSE && node->src[0]->op == GGML_OP_PERMUTE && node->src[0]->src[0]->op == GGML_OP_VIEW) { compute_params.attention_size = node->ne[0]; + compute_params.mask_name.clear(); if (is_static) { compute_params.attention_size = model_params.ctx_per_seq; } @@ -984,7 +863,8 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr GGML_ASSERT(model_params.n_rs_slots == node->view_src->ne[1]); } compute_params.cache_rs_reset_len = ggml_nelements(node) / node->view_src->ne[0]; - compute_params.cache_rs_reset_idx = node->src[0]->view_offs / node->view_src->ne[0]; + // view_offs is in bytes; the reset range is counted in cache rows. + compute_params.cache_rs_reset_idx = node->src[0]->view_offs / node->view_src->nb[1]; } // Capture the destination slot block of every recurrent state cache writeback, plus the // source window needed by conv state and packed GDN rollback writes. The active sequences @@ -1000,6 +880,11 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr const ggml_tensor * dest_view = node->src[1]; const ggml_tensor * cache = node->view_src; + // Writes into a flat buffer (bailingmoe3 conv_states_all, whisper's KV buffers) land at an element + // offset that moves with the cache head; the translated CPY updates the buffer from that offset. + const bool is_flat = !is_conv && !is_gdn && !is_extra && + (is_conv_states_all_tensor(cache) || + (cache->ne[1] == 1 && cache->ne[2] == 1 && cache->ne[3] == 1)); const size_t row_bytes = cache->ne[0] * ggml_type_size(cache->type); if (is_gdn_rollback) { // Rollback GDN exposes an already-flattened [state, seq, snapshot] VIEW and copies @@ -1008,9 +893,10 @@ std::pair GgmlOvDecoder::compute_llm_params(ggml_cgr // signature is local to the CPY and still works when fallback splits the graph. model_params.has_rs_rollback = true; } - if (row_bytes > 0 && (is_conv || is_gdn || is_extra) && !is_full_single_slot_writeback(node)) { + if (row_bytes > 0 && (is_conv || is_gdn || is_extra || is_flat) && !is_full_single_slot_writeback(node)) { ComputeParams::RsWriteback writeback; - writeback.slot_begin = (int) (dest_view->view_offs / row_bytes); + writeback.slot_begin = is_flat ? (int) (dest_view->view_offs / ggml_type_size(cache->type)) : + (int) (dest_view->view_offs / row_bytes); if (is_conv) { writeback.src_begin = (int) (node->src[0]->view_offs / node->src[0]->view_src->nb[0]); } else if (is_gdn_rollback) { @@ -1082,29 +968,35 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op, if (is_inp_tok(input, op) || is_inp_pos(input, op)) { // tokens or positions - int len = m_is_static ? (m_is_prefill ? m_prefill_chunk_size : 1) : -1; + const ov::Dimension len = m_is_static ? ov::Dimension(m_is_prefill ? m_prefill_chunk_size : 1) : + ov::Dimension::dynamic(); if (m_is_static && is_inp_pos(input, op)) { // IMROPE stacks n_planes (t/h/w/e) position planes back to back - len *= get_inp_pos_n_planes(op); + input_shape = ov::PartialShape{1, 1, 1, len.get_length() * get_inp_pos_n_planes(op)}; + } else { + input_shape = ov::PartialShape{1, 1, 1, len}; } - input_shape = ov::PartialShape{1, 1, 1, len}; } else if (is_output_idx(input, op)) { // output index - input_shape = ov::PartialShape{1, 1, 1, m_is_static ? m_compute_params.output_len : -1}; + input_shape = ov::PartialShape{1, 1, 1, + m_is_static ? ov::Dimension(m_compute_params.output_len) : ov::Dimension::dynamic()}; } else if (is_inp_mean(input, op)) { input_shape = m_is_static ? ov::PartialShape{1, 1, input->ne[1], m_prefill_chunk_size} : - ov::PartialShape{1, 1, -1, -1}; + ov::PartialShape{1, 1, ov::Dimension::dynamic(), ov::Dimension::dynamic()}; + + } else if (is_inp_scale_rows(input, op)) { + input_shape = ov::PartialShape{1, 1, m_is_static ? (m_is_prefill ? m_prefill_chunk_size : 1) : -1, 1}; } else if (is_inp_mask(input, op)) { // mask if (m_is_static) { input_shape = ov::PartialShape{1, 1, m_is_prefill ? m_prefill_chunk_size : 1, m_model_params.ctx}; } else if (m_is_stateful) { - input_shape = ov::PartialShape{1, 1, -1, -1}; + input_shape = ov::PartialShape{1, 1, ov::Dimension::dynamic(), ov::Dimension::dynamic()}; } else { - input_shape = ov::PartialShape{-1, 1, -1, -1}; + input_shape = ov::PartialShape{ov::Dimension::dynamic(), 1, ov::Dimension::dynamic(), ov::Dimension::dynamic()}; } } else if (is_recurrent_cache(input)) { @@ -1122,11 +1014,12 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op, const bool is_flat_kv = (input->ne[1] == 1 && input->ne[2] == 1 && input->ne[3] == 1); if (!m_is_static && !is_flat_kv) { // do not fix ctx size to make llama-bench work across test params - input_shape[2] = -1; + input_shape[2] = ov::Dimension::dynamic(); } - if (is_stateful() && !is_flat_kv) { + if ((is_stateful() && !is_flat_kv) || is_kvcache_4d(input)) { // Convert stateless KV cache layout [1, 1, seq, n_heads_kv * head_size] - // to stateful layout [1, seq, n_heads_kv, head_size]. + // to stateful layout [1, seq, n_heads_kv, head_size]. Stateless GPU graphs use the same + // layout for the caches that qualify (see is_kvcache_4d), bound over the same memory. // NOTE: Gemma4 uses per-layer-type KV shapes, so no single scalar describes every // layer. E2B varies only the head size (sliding 256, full 512); 12B also varies the // head COUNT (sliding 8 x 256, full 1 x 512). Take the head count for this tensor's @@ -1144,19 +1037,21 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op, } else if (is_kv_idx(input, op)) { // kv update index - int len = m_is_static ? (m_is_prefill ? m_prefill_chunk_size : 1) : -1; + const ov::Dimension len = m_is_static ? ov::Dimension(m_is_prefill ? m_prefill_chunk_size : 1) : + ov::Dimension::dynamic(); input_shape = ov::PartialShape{1, 1, 1, len}; } else if (is_inp_s_copy(input, op) || is_s_copy_leaf(input)) { // On NPU the total slot count (n_seq_max) is fixed at translation time, so the s_copy // index list has a static length; on CPU/GPU it may change across compiles (defrag). - input_shape = m_is_static ? ov::PartialShape{get_shape(input)} : ov::PartialShape{1, 1, 1, -1}; + input_shape = m_is_static ? ov::PartialShape{get_shape(input)} : + ov::PartialShape{1, 1, 1, ov::Dimension::dynamic()}; } else { input_shape = ov::PartialShape{get_shape(input)}; } if (dynamic_dim_index != -1 && m_model_is_splitted) { - input_shape[3 - dynamic_dim_index] = -1; + input_shape[3 - dynamic_dim_index] = ov::Dimension::dynamic(); } if (op->op == GGML_OP_SOFT_MAX && op->src[1] != nullptr && op->src[1]->op == GGML_OP_NONE && op->src[1]->flags & GGML_TENSOR_FLAG_INPUT && op->src[1] == input) { @@ -1166,8 +1061,8 @@ ov::PartialShape GgmlOvDecoder::get_graph_input_shape(const ggml_tensor * op, input_shape[2] = seq_active; input_shape[3] = seq_active; } else { - input_shape[2] = -1; - input_shape[3] = -1; + input_shape[2] = ov::Dimension::dynamic(); + input_shape[3] = ov::Dimension::dynamic(); } } return input_shape; @@ -1202,65 +1097,74 @@ bool GgmlOvDecoder::is_s_copy_leaf(const ggml_tensor * tensor) const { return false; } -void GgmlOvDecoder::add_extra_inputs() { - // Extra inputs: - // 1. `attention_size`, used in FLASH_ATTN where the shape of the matmul's are 256 aligned, - // see llama_kv_cache_unified::get_n_kv and llama_kv_cache_unified::get_padding. - // 2. `n_seq_active` and `seq_active_start`, used in FLASH_ATTN_EXT to indicate the active sequences in the batch +bool GgmlOvDecoder::is_external_view(const ggml_cgraph * cgraph, const ggml_tensor * tensor) { + if (cgraph == nullptr || tensor == nullptr || tensor->op != GGML_OP_VIEW || tensor->view_src == nullptr) { + return false; + } + const auto nodes_end = cgraph->nodes + cgraph->n_nodes; + if (std::find(cgraph->nodes, nodes_end, tensor) != nodes_end) { + return false; + } + const ggml_tensor * root = tensor; + while (root->op == GGML_OP_VIEW && root->src[0] != nullptr) { + root = root->src[0]; + } + if (is_kvcache(root, nullptr)) { + return false; + } + const auto leafs_end = cgraph->leafs + cgraph->n_leafs; + return std::find(cgraph->nodes, nodes_end, root) == nodes_end && + std::find(cgraph->leafs, leafs_end, root) == leafs_end; +} - auto create_1d_input = [this](const std::string & name, int64_t value, bool force_parameter = false) { - m_model_extra_inputs[name] = {ov::element::i64, ov::Shape{1}, value, force_parameter || !m_is_static}; - }; +bool GgmlOvDecoder::is_external_view_input(const ggml_tensor * tensor) const { + return is_external_view(m_cgraph, tensor); +} - if (m_compute_params.attention_size != -1) { - create_1d_input("attention_size", m_compute_params.attention_size); +std::optional GgmlOvDecoder::get_runtime_input_value(const std::string & name) const { + if (name == "cache_rs_reset_idx") { + return std::max(0, m_compute_params.cache_rs_reset_idx); } - if (m_compute_params.attention_size_static != -1) { - create_1d_input("attention_size_static", m_compute_params.attention_size_static); + if (name == "cache_rs_reset_len") { + // A step without a reset must not reuse the previous step's reset range. + return m_compute_params.cache_rs_reset_idx < 0 ? 0 : m_compute_params.cache_rs_reset_len; } - if (m_compute_params.attention_size_swa != -1) { - create_1d_input("attention_size_swa", m_compute_params.attention_size_swa); + // KV-cache reads (see describe_kv_view). + if (name == "attention_size") { + return m_compute_params.attention_size; } - // only the stateful SWA mask consumes this - if (is_stateful() && m_compute_params.swa_window != -1) { - create_1d_input("swa_window", m_compute_params.swa_window); + if (name == "attention_size_swa") { + return m_compute_params.attention_size_swa; } - create_1d_input("n_seq_active", m_compute_params.n_seq_active); - create_1d_input("seq_active_start", m_compute_params.seq_active_start); - create_1d_input("seq_active_end", m_compute_params.seq_active_start + m_compute_params.n_seq_active); - if (m_compute_params.token_len_per_seq != -1) { - create_1d_input("token_len_per_seq", m_compute_params.token_len_per_seq); + if (name == "seq_active_start") { + return m_compute_params.seq_active_start; } - // create_1d_input("token_len", m_compute_params.token_len_per_seq * m_compute_params.n_seq_active); - - if (m_compute_params.cache_rs_reset_idx != -1 && m_model_params.n_rs_slots != 1) { - // Whether/which cache slot to reset varies per compute call (e.g. a new sequence starting - // vs. continued decoding). can_reuse_statically() does not invalidate the cached static - // model on ComputeParams changes, so these must stay runtime Parameters even when static - // (scale.cpp op_case 1 only uses them in value comparisons, never as Slice bounds, so this - // does not reintroduce dynamic shapes). - create_1d_input("cache_rs_reset_idx", m_compute_params.cache_rs_reset_idx, /*force_parameter=*/true); - create_1d_input("cache_rs_reset_len", m_compute_params.cache_rs_reset_len, /*force_parameter=*/true); + if (name == "swa_window") { + // Stateful sliding-window mask (see rewrite_stateful_masks). An unknown window keeps every cached + // position, which matches ggml as long as the context still fits in the window. + return m_compute_params.swa_window > 0 ? m_compute_params.swa_window : std::numeric_limits::max(); } - - if (m_compute_params.s_copy_active_slot_len != -1) { - create_1d_input("s_copy_active_slot_len", m_compute_params.s_copy_active_slot_len); - if (m_is_static) { - // Number of real tokens in the current prefill chunk. The last chunk is padded with - // fabricated token ids; attention masks them out, but the recurrent (GDN/conv) path - // would otherwise fold them into cache_r/cache_s permanently. Varies per chunk, so it - // must stay a runtime Parameter; it is only compared against a Range or used as Gather - // indices, so it does not make any shape dynamic. - create_1d_input("chunk_valid_len", get_static_n_tokens(), /*force_parameter=*/true); + const std::string slot_prefix = "rs_slot_begin_"; + const std::string src_prefix = "rs_src_begin_"; + const auto writeback = [&](const std::string & key) -> const ComputeParams::RsWriteback & { + const auto it = m_compute_params.rs_writebacks.find(key); + if (it == m_compute_params.rs_writebacks.end()) { + std::string known; + for (const auto & entry : m_compute_params.rs_writebacks) { + known += " '" + entry.first + "'"; + } + throw std::runtime_error("ggml-openvino: no recurrent-state writeback '" + key + "' for input '" + name + + "'; graph has:" + known); } + return it->second; + }; + if (name.compare(0, slot_prefix.size(), slot_prefix) == 0) { + return writeback(name.substr(slot_prefix.size())).slot_begin; } - - for (const auto & [node_name, writeback] : m_compute_params.rs_writebacks) { - create_1d_input("rs_slot_begin_" + node_name, writeback.slot_begin); - if (!m_is_static && writeback.src_begin >= 0) { - create_1d_input("rs_src_begin_" + node_name, writeback.src_begin); - } + if (name.compare(0, src_prefix.size(), src_prefix) == 0) { + return writeback(name.substr(src_prefix.size())).src_begin; } + return std::nullopt; } bool GgmlOvDecoder::node_is_used_as_src(const int node_idx) { @@ -1278,7 +1182,16 @@ bool GgmlOvDecoder::node_is_used_as_src(const int node_idx) { void GgmlOvDecoder::compute_model_inputs() { m_model_inputs.clear(); + m_model_input_nodes.clear(); m_inputs.clear(); + m_kvcache_4d.clear(); + // remember the KV caches that get_graph_input_shape gave the 4-D layout, so that they are bound with that shape + auto note_kvcache_4d = [this](const ggml_tensor * input, const ggml_tensor * op, const ov::PartialShape & shape) { + if (!m_is_stateful && is_kvcache(input, op) && shape.rank().is_static() && shape.rank().get_length() == 4 && + shape[1].is_dynamic()) { + m_kvcache_4d.insert(input); + } + }; for (int i = 0; i < m_cgraph->n_nodes; i++) { ggml_tensor * node = m_cgraph->nodes[i]; // the node op is NONE means this node maybe as input of later nodes, we should add it to model inputs for this node. @@ -1288,6 +1201,7 @@ void GgmlOvDecoder::compute_model_inputs() { m_inputs[node_name] = node; m_model_inputs[node_name] = {get_ov_type(node), get_graph_input_shape(node, nullptr, m_node_dynamic_dims[node])}; + note_kvcache_4d(node, nullptr, m_model_inputs[node_name].shape); } continue; } @@ -1333,14 +1247,73 @@ void GgmlOvDecoder::compute_model_inputs() { m_model_params.kv_names.push_back(src_name); } } - // Resolve nested VIEW nodes by following src[0] until the first non-VIEW tensor. - while (src->op == GGML_OP_VIEW && src->src[0] != nullptr) { - src = src->src[0]; + // Resolve nested VIEW nodes by following src[0] until the first non-VIEW tensor. A VIEW whose + // storage root lives outside this graph (e.g. the s_copy index slices computed by an earlier + // scheduler split) is bound with its own data and extent, under the name its consumer uses. + const ggml_tensor * view_root = src; + while (view_root->op == GGML_OP_VIEW && view_root->src[0] != nullptr) { + view_root = view_root->src[0]; + } + if (view_root != src && !is_external_view_input(src)) { + src = const_cast(view_root); src_name = get_tensor_ov_name(m_cgraph, src); } m_inputs[src_name] = src; m_model_inputs[src_name] = {get_ov_type(src), get_graph_input_shape(node, src, m_node_dynamic_dims[src])}; + note_kvcache_4d(src, node, m_model_inputs[src_name].shape); + } + } + + compute_kv_views(); + + // Slot offsets are independent runtime information, even for a static-shape model. + if (!m_naive) { + if (m_compute_params.cache_rs_reset_idx != -1 && m_model_params.n_rs_slots != 1) { + m_model_inputs["cache_rs_reset_idx"] = {ov::element::i64, ov::PartialShape{1}}; + m_model_inputs["cache_rs_reset_len"] = {ov::element::i64, ov::PartialShape{1}}; + } + for (const auto & entry : m_compute_params.rs_writebacks) { + m_model_inputs["rs_slot_begin_" + entry.first] = {ov::element::i64, ov::PartialShape{1}}; + // Only a writeback that copies a moving source window has a source offset. + if (entry.second.src_begin >= 0) { + m_model_inputs["rs_src_begin_" + entry.first] = {ov::element::i64, ov::PartialShape{1}}; + } + } + } + for (const auto & input : m_model_inputs) { + m_model_input_nodes[input.first] = create_parameter(input.first, input.second); + } + + // attention_size(_swa) is the key dimension of the KQ mask. Read it from the mask shape inside the model instead + // of binding it as a runtime input: the GPU plugin then gets the KV slice shapes from shape inference, rather than + // waiting for the input value on the device, and can keep the fused attention and StatelessKV kernels. + if (!m_is_static && !m_is_stateful && ggml_openvino_shape_from_mask_enabled()) { + const auto from_mask = [this](const std::string & name, const std::string & mask) { + const auto input_it = m_model_input_nodes.find(name); + const auto mask_it = m_model_input_nodes.find(mask); + if (input_it == m_model_input_nodes.end() || mask.empty() || mask_it == m_model_input_nodes.end()) { + return; + } + auto mask_shape = std::make_shared(mask_it->second, ov::element::i64); + auto key_dim = std::make_shared( + mask_shape, ov::op::v0::Constant::create(ov::element::i64, {1}, {3}), + ov::op::v0::Constant::create(ov::element::i64, {}, {0})); + key_dim->set_friendly_name(name); + input_it->second = key_dim; + }; + from_mask("attention_size", m_compute_params.mask_name); + from_mask("attention_size_swa", m_compute_params.mask_swa_name); + } + + // The gguf frontend seeds get_model_inputs() into its tensor map but seeds no weights; it + // expects the decoder to supply them. This backend already decoded every weight into a ready + // f32 decompression subgraph at load time (ggml_openvino_*_weight_extra::weight_node, surfaced + // via m_model_weights), so hand those pre-built nodes over here rather than have translate_weight + // re-decode raw bytes -- which the backend has overwritten in place with the extracted layout. + for (const auto & [weight_name, weight_node] : m_model_weights) { + if (weight_node != nullptr) { + m_model_input_nodes[weight_name] = weight_node; } } } @@ -1400,6 +1373,59 @@ const ggml_tensor * GgmlOvDecoder::get_tensor_used_op(const ggml_tensor * tensor return nullptr; } +bool GgmlOvDecoder::is_kvcache_4d(const ggml_tensor * cache) const { + static const bool disabled = ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_KV_4D") != 0; + if (disabled || m_is_static || m_is_stateful || m_model_is_splitted || !ggml_openvino_is_gpu()) { + return false; + } + if (cache->op != GGML_OP_NONE || cache->view_src != nullptr || !is_kvcache(cache, nullptr)) { + return false; + } + // one sequence, [n_heads_kv * head_size, ctx, 1, 1]; whisper's flat KV buffer has ctx == 1 + if (cache->ne[1] <= 1 || cache->ne[2] != 1 || cache->ne[3] != 1) { + return false; + } + const int n_heads_kv = get_n_heads_kv_for_tensor(cache); + if (n_heads_kv <= 0 || cache->ne[0] % n_heads_kv != 0) { + return false; + } + // check the nodes that take the cache itself as a source (views of those views, such as the PERMUTE of the read + // VIEW, are checked through them) + auto feeds_only_permutes = [this](const ggml_tensor * view) { + for (int i = 0; i < m_cgraph->n_nodes; i++) { + const auto * node = m_cgraph->nodes[i]; + for (int j = 0; j < GGML_MAX_SRC; j++) { + if (node->src[j] == view && node->op != GGML_OP_PERMUTE) { + return false; + } + } + } + return true; + }; + for (int i = 0; i < m_cgraph->n_nodes; i++) { + const auto * node = m_cgraph->nodes[i]; + bool uses_cache = false; + for (int j = 0; j < GGML_MAX_SRC; j++) { + uses_cache = uses_cache || node->src[j] == cache; + } + if (!uses_cache) { + continue; + } + // write: whole rows into the cache itself + if (node->op == GGML_OP_SET_ROWS && node->src[2] == cache && node->src[0]->ne[0] == cache->ne[0]) { + continue; + } + // read: the first rows of the cache, each row split into heads, permuted for attention + if (node->op == GGML_OP_VIEW && node->src[0] == cache && node->view_offs == 0 && + node->ne[0] * node->ne[1] == cache->ne[0] && node->nb[2] == cache->nb[1] && node->ne[3] == 1 && + feeds_only_permutes(node)) { + continue; + } + return false; + } + return true; +} + const ggml_tensor * GgmlOvDecoder::get_tensor_from_name(const std::string & name) const { for (int i = 0; i < m_cgraph->n_nodes; i++) { const auto * node = m_cgraph->nodes[i]; @@ -1770,220 +1796,59 @@ ov::element::Type GgmlOvDecoder::get_ov_type(const ggml_tensor * tensor) { } ov::PartialShape GgmlOvDecoder::get_input_shape(int node_idx, const std::string & name) const { + // Translators use this as the static GGML shape metadata. The actual OpenVINO + // tensors carry dynamic extents from Parameters created with get_graph_input_shape(). return ov::PartialShape(get_shape(m_node_info_list[node_idx].node_inputs.at(name))); } -std::vector GgmlOvDecoder::get_input_stride(int node_idx, const std::string & name) const { - return get_stride(m_node_info_list[node_idx].node_inputs.at(name)); -} - -size_t GgmlOvDecoder::get_view_input_size(int node_idx, const std::string & name) const { - auto it = m_node_info_list[node_idx].node_inputs_views.find(name); - if (it != m_node_info_list[node_idx].node_inputs_views.end()) { - return it->second.size(); - } - return 0; -} - -size_t GgmlOvDecoder::get_view_input_offset(int node_idx, const std::string & name, size_t view_index) const { - auto it = m_node_info_list[node_idx].node_inputs_views.find(name); - if (it != m_node_info_list[node_idx].node_inputs_views.end()) { - if (view_index < it->second.size()) { - return it->second[view_index].second->view_offs; - } - } - return 0; -} - -size_t GgmlOvDecoder::get_view_input_src_offset(int node_idx, const std::string & name, size_t view_index) const { - auto it = m_node_info_list[node_idx].node_inputs_views.find(name); - if (it != m_node_info_list[node_idx].node_inputs_views.end()) { - if (view_index < it->second.size()) { - auto * view_tensor = it->second[view_index].second; - if (view_tensor && view_tensor->src[0]) { - return view_tensor->src[0]->view_offs; - } - } - } - return 0; -} - -std::vector GgmlOvDecoder::get_view_input_stride(int node_idx, - const std::string & name, - size_t view_index) const { - auto it = m_node_info_list[node_idx].node_inputs_views.find(name); - if (it != m_node_info_list[node_idx].node_inputs_views.end()) { - if (view_index < it->second.size()) { - return get_stride(it->second[view_index].second); - } - } - return {}; -} - -std::vector GgmlOvDecoder::get_view_input_src_stride(int node_idx, - const std::string & name, - size_t view_index) const { - auto it = m_node_info_list[node_idx].node_inputs_views.find(name); - if (it != m_node_info_list[node_idx].node_inputs_views.end()) { - if (view_index < it->second.size()) { - auto * view_tensor = it->second[view_index].second; - if (view_tensor && view_tensor->src[0]) { - return get_stride(view_tensor->src[0]); - } - } - } - return {}; -} - -ov::Shape GgmlOvDecoder::get_view_input_ggml_shape(int node_idx, const std::string & name, size_t view_index) const { - auto it = m_node_info_list[node_idx].node_inputs_views.find(name); - if (it != m_node_info_list[node_idx].node_inputs_views.end()) { - if (view_index < it->second.size()) { - return get_shape(it->second[view_index].second); - } - } - return {}; -} - -ov::Shape GgmlOvDecoder::get_view_input_src_ggml_shape(int node_idx, - const std::string & name, - size_t view_index) const { - auto it = m_node_info_list[node_idx].node_inputs_views.find(name); - if (it != m_node_info_list[node_idx].node_inputs_views.end()) { - if (view_index < it->second.size()) { - auto * view_tensor = it->second[view_index].second; - if (view_tensor && view_tensor->src[0]) { - return get_shape(view_tensor->src[0]); - } - } - } - return {}; -} - -ov::PartialShape GgmlOvDecoder::get_view_input_ov_shape(int node_idx, - const std::string & name, - size_t view_index) const { - auto it = m_node_info_list[node_idx].node_inputs_views.find(name); - if (it != m_node_info_list[node_idx].node_inputs_views.end()) { - if (view_index < it->second.size()) { - auto * tensor = it->second[view_index].second; - ov::PartialShape shape = ov::PartialShape{get_shape(tensor)}; - - // Check if this tensor has a dynamic dimension - auto dynamic_it = m_node_dynamic_dims.find(tensor); - if (dynamic_it != m_node_dynamic_dims.end() && dynamic_it->second != -1) { - int dynamic_dim_index = dynamic_it->second; - // GGML uses reverse indexing, so convert to OpenVINO indexing - shape[3 - dynamic_dim_index] = m_is_static ? get_static_n_tokens() : -1; - } - - return shape; - } - } - return {}; -} - -ov::PartialShape GgmlOvDecoder::get_view_input_src_ov_shape(int node_idx, - const std::string & name, - size_t view_index) const { - auto it = m_node_info_list[node_idx].node_inputs_views.find(name); - if (it != m_node_info_list[node_idx].node_inputs_views.end()) { - if (view_index < it->second.size()) { - auto * view_tensor = it->second[view_index].second; - if (view_tensor && view_tensor->src[0]) { - auto * src_tensor = view_tensor->src[0]; - ov::PartialShape shape = ov::PartialShape{get_shape(src_tensor)}; - - // Check if this tensor has a dynamic dimension - auto dynamic_it = m_node_dynamic_dims.find(src_tensor); - if (dynamic_it != m_node_dynamic_dims.end() && dynamic_it->second != -1) { - int dynamic_dim_index = dynamic_it->second; - // GGML uses reverse indexing, so convert to OpenVINO indexing - shape[3 - dynamic_dim_index] = m_is_static ? get_static_n_tokens() : -1; - } - - return shape; - } - } - } - return {}; -} - -std::string GgmlOvDecoder::get_view_input_name(int node_idx, const std::string & name, size_t view_index) const { - auto it = m_node_info_list[node_idx].node_inputs_views.find(name); - if (it != m_node_info_list[node_idx].node_inputs_views.end()) { - if (view_index < it->second.size()) { - return it->second[view_index].first; - } - } - return ""; -} - -std::string GgmlOvDecoder::get_view_input_src_name(int node_idx, const std::string & name, size_t view_index) const { - auto it = m_node_info_list[node_idx].node_inputs_views.find(name); - if (it != m_node_info_list[node_idx].node_inputs_views.end()) { - if (view_index < it->second.size()) { - auto * view_tensor = it->second[view_index].second; - if (view_tensor && view_tensor->src[0]) { - return get_tensor_ov_name(m_cgraph, view_tensor->src[0]); - } - } - } - return ""; -} - -ov::element::Type GgmlOvDecoder::get_input_type(int node_idx, const std::string & name) const { - return get_ov_type(m_node_info_list[node_idx].node_inputs.at(name)); -} - size_t GgmlOvDecoder::get_input_size() const { + if (m_node_idx >= 0) { + return get_input_size(m_node_idx); + } return m_model_inputs.size(); } size_t GgmlOvDecoder::get_input_size(int node_idx) const { - return m_node_info_list[node_idx].node_inputs_names.size(); + return get_input_names(node_idx).size(); } std::vector GgmlOvDecoder::get_input_names(int node_idx) const { - return m_node_info_list[node_idx].node_inputs_names; + const auto & info = m_node_info_list[node_idx]; + const auto * node = info.node; + auto inputs = info.node_inputs_names; + if (node->op == GGML_OP_SCALE && info.node_op_case == 1 && m_model_inputs.count("cache_rs_reset_idx")) { + inputs[0] = get_tensor_ov_name(m_cgraph, node->view_src); + inputs.push_back("cache_rs_reset_idx"); + inputs.push_back("cache_rs_reset_len"); + } else if (node->op == GGML_OP_CPY && info.node_op_case >= 1 && info.node_op_case <= 3 && + m_model_inputs.count("rs_slot_begin_" + info.node_name)) { + if (info.node_op_case == 1 || info.node_op_case == 2) { + inputs[0] = get_tensor_ov_name(m_cgraph, node->src[0]->src[0]); + } + inputs[1] = get_tensor_ov_name(m_cgraph, node->view_src); + inputs.push_back("rs_slot_begin_" + info.node_name); + if (info.node_op_case == 2) { + inputs.push_back("rs_src_begin_" + info.node_name); + } + } else if (node->op == GGML_OP_CPY && info.node_op_case >= 4 && info.node_op_case <= 6) { + // Flat-buffer writes (conv_states_all, whisper KV): update the whole buffer from a runtime element offset. + inputs[1] = get_tensor_ov_name(m_cgraph, node->view_src); + if (m_model_inputs.count("rs_slot_begin_" + info.node_name)) { + inputs.push_back("rs_slot_begin_" + info.node_name); + } + } + inputs.insert(inputs.end(), info.kv_view_inputs.begin(), info.kv_view_inputs.end()); + return inputs; } ov::PartialShape GgmlOvDecoder::get_output_shape(int node_idx) const { auto * ggml_tensor = m_node_info_list[node_idx].node; + // Keep translator-facing output metadata static. Shape polymorphism is already + // represented by the graph's dynamic input Parameters and propagated by OV ops. return ov::PartialShape(get_shape(ggml_tensor)); } -ov::element::Type GgmlOvDecoder::get_output_type(const int node_idx) const { - return get_ov_type(m_node_info_list[node_idx].node); -} - -std::vector GgmlOvDecoder::get_output_stride(int node_idx) const { - auto * ggml_tensor = m_node_info_list[node_idx].node; - return get_stride(ggml_tensor); -} - std::vector GgmlOvDecoder::get_output_names(int node_idx) const { - auto * node = m_node_info_list[node_idx].node; - if (node->op == GGML_OP_GATED_DELTA_NET && !m_model_params.has_rs_rollback) { - std::string attn_name; - std::string state_name; - for (int i = node_idx + 1; i < m_cgraph->n_nodes; i++) { - auto * consumer = m_cgraph->nodes[i]; - if (consumer->op != GGML_OP_VIEW || consumer->src[0] != node) { - continue; - } - // GGML packs [attention | state]. The attention VIEW starts at offset 0 and the - // state VIEW starts after the token-dependent attention segment. - auto & name = consumer->view_offs == 0 ? attn_name : state_name; - if (!name.empty()) { - return {m_node_info_list[node_idx].node_name}; - } - name = get_tensor_ov_name(m_cgraph, consumer); - } - if (!attn_name.empty() && !state_name.empty()) { - return {attn_name, state_name}; - } - } return {m_node_info_list[node_idx].node_name}; } @@ -2009,8 +1874,7 @@ bool GgmlOvDecoder::is_view_like_alias_of(int node_idx, const std::string & view } const std::string & GgmlOvDecoder::get_op_name() const { - static const std::string unknown_name = "UNKNOWN_OP_NAME"; - return unknown_name; + return m_node_idx >= 0 ? get_op_name(m_node_idx) : m_node_info_list.front().node_name; } int32_t GgmlOvDecoder::get_op_dynamic_dim(int node_idx) const { @@ -2025,25 +1889,313 @@ const std::string & GgmlOvDecoder::get_op_name(int node_idx) const { return m_node_info_list[node_idx].node_name; } -int32_t * GgmlOvDecoder::get_input_op_params(int node_idx, const std::string & name) const { - return m_node_info_list[node_idx].node_inputs.at(name)->op_params; +void GgmlOvDecoder::visit_subgraph(std::function)> node_visitor) const { + // Copy the decoder's (heavy) model-scope state once, then rebind it to each node by + // advancing m_node_idx. translate_node consumes the bound decoder synchronously and never + // retains it past the call, so a single reused instance is safe -- and it keeps the whole + // model walk O(n_nodes) instead of deep-copying every map per node. + auto decoder = std::make_shared(*this); + for (int node_idx = 0; node_idx < m_cgraph->n_nodes; node_idx++) { + decoder->m_node_idx = node_idx; + node_visitor(decoder); + } } -int32_t * GgmlOvDecoder::get_output_op_params(int node_idx) const { - return m_node_info_list[node_idx].node->op_params; -} +ov::Any GgmlOvDecoder::get_attribute(const std::string & name) const { + const ggml_tensor * node = m_node_idx >= 0 ? m_node_info_list[m_node_idx].node : nullptr; + if (name == "rope_config") { + return rope_config_from_params(node != nullptr && node->op == GGML_OP_ROPE ? node->op_params : m_model_params.rope_params, + m_model_params.mixed_rope_params); + } + if (name == "swa_window_size") { + return m_compute_params.swa_window; + } + if (node == nullptr) { + return ov::Any{}; + } + const int op_case = m_node_info_list[m_node_idx].node_op_case; -size_t GgmlOvDecoder::get_output_op_offset(int node_idx) const { - return m_node_info_list[node_idx].node->view_offs; -} + const bool recurrent_copy = node->op == GGML_OP_CPY && op_case >= 1 && op_case <= 3 && + m_model_inputs.count("rs_slot_begin_" + get_op_name(m_node_idx)); + switch (node->op) { + case GGML_OP_NONE: { + if (name == "data" || name == "quant_type") { + const auto node_name = get_tensor_ov_name(m_cgraph, node); + if (m_model_weights.find(node_name) != m_model_weights.end()) { + if (name == "data") { + const size_t nbytes = ggml_nbytes(node); + return ov::Tensor(ov::element::u8, ov::Shape{nbytes}, node->data); + } + return std::string(ggml_type_name(node->type)); + } + } + break; + } -void GgmlOvDecoder::visit_subgraph(std::function, int node_idx)> node_visitor) const { - for (int node_idx = 0; node_idx < m_cgraph->n_nodes; node_idx++) { - if (m_cgraph->nodes[node_idx]->op == GGML_OP_NONE) { - continue; + case GGML_OP_SCALE: + if (name == "axis" && op_case == 1 && m_model_inputs.count("cache_rs_reset_idx")) return int64_t{2}; + if (name == "scale") return op_param_float(node, 0); + if (name == "bias") return op_param_float(node, 1); + break; + + case GGML_OP_CPY: + // A full single-slot recurrent-state writeback (op_case 7: GDN state, 8: conv state) replaces the + // whole flat cache; it is expressed as an update at offset 0 so the cache storage is rebound. + if (name == "axis") { + if (recurrent_copy) return int64_t{2}; + if (op_case >= 4 && op_case <= 8) return int64_t{3}; + } + if (name == "destination_offset" && op_case >= 4 && op_case <= 6) { + return static_cast(node->view_offs / ggml_type_size(node->type)); + } + if (name == "destination_offset" && (op_case == 7 || op_case == 8)) { + return int64_t{0}; + } + if (name == "source_slice" && recurrent_copy) { + if (op_case == 1) { + // The state tail of the GATED_DELTA_NET result holds exactly the copied elements, in rows + // of the result's packed width (S_v * H). + const ggml_tensor * gdn = node->src[0]->src[0]; + OPENVINO_ASSERT(gdn->ne[0] > 0 && ggml_nelements(node) % gdn->ne[0] == 0, + "GDN copy must cover whole packed rows"); + return std::vector{2, -ggml_nelements(node) / gdn->ne[0], -1}; + } + if (op_case == 2) return std::vector{3, 0, node->src[0]->ne[0]}; + } + if (name == "reshape_target") { + if (recurrent_copy && op_case <= 2) return std::vector{1, 1, -1, node->ne[0]}; + if (op_case >= 4 && op_case <= 8) return std::vector{1, 1, 1, -1}; + } + break; + + case GGML_OP_RESHAPE: + if (name == "reshape_target") { + const auto shape = get_shape(node); + std::vector target(shape.begin(), shape.end()); + const int dynamic_dim = m_naive ? -1 : get_op_dynamic_dim(m_node_idx); + if (dynamic_dim >= 0) target[GGML_MAX_DIMS - 1 - dynamic_dim] = -1; + // The op_case axis is inferred unless the dynamic axis already is (a Reshape takes one -1). + const bool has_inferred = dynamic_dim >= 0; + if (op_case == 1) { + target[0] = 0; + if (!has_inferred) target[1] = -1; + } else if (op_case == 2) { + target[0] = 0; + if (!has_inferred) target[2] = -1; + } + return target; + } + if (name == "special_zero") return op_case == 1 || op_case == 2; + break; + + case GGML_OP_CONT: + if (name == "cont_reshape" && !ggml_are_same_shape(node, node->src[0])) { + const auto shape = get_shape(node); + std::vector target(shape.begin(), shape.end()); + const int dynamic_dim = m_naive ? -1 : get_op_dynamic_dim(m_node_idx); + if (dynamic_dim >= 0) target[GGML_MAX_DIMS - 1 - dynamic_dim] = -1; + return target; + } + [[fallthrough]]; + case GGML_OP_VIEW: + if (name == "kv_view" && !m_node_info_list[m_node_idx].kv_view.empty()) { + return m_node_info_list[m_node_idx].kv_view; + } + if (name == "view_slice" || name == "view_reshape") { + auto description = describe_view(node, name == "view_reshape"); + if (!description.empty()) return description; + } + if (name == "output_dynamic_axis") { + const int axis = m_naive ? -1 : get_op_dynamic_dim(m_node_idx); + return axis < 0 ? -1 : GGML_MAX_DIMS - 1 - axis; + } + break; + + case GGML_OP_ROPE: + if (name == "rope_mode") { + switch (node->op_params[2]) { + case GGML_ROPE_TYPE_NORMAL: return std::string("normal"); + case GGML_ROPE_TYPE_NEOX: return std::string("neox"); + case GGML_ROPE_TYPE_IMROPE: return std::string("imrope"); + case GGML_ROPE_TYPE_MROPE: return std::string("mrope"); + case GGML_ROPE_TYPE_VISION: return std::string("vision"); + default: throw std::runtime_error("Unsupported GGML RoPE mode: " + std::to_string(node->op_params[2])); + } + } + break; + + case GGML_OP_RMS_NORM: + case GGML_OP_NORM: + case GGML_OP_L2_NORM: + if (name == "eps") return op_param_float(node, 0); + break; + + case GGML_OP_SOFT_MAX: + if (name == "scale") return op_param_float(node, 0); + if (name == "max_bias") return op_param_float(node, 1); + break; + + case GGML_OP_FLASH_ATTN_EXT: + if (name == "scale") return op_param_float(node, 0); + if (name == "max_bias") return op_param_float(node, 1); + if (name == "kq_soft_cap") return op_param_float(node, 2); + if (name == "has_mask") return node->src[3] != nullptr; + // Stateful graphs keep the fused SDPA, which the stateful-SDPA fusion expects. + if (name == "manual_gqa_attention") return !m_is_stateful && ggml_openvino_manual_gqa_attn_enabled(); + break; + + case GGML_OP_GLU: + // GLU family (SWIGLU / GEGLU / SWIGLU_OAI): op_params[1]=swapped, [2]=alpha, [3]=limit. + if (name == "swapped") return node->op_params[1] != 0; + if (name == "glu_alpha") return op_param_float(node, 2); + if (name == "glu_limit") return op_param_float(node, 3); + break; + + case GGML_OP_ARGSORT: + if (name == "sort_order") return node->op_params[0]; + break; + + case GGML_OP_CLAMP: + if (name == "clamp_min") return op_param_float(node, 0); + if (name == "clamp_max") return op_param_float(node, 1); + break; + + case GGML_OP_CONCAT: + if (name == "concat_axis") return node->op_params[0]; + break; + + case GGML_OP_FILL: + if (name == "fill_value") return op_param_float(node, 0); + break; + + case GGML_OP_TOP_K: + if (name == "k") return static_cast(node->ne[0]); + break; + + case GGML_OP_REPEAT: + if (name == "repeats") { + OPENVINO_ASSERT(node->src[0] != nullptr, "REPEAT requires a source operand"); + const auto input_shape = get_shape(node->src[0]); + const auto output_shape = get_shape(node); + std::vector repeats(input_shape.size(), 1); + for (size_t axis = 0; axis < repeats.size(); ++axis) { + OPENVINO_ASSERT(input_shape[axis] > 0 && output_shape[axis] % input_shape[axis] == 0, + "REPEAT output dimensions must be integer multiples of the input dimensions"); + repeats[axis] = static_cast(output_shape[axis] / input_shape[axis]); + } + const auto dynamic_dim = get_op_dynamic_dim(m_node_idx); + if (dynamic_dim >= 0) { + const size_t dynamic_axis = repeats.size() - 1 - static_cast(dynamic_dim); + if (repeats[dynamic_axis] != 1) return ov::Any{}; + } + return repeats; + } + break; + + case GGML_OP_PERMUTE: + if (name == "perm") return permute_order_from_params(node->op_params); + break; + + case GGML_OP_IM2COL: + if (name == "im2col_params") return std::vector(node->op_params, node->op_params + 7); + break; + + case GGML_OP_IM2COL_3D: + if (name == "im2col_params") return std::vector(node->op_params, node->op_params + 10); + break; + + case GGML_OP_CONV_2D_DW: + if (name == "conv_params") return std::vector(node->op_params, node->op_params + 6); + break; + + case GGML_OP_CONV_TRANSPOSE_1D: + if (name == "conv_params") return std::vector(node->op_params, node->op_params + 3); + break; + + case GGML_OP_CONV_TRANSPOSE_2D: + if (name == "conv_params") return std::vector(node->op_params, node->op_params + 1); + break; + + case GGML_OP_CONV_3D: + if (name == "conv_params") return std::vector(node->op_params, node->op_params + 12); + break; + + case GGML_OP_UPSCALE: + if (name == "interpolation_mode") return node->op_params[0]; + if (name == "upscale_sizes") return std::vector{node->ne[1], node->ne[0]}; + break; + + case GGML_OP_PAD: + if (name == "pad_params") return std::vector(node->op_params, node->op_params + 8); + if (name == "pad_circular") return node->op_params[8] != 0; + break; + + case GGML_OP_TRI: + if (name == "tri_type") return node->op_params[0]; + break; + + case GGML_OP_POOL_2D: + // {op, k0, k1, s0, s1, p0, p1} + if (name == "pool_params") return std::vector(node->op_params, node->op_params + 7); + break; + + case GGML_OP_ROLL: + if (name == "roll_shifts") return std::vector(node->op_params, node->op_params + 4); + break; + + case GGML_OP_SET: + if (name == "set_offset_elems") { + return static_cast(static_cast(reinterpret_cast(node->op_params)[3]) / + ggml_type_size(node->type)); + } + break; + + default: + break; + } + + if (name == "output_type" || name == "dst_type") return get_ov_type(node); + if (name == "is_swa") { + if (auto layer = extract_layer_from_name(node->name); layer.has_value()) return is_swa_layer(layer.value()) != 0; + return false; + } + if (name == "view_offset_bytes") { + size_t offset = node->view_offs; + if (node->op == GGML_OP_VIEW) { + // view_offs is relative to the storage root; op_params retains the src[0]-relative offset. + memcpy(&offset, node->op_params, sizeof(offset)); } - node_visitor(std::make_shared(*this), node_idx); + return static_cast(offset); } + if (name == "view_input_strides" && node->src[0] != nullptr) return std::vector{(int64_t) node->src[0]->nb[0], (int64_t) node->src[0]->nb[1], (int64_t) node->src[0]->nb[2], (int64_t) node->src[0]->nb[3]}; + if (name == "view_output_strides") return std::vector{(int64_t) node->nb[0], (int64_t) node->nb[1], (int64_t) node->nb[2], (int64_t) node->nb[3]}; + if (name == "input_ggml_shape" && node->src[0] != nullptr) return get_shape(node->src[0]); + if (name == "gdn_state_slots" && node->op == GGML_OP_GATED_DELTA_NET) return static_cast(node->op_params[0]); + if (name == "force_ref") return false; + return ov::Any{}; +} + +ov::PartialShape GgmlOvDecoder::get_input_shape(const std::string & name) const { + return get_input_shape(m_node_idx, name); +} + +int64_t GgmlOvDecoder::get_input_view_element_offset(const std::string & name) const { + if (m_node_idx < 0) return 0; + const auto * view = m_node_info_list[m_node_idx].node_inputs.at(name); + if (view->op != GGML_OP_VIEW) return 0; + return static_cast(view->view_offs / ggml_type_size(view->type)); +} + +std::vector GgmlOvDecoder::get_input_names() const { + return get_input_names(m_node_idx); +} + +ov::PartialShape GgmlOvDecoder::get_output_shape() const { + return get_output_shape(m_node_idx); +} + +std::vector GgmlOvDecoder::get_output_names() const { + return get_output_names(m_node_idx); } std::string GgmlOvDecoder::compute_op_type(const ggml_tensor * node) { @@ -2062,10 +2214,197 @@ const std::string & GgmlOvDecoder::get_op_type(int node_idx) const { } const std::string & GgmlOvDecoder::get_op_type() const { + if (m_node_idx >= 0) { + return get_op_type(m_node_idx); + } static const std::string unknown_op = "UNKNOWN_GGML_OP"; return unknown_op; } +std::vector GgmlOvDecoder::describe_view(const ggml_tensor * node, bool reshape) const { + const ggml_tensor * src = node->src[0]; + if (node->op != GGML_OP_VIEW || src == nullptr) { + return {}; + } + size_t offset = 0; + memcpy(&offset, node->op_params, sizeof(offset)); // relative to src[0] + const int dynamic_src = [&] { + auto it = m_node_dynamic_dims.find(const_cast(src)); + return m_naive || it == m_node_dynamic_dims.end() ? -1 : it->second; + }(); + + // GATED_DELTA_NET packs [attention rows | recurrent-state rows] along ggml axis 1. The attention + // view keeps everything ahead of the trailing state rows, whose count is fixed while the token + // count is not; the state view is that fixed-size tail. + if (src->op == GGML_OP_GATED_DELTA_NET) { + // ggml_gated_delta_net: K snapshot slots (op_params[0]) of S_v rows per sequence. + const int64_t state_rows = static_cast(src->op_params[0]) * src->src[2]->ne[0] * src->src[2]->ne[3]; + if (offset == 0) { + // [n_seqs, tokens, heads, head_size]; n_seqs is fixed for the converted model (same_graph_extents). + if (reshape) return {node->ne[3], -1, node->ne[1], node->ne[0]}; + return {2, 0, -state_rows}; + } + if (reshape) return {node->ne[3], node->ne[2], node->ne[1], node->ne[0]}; + return {2, -state_rows, state_rows}; + } + + // A pure slice: same strides as the source and exactly one axis shortened. Expressing it as an + // explicit slice keeps a dynamic source axis dynamic, which the strided fallback cannot do. + if (reshape) { + return {}; + } + int axis = -1; + for (int i = 0; i < GGML_MAX_DIMS; i++) { + if (node->nb[i] != src->nb[i]) { + return {}; + } + if (node->ne[i] != src->ne[i]) { + if (axis != -1) { + return {}; + } + axis = i; + } + } + if (axis == -1 || ggml_is_quantized(src->type) || node->nb[axis] == 0 || offset % node->nb[axis] != 0) { + return {}; + } + int64_t start = static_cast(offset / node->nb[axis]); + const int64_t length = node->ne[axis]; + // Anything inside the offset must be a whole number of steps along the sliced axis only. + if (start + length > src->ne[axis] || offset >= node->nb[axis] * static_cast(src->ne[axis])) { + return {}; + } + if (dynamic_src == axis && start > 0) { + // A window of the dynamic axis is only expressible when it ends at the axis end. + if (start + length != src->ne[axis]) { + return {}; + } + start = -length; + } + return {GGML_MAX_DIMS - 1 - axis, start, length}; +} + +// Attention reads a KV cache through a VIEW whose live length (n_kv) and first stream (sinfo.s0) move with +// the batch while the converted model is reused. Describe the read for the frontend's "kv_view" and name the +// runtime operands that carry those values. Layouts (ggml order): +// llama.cpp K/V: cache [F, cells, streams], view [D, H, n_kv, ns] +// llama.cpp V, `-fa off`: cache [F, cells, streams], view [n_kv, H, D, ns], cells innermost +// whisper flat buffer: cache [N], view [D, n_kv, H] at a whole-row offset +std::pair, std::vector> GgmlOvDecoder::describe_kv_view( + const ggml_tensor * node, bool & transposed) const { + transposed = false; + if (m_naive || m_is_static || m_is_stateful || node->op != GGML_OP_VIEW) { + return {}; + } + const ggml_tensor * cache = node->src[0]; + if (cache == nullptr || cache->op != GGML_OP_NONE || cache->view_src != nullptr || !is_kvcache(cache, nullptr) || + ggml_is_quantized(cache->type)) { + return {}; + } + // Only attention reads qualify; recurrent state caches share the buffer type but are read by + // SCALE / CPY / GET_ROWS. + bool consumed = false; + bool masked_attention = false; + for (int i = 0; i < m_cgraph->n_nodes; i++) { + const ggml_tensor * user = m_cgraph->nodes[i]; + for (int j = 0; j < GGML_MAX_SRC; j++) { + if (user->src[j] != node) { + continue; + } + if (user->op != GGML_OP_PERMUTE && user->op != GGML_OP_FLASH_ATTN_EXT && user->op != GGML_OP_MUL_MAT) { + return {}; + } + consumed = true; + masked_attention |= user->op == GGML_OP_FLASH_ATTN_EXT && user->src[3] != nullptr; + } + } + if (!consumed) { + return {}; + } + + size_t offset = 0; + memcpy(&offset, node->op_params, sizeof(offset)); // relative to src[0] + const int64_t es = static_cast(ggml_type_size(cache->type)); + const int64_t * ne = node->ne; + const size_t * nb = node->nb; + const auto bytes = [&](int64_t elements) { return static_cast(elements * es); }; + + if (cache->ne[1] == 1 && cache->ne[2] == 1 && cache->ne[3] == 1) { + if (ne[3] != 1 || nb[0] != bytes(1) || nb[2] != bytes(ne[0]) || nb[1] % es != 0) { + return {}; + } + const int64_t width = static_cast(nb[1]) / es; + if (width != ne[0] * ne[2] || offset % bytes(width) != 0 || cache->ne[0] % width != 0) { + return {}; + } + std::vector d{1, width, ne[2], ne[0], 1, static_cast(offset / bytes(width)), 0, 0, 2, 1, 3, + ne[1]}; + // Self-attention grows with the decoded tokens; cross-attention reads the fixed encoder length. + if (masked_attention && m_compute_params.attention_size == ne[1]) { + return {d, {"attention_size"}}; + } + return {d, {}}; + } + + const int64_t width = cache->ne[0]; + const int64_t cells = cache->ne[1]; + const int64_t streams = cache->ne[2]; + const size_t stream_bytes = bytes(width * cells); + if (cache->ne[3] != 1 || offset % stream_bytes != 0 || (ne[3] != 1 && nb[3] != stream_bytes)) { + return {}; + } + std::vector d; + int64_t length = 0; + if (nb[0] == bytes(1) && nb[1] == bytes(ne[0]) && nb[2] == bytes(width) && ne[0] * ne[1] <= width && + ne[2] <= cells) { + d = {streams, width, ne[1], ne[0], ne[3], 0, 0, 0, 1, 2, 3, ne[2]}; + length = ne[2]; + } else if (nb[0] == bytes(1) && nb[2] == bytes(cells) && nb[1] == bytes(ne[2] * cells) && ne[1] * ne[2] <= width && + ne[0] <= cells) { + d = {streams, width, ne[1], ne[2], ne[3], 0, 1, 0, 3, 2, 1, ne[0]}; + length = ne[0]; + transposed = true; + } else { + return {}; + } + const int64_t first_stream = static_cast(offset / stream_bytes); + if (streams > 1 && first_stream != m_compute_params.seq_active_start) { + return {}; + } + const auto layer = extract_layer_from_name(cache->name); + const bool swa = layer.has_value() && is_swa_layer(layer.value()); + const int live_length = swa ? m_compute_params.attention_size_swa : m_compute_params.attention_size; + // The live length must be the runtime value the mask describes; a fixed length would be frozen into a + // model that is reused across decode steps, so leave such a view to the generic path. + if (live_length != length) { + return {}; + } + std::vector inputs{swa ? "attention_size_swa" : "attention_size"}; + if (streams > 1) { + inputs.push_back("seq_active_start"); + } + return {d, inputs}; +} + +void GgmlOvDecoder::compute_kv_views() { + m_transposed_kv_caches.clear(); + for (auto & info : m_node_info_list) { + bool transposed = false; + auto description = describe_kv_view(info.node, transposed); + if (transposed) { + m_transposed_kv_caches.insert(info.node->src[0]); + } + info.kv_view = std::move(description.first); + info.kv_view_inputs = std::move(description.second); + if (info.kv_view.empty()) { + continue; + } + for (const auto & name : info.kv_view_inputs) { + m_model_inputs[name] = {ov::element::i64, ov::PartialShape{1}}; + } + } +} + void GgmlOvDecoder::compute_node_dynamic_dims() { auto visit_node = [&](auto && self, ggml_tensor * node) -> void { if (!node) { @@ -2101,7 +2440,12 @@ void GgmlOvDecoder::compute_node_dynamic_dims() { m_node_dynamic_dims[src] = 0; continue; } - if (node->op == GGML_OP_VIEW && src->op == GGML_OP_NONE && !is_stateful() && !m_model_is_splitted) { + if (is_inp_scale_rows(src, node)) { + m_node_dynamic_dims[src] = 1; + continue; + } + if (node->op == GGML_OP_VIEW && src->op == GGML_OP_NONE && is_kvcache(src, node) && + src->ne[1] > 1 && !m_model_is_splitted) { m_node_dynamic_dims[src] = 1; continue; } @@ -2113,28 +2457,17 @@ void GgmlOvDecoder::compute_node_dynamic_dims() { m_node_dynamic_dims[node] = -1; break; case GGML_OP_GET_ROWS: + // out.ne = {src0.ne[0], idx.ne[0], idx.ne[1], idx.ne[2]}: index axis d lands on output axis d + 1. + // A dynamic src0 only changes which rows exist, never the output extent. m_node_dynamic_dims[node] = -1; if (m_node_dynamic_dims[node->src[1]] != -1) { - auto dynamic_dim_idx = m_node_dynamic_dims[node->src[1]]; - if (dynamic_dim_idx == 0) { - m_node_dynamic_dims[node] = 1; - } else { - auto dynamic_dim_stride = node->src[1]->nb[dynamic_dim_idx] / ggml_type_size(node->src[1]->type) * - ggml_type_size(node->src[0]->type); - for (int i = 0; i < GGML_MAX_DIMS; i++) { - if (dynamic_dim_stride == node->src[0]->nb[i]) { - m_node_dynamic_dims[node] = i; - break; - } - } + const int idx_dyn = m_node_dynamic_dims[node->src[1]]; + if (idx_dyn < GGML_MAX_DIMS - 1) { + m_node_dynamic_dims[node] = idx_dyn + 1; } - // OPENVINO_ASSERT(dynamic_dim_value == node->ne[m_node_dynamic_dims[node]], - // "Dynamic dim value mismatch for node: " + std::string(node->name) + - // " and its src[1]: " + std::string(node->src[1]->name)); } break; case GGML_OP_MUL: - case GGML_OP_MUL_MAT: m_node_dynamic_dims[node] = -1; if (m_node_dynamic_dims[node->src[0]] != -1) { m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[0]]; @@ -2143,17 +2476,28 @@ void GgmlOvDecoder::compute_node_dynamic_dims() { m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[1]]; } break; + case GGML_OP_MUL_MAT: { + // out.ne = {src0.ne[1], src1.ne[1], src1.ne[2], src1.ne[3]}. Axis 0 of both operands is the + // contracted axis and vanishes. src1 (activations / tokens) takes priority over src0 when both + // carry a dynamic axis, matching the token-count axis the rest of the graph tracks. + m_node_dynamic_dims[node] = -1; + const int d0 = m_node_dynamic_dims[node->src[0]]; + const int d1 = m_node_dynamic_dims[node->src[1]]; + if (d1 >= 1) { + m_node_dynamic_dims[node] = d1; + } else if (d0 == 1) { + m_node_dynamic_dims[node] = 0; + } else if (d0 >= 2) { + m_node_dynamic_dims[node] = d0; + } + break; + } case GGML_OP_PERMUTE: m_node_dynamic_dims[node] = -1; if (m_node_dynamic_dims[node->src[0]] != -1) { - auto dynamic_dim_idx = m_node_dynamic_dims[node->src[0]]; - // auto dynamic_dim_value = node->src[0]->ne[dynamic_dim_idx]; - for (int i = 0; i < GGML_MAX_DIMS; i++) { - if (node->op_params[i] == dynamic_dim_idx) { - m_node_dynamic_dims[node] = i; - break; - } - } + // ggml_permute stores op_params[i] = the result axis that source axis i moves to. + const int dynamic_dim_idx = m_node_dynamic_dims[node->src[0]]; + m_node_dynamic_dims[node] = node->op_params[dynamic_dim_idx]; // OPENVINO_ASSERT(dynamic_dim_value == node->ne[m_node_dynamic_dims[node]], // "Dynamic dim value mismatch for node: " + std::string(node->name) + // " and its src[0]: " + std::string(node->src[0]->name)); @@ -2164,25 +2508,43 @@ void GgmlOvDecoder::compute_node_dynamic_dims() { // encodes which source dimension it indexes into, so it uniquely // identifies the dynamic dim even when two dims share the same size. m_node_dynamic_dims[node] = -1; - if (m_node_dynamic_dims[node->src[0]] != -1) { - if (node->src[0]->op == GGML_OP_NONE) { - m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[0]]; - break; + if (node->src[0]->op == GGML_OP_GATED_DELTA_NET) { + // The attention rows are reshaped to [n_seq, tokens, heads, head_size]; the state tail is static. + size_t view_offset = 0; + memcpy(&view_offset, node->op_params, sizeof(view_offset)); + if (view_offset == 0 && m_node_dynamic_dims[node->src[0]] != -1) { + m_node_dynamic_dims[node] = 2; } + break; + } + if (m_node_dynamic_dims[node->src[0]] != -1) { auto dynamic_dim_idx = m_node_dynamic_dims[node->src[0]]; auto dynamic_dim_value = node->src[0]->ne[dynamic_dim_idx]; auto dynamic_dim_stride = node->src[0]->nb[dynamic_dim_idx] / ggml_type_size(node->src[0]->type) * ggml_type_size(node->type); + // Size-1 axes can share a stride (a graph captured with one token, or a single KV head); + // prefer the matching axis that keeps the source extent, then one that is not size 1. for (int i = 0; i < GGML_MAX_DIMS; i++) { - if (node->nb[i] == dynamic_dim_stride) { + if (node->nb[i] != dynamic_dim_stride) { + continue; + } + if (m_node_dynamic_dims[node] == -1 || + (node->ne[m_node_dynamic_dims[node]] == 1 && node->ne[i] > 1)) { + m_node_dynamic_dims[node] = i; + } + if (node->ne[i] == dynamic_dim_value) { m_node_dynamic_dims[node] = i; break; } } - if (m_node_dynamic_dims[node] != -1 && dynamic_dim_value != node->ne[m_node_dynamic_dims[node]]) { + if (m_node_dynamic_dims[node] != -1 && dynamic_dim_value != node->ne[m_node_dynamic_dims[node]] && + !is_kvcache(node->src[0], node)) { m_node_dynamic_dims[node] = -1; - GGML_LOG_WARN("ggml-openvino: dynamic dim value mismatch for VIEW node '%s', src[0]: '%s'\n", - node->name, node->src[0]->name); + // an empty view, e.g. the extra states of a single-slot recurrent cache, always mismatches + if (ggml_nelements(node) > 0) { + GGML_LOG_WARN("ggml-openvino: dynamic dim value mismatch for VIEW node '%s', src[0]: '%s'\n", + node->name, node->src[0]->name); + } } } break; @@ -2278,6 +2640,7 @@ void GgmlOvDecoder::compute_node_dynamic_dims() { } break; case GGML_OP_CONCAT: + m_node_dynamic_dims[node] = -1; for (int i = 0; i < GGML_MAX_DIMS; i++) { if (node->src[0]->ne[i] != node->ne[i]) { m_node_dynamic_dims[node] = i; @@ -2306,7 +2669,7 @@ void GgmlOvDecoder::compute_node_dynamic_dims() { case GGML_OP_SET: case GGML_OP_DIAG: case GGML_OP_TRI: - case GGML_OP_REPEAT: + case GGML_OP_DUP: // Shape-preserving elementwise ops: the dynamic dim is unchanged from src[0]. // DIV/CLAMP are used in the MoE routing-weight normalization // (sum_rows -> clamp -> div). If they are left untracked here the dynamic @@ -2316,13 +2679,38 @@ void GgmlOvDecoder::compute_node_dynamic_dims() { case GGML_OP_DIV: case GGML_OP_CLAMP: case GGML_OP_PAD: - case GGML_OP_UPSCALE: + // Element-wise math (e.g. the squared-ReLU FFN uses SQR), ROLL and UPSCALE keep the dynamic extent. + case GGML_OP_ADD1: + case GGML_OP_SQR: + case GGML_OP_SQRT: + case GGML_OP_LOG: case GGML_OP_SIN: case GGML_OP_COS: - case GGML_OP_LOG: case GGML_OP_ROLL: + case GGML_OP_UPSCALE: m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[0]]; break; + case GGML_OP_TOP_K: + // ne[0] becomes k; the other axes are kept. + m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[0]] > 0 ? m_node_dynamic_dims[node->src[0]] : -1; + break; + case GGML_OP_POOL_2D: + // Pooling resizes ggml axes 0 and 1 only. + m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[0]] >= 2 ? m_node_dynamic_dims[node->src[0]] : -1; + break; + case GGML_OP_REPEAT: + // REPEAT's output shape is supplied by src[1]. Keep a varying repeat factor as a + // runtime shape operand instead of freezing the captured ratio into an attribute. + if (node->src[1] != nullptr) { + m_node_dynamic_dims[node] = m_node_dynamic_dims[node->src[1]]; + } else { + // ggml_repeat_4d has no shape operand: the source's dynamic axis survives only if it is + // not tiled. + const int src_dim = m_node_dynamic_dims[node->src[0]]; + m_node_dynamic_dims[node] = + src_dim >= 0 && node->src[0]->ne[src_dim] == node->ne[src_dim] ? src_dim : -1; + } + break; case GGML_OP_SUM_ROWS: // SUM_ROWS reduces ggml axis 0 to size 1 and preserves all other axes, so the // dynamic dim is preserved unless it was axis 0 (then it is summed away). @@ -2387,6 +2775,9 @@ void GgmlOvDecoder::compute_node_dynamic_dims() { break; } default: + // Record "no tracked dynamic axis" explicitly. Consumers read this map with operator[], which + // would otherwise insert 0 (= dynamic ggml axis 0) for an op that never wrote an entry. + m_node_dynamic_dims[node] = -1; GGML_LOG_DEBUG("ggml-openvino: compute_node_dynamic_dims: unhandled op %s for node '%s'\n", ggml_op_name(node->op), node->name); break; diff --git a/ggml/src/ggml-openvino/ggml-decoder.h b/ggml/src/ggml-openvino/ggml-decoder.h index 33b95340a1b4..bf4e7c418e0e 100644 --- a/ggml/src/ggml-openvino/ggml-decoder.h +++ b/ggml/src/ggml-openvino/ggml-decoder.h @@ -3,18 +3,24 @@ #include "ggml-backend-impl.h" #include "ggml-backend.h" #include "ggml.h" -#include "openvino/decoder.h" +#include "openvino/frontend/gguf/decoder.hpp" #include #include #include #include +#include #include #include #include #include #include +struct BackendInputInfo { + ov::element::Type type; + ov::PartialShape shape; +}; + struct ModelParams { int ctx = -1; int ctx_per_seq = -1; @@ -47,14 +53,15 @@ struct ModelParams { memcmp(rope_params, other.rope_params, sizeof(int32_t) * 16) == 0; } + // swa_layers: the masks and KV write indices of these layers are separate graph inputs bool can_reuse_dynamically(const ModelParams & other) const { - return same_rope_params(other) && n_rs_slots == other.n_rs_slots && - has_rs_rollback == other.has_rs_rollback; + return same_rope_params(other) && n_rs_slots == other.n_rs_slots && has_rs_rollback == other.has_rs_rollback && + swa_layers == other.swa_layers; } bool can_reuse_statically(const ModelParams & other) const { return same_rope_params(other) && ctx == other.ctx && n_rs_slots == other.n_rs_slots && - has_rs_rollback == other.has_rs_rollback; + has_rs_rollback == other.has_rs_rollback && swa_layers == other.swa_layers; } bool kv_buffer_changed(const ModelParams & other) const { return kv_buffer_ctx_id != other.kv_buffer_ctx_id; } @@ -66,6 +73,9 @@ struct ComputeParams { int attention_size = -1; int attention_size_swa = -1; int attention_size_static = -1; // encoder/cross-attn KV fill level (whisper) + // OV names of the KQ mask inputs that n_seq_active (mask->ne[3]) and attention_size(_swa) (mask->ne[0]) were read from + std::string mask_name; + std::string mask_swa_name; // Sliding window width, read back from the band of ggml's own SWA mask. ggml never passes // n_swa down to a backend, but fill_mask() bakes it into the mask contents, so the widest // unmasked row recovers it. Shorter than n_swa while the sequence is still short, which is @@ -115,8 +125,15 @@ struct ComputeParams { // Destination slot offset of each state cache writeback CPY node, keyed by node name. It // changes with the batch (kv head, active sequence count) and, with rollback enabled // (cparams.n_rs_seq > 0), the conv state is written back once per snapshot slot. Passed to the - // cached model as a runtime input. Dynamic models also receive the source-side offset; static - // models use a fixed end-anchored offset in the translator. + // cached model as a runtime input, together with the source-side offset. + + // The active sequence count and the active recurrent slot count change the extents of many + // tensors at once (queries, masks, the s_copy split, the gated-delta-net state rows), and only one + // dynamic axis is tracked per tensor. A converted model therefore holds for one value of each; + // offsets that move with the batch are runtime inputs instead. + bool same_graph_extents(const ComputeParams & other) const { + return n_seq_active == other.n_seq_active && s_copy_active_slot_len == other.s_copy_active_slot_len; + } }; // defined below; declared here because GgmlOvDecoder uses it inline @@ -125,7 +142,7 @@ std::optional extract_layer_from_name(const std::string & name); // detects the MoE expert-plane-sum ADD chain (see definition); used by supports_op too bool is_moe_expert_sum_add(const ggml_tensor * node); -class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { +class GgmlOvDecoder : public ov::frontend::gguf::GgufDecoder { public: static std::string get_tensor_name(const ggml_cgraph * cgraph, const ggml_tensor * tensor); struct NodeInfo { @@ -133,8 +150,11 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { std::string node_name; std::string node_op_type; std::map node_inputs; - std::map>> node_inputs_views; std::vector node_inputs_names; + // A VIEW reading a KV cache with runtime bounds: the frontend "kv_view" descriptor and the runtime + // operands (live length, first stream) appended after the cache. + std::vector kv_view; + std::vector kv_view_inputs; int node_op_case = 0; void * data_addr; }; @@ -153,56 +173,17 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { // Naive graph decoder GgmlOvDecoder(ggml_cgraph * cgraph, std::map> & model_weights); - virtual ov::Any get_attribute(const std::string & name) const override { - return nullptr; - GGML_UNUSED(name); - } - - virtual ov::PartialShape get_input_shape(int node_idx, const std::string & name) const override; - - virtual std::vector get_input_stride(int node_idx, const std::string & name) const override; - - virtual size_t get_view_input_size(int node_idx, const std::string & name) const override; - - virtual size_t get_view_input_offset(int node_idx, const std::string & name, size_t view_index) const override; - - virtual size_t get_view_input_src_offset(int node_idx, const std::string & name, size_t view_index) const override; - - virtual std::vector get_view_input_stride(int node_idx, - const std::string & name, - size_t view_index) const override; - - virtual std::vector get_view_input_src_stride(int node_idx, - const std::string & name, - size_t view_index) const override; - - virtual ov::Shape get_view_input_ggml_shape(int node_idx, - const std::string & name, - size_t view_index) const override; - - virtual ov::Shape get_view_input_src_ggml_shape(int node_idx, - const std::string & name, - size_t view_index) const override; + virtual ov::Any get_attribute(const std::string & name) const override; - virtual ov::PartialShape get_view_input_ov_shape(int node_idx, - const std::string & name, - size_t view_index) const override; + virtual ov::PartialShape get_input_shape(const std::string & name) const override; - virtual ov::PartialShape get_view_input_src_ov_shape(int node_idx, - const std::string & name, - size_t view_index) const override; + virtual int64_t get_input_view_element_offset(const std::string & name) const override; - virtual std::string get_view_input_name(int node_idx, const std::string & name, size_t view_index) const override; - - virtual std::string get_view_input_src_name(int node_idx, - const std::string & name, - size_t view_index) const override; - - virtual ov::element::Type get_input_type(int node_idx, const std::string & name) const override; + virtual ov::PartialShape get_input_shape(int node_idx, const std::string & name) const; virtual size_t get_input_size() const override; - virtual size_t get_input_size(int node_idx) const override; + virtual size_t get_input_size(int node_idx) const; virtual void get_input_node(size_t input_port_idx, std::string & producer_name, @@ -214,57 +195,49 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { GGML_UNUSED(producer_output_port_index); } - virtual std::vector get_input_names(int node_idx) const override; - - virtual ov::PartialShape get_output_shape(int node_idx) const override; - - virtual ov::element::Type get_output_type(int node_idx) const override; + virtual std::vector get_input_names(int node_idx) const; - virtual std::vector get_output_stride(int node_idx) const override; + virtual std::vector get_input_names() const override; - virtual int32_t * get_input_op_params(int node_idx, const std::string & name) const override; + virtual ov::PartialShape get_output_shape(int node_idx) const; - virtual int32_t * get_output_op_params(int node_idx) const override; + virtual ov::PartialShape get_output_shape() const override; - virtual size_t get_output_op_offset(int node_idx) const override; + virtual std::vector get_output_names(int node_idx) const; - virtual std::vector get_output_names(int node_idx) const override; + virtual std::vector get_output_names() const override; - virtual std::string get_inplace_op_src(int node_idx) const override; + virtual std::string get_inplace_op_src(int node_idx) const; - virtual bool is_view_like_alias_of(int node_idx, const std::string & view_src_name) const override; + virtual bool is_view_like_alias_of(int node_idx, const std::string & view_src_name) const; virtual const std::string & get_op_type() const override; - virtual const std::string & get_op_type(int node_idx) const override; + virtual const std::string & get_op_type(int node_idx) const; virtual const std::string & get_op_name() const override; - virtual const std::string & get_op_name(int node_idx) const override; + virtual const std::string & get_op_name(int node_idx) const; - virtual int32_t get_op_dynamic_dim(int node_idx) const override; + virtual int32_t get_op_dynamic_dim(int node_idx) const; - virtual void visit_subgraph( - std::function, int node_idx)> node_visitor) const override; + virtual void visit_subgraph(std::function)> node_visitor) const override; ggml_tensor * get_input_ggml_tensor(const std::string & name) const { return m_inputs.at(name); } - virtual int get_op_case(int node_idx) const override { return m_node_info_list[node_idx].node_op_case; } + // Element type and shape of every model input, for compiled-model reuse checks. + const std::map & get_model_input_infos() const { return m_model_inputs; } - virtual const std::map & get_model_inputs() const override { - return m_model_inputs; + virtual const std::map> & get_model_inputs() const override { + return m_model_input_nodes; } - virtual const std::map & get_model_extra_inputs() const override { - return m_model_extra_inputs; - } + std::optional get_runtime_input_value(const std::string & name) const; - virtual const std::map> & get_model_weights() const override { - return m_model_weights; + virtual std::vector get_model_output_names() const override { + return {m_model_output_names.begin(), m_model_output_names.end()}; } - virtual std::set get_model_output_names() const override { return m_model_output_names; } - const std::map & get_model_outputs() const { return m_model_outputs; } virtual int get_ctx_size() const { return m_model_params.ctx; } @@ -275,7 +248,11 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { virtual int get_n_seq() const { return m_model_params.n_seq; } - virtual int is_swa_layer(int layer) const override { + // True if a KV cache is read with cells as the innermost axis (the `-fa off` V layout). A prefix of + // its rows is not a prefix of its cells, so it must be bound at full size. + bool is_transposed_kv_cache(const ggml_tensor * cache) const { return m_transposed_kv_caches.count(cache) != 0; } + + virtual int is_swa_layer(int layer) const { return std::find(m_model_params.swa_layers.begin(), m_model_params.swa_layers.end(), layer) != m_model_params.swa_layers.end(); } @@ -299,21 +276,25 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { int get_input_len() const { return m_compute_params.input_len; } - virtual int32_t * get_rope_params() const override { return const_cast(m_model_params.rope_params); } + int32_t * get_rope_params() const { return const_cast(m_model_params.rope_params); } - virtual bool has_mixed_rope_params() const override { return m_model_params.mixed_rope_params; } + bool has_mixed_rope_params() const { return m_model_params.mixed_rope_params; } - virtual int get_ssm_state_size() const override { return m_model_params.state_size; } + int get_ssm_state_size() const { return m_model_params.state_size; } - virtual std::map get_kv_param_res_names() const override; + virtual std::map get_kv_param_res_names() const; - virtual bool is_static() const override { return m_is_static; } + bool is_static() const { return m_is_static; } - virtual bool is_stateful() const override { return m_is_stateful; } + bool is_stateful() const { return m_is_stateful; } int get_static_n_tokens() const { return m_is_prefill ? m_prefill_chunk_size : 1; } - virtual bool is_splited_model() const override { return m_model_is_splitted; } + bool is_splited_model() const { return m_model_is_splitted; } + // A VIEW consumed by this graph whose storage root is neither a node nor a leaf of it (a scheduler + // split input); it is bound as its own model input rather than through its root. + bool is_external_view_input(const ggml_tensor * tensor) const; + static bool is_external_view(const ggml_cgraph * cgraph, const ggml_tensor * tensor); ov::PartialShape get_graph_input_shape(const ggml_tensor * op, const ggml_tensor * input, @@ -333,9 +314,27 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { const ggml_tensor * get_tensor_used_op(const ggml_tensor * tensor) const; + // True if the KV cache is a model input of shape [1, ctx, n_heads_kv, head_size] instead of + // [1, 1, ctx, n_heads_kv * head_size]. Used for stateless GPU graphs when the cache holds one sequence, one token + // per row, and is only written by SET_ROWS and read through views that split the rows into heads. The GPU + // plugin can then fuse the cache write and read into StatelessKV: the new rows are written in place and attention + // reads the cache without a copy. + bool is_kvcache_4d(const ggml_tensor * cache) const; + + // True if the KV cache is an input of this graph in the 4-D layout (see is_kvcache_4d) + bool is_kvcache_4d_input(const ggml_tensor * cache) const { return m_kvcache_4d.count(cache) != 0; } + const ggml_tensor * get_tensor_from_name(const std::string & name) const; - void clear_model_weights() { m_model_weights.clear(); } + // Drops the decoder's references to the (large) weight subgraphs after conversion. The + // pre-built nodes are also seeded into m_model_input_nodes for the frontend, so remove them + // from there too, otherwise the cached decoder would keep every weight alive. + void clear_model_weights() { + for (const auto & [weight_name, weight_node] : m_model_weights) { + m_model_input_nodes.erase(weight_name); + } + m_model_weights.clear(); + } static std::pair compute_llm_params(ggml_cgraph * cgraph, bool is_static); @@ -353,12 +352,13 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { bool m_naive = false; int m_prefill_chunk_size = 0; bool m_model_is_splitted = false; // label the cgraph is splited or not + // Keyed by tensor: inputs and outputs refer to the same cache under different names. + std::set m_transposed_kv_caches; static ov::Shape get_shape(const ggml_tensor * tensor); static std::vector get_stride(const ggml_tensor * tensor); static ov::element::Type get_ov_type(const ggml_tensor * tensor); static std::string compute_op_type(const ggml_tensor * node); - void add_extra_inputs(); void update_io(ggml_cgraph * cgraph); bool is_bound_to(const ggml_cgraph * cgraph) const; @@ -393,6 +393,11 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { op->src[0] != nullptr && op->src[0]->op != GGML_OP_NONE; } + // per-token embedding scale [1, n_tokens] (inp->scale_rows in llama-graph.cpp) + static bool is_inp_scale_rows(const ggml_tensor * tensor, const ggml_tensor * op) { + return op->op == GGML_OP_MUL && tensor == op->src[1] && strcmp(tensor->name, "inp_scale_rows") == 0; + } + static bool is_rope_freqs_weight(const ggml_tensor * tensor, const ggml_tensor * op) { return op->op == GGML_OP_ROPE && tensor == op->src[2]; } @@ -481,22 +486,33 @@ class GgmlOvDecoder : public ov::frontend::ggml::GgmlDecoder { // Infer and propagate dynamic-dimension indices for all tensors in the GGML graph. void compute_node_dynamic_dims(); + // "view_slice" ({ov_axis, start, length}) / "view_reshape" (OV-order target, -1 on the token axis) for + // VIEWs the frontend cannot recover from strides alone; empty when the generic stride path applies. + std::vector describe_view(const ggml_tensor * node, bool reshape) const; + // "kv_view" descriptor for a VIEW that reads a KV cache, with the runtime operands it needs; empty + // when the view is not a recognized KV-cache read. `transposed` reports a cells-innermost read even + // when no descriptor is produced. + std::pair, std::vector> describe_kv_view(const ggml_tensor * node, + bool & transposed) const; + void compute_kv_views(); void validate_cgraph() const; ggml_cgraph * m_cgraph = nullptr; std::map m_inputs; - std::map m_model_inputs; - std::map m_model_extra_inputs; + std::map m_model_inputs; + std::map> m_model_input_nodes; std::map> m_model_weights; std::map m_model_outputs; std::set m_model_output_names; std::vector m_node_info_list; std::map m_node_dynamic_dims; + std::set m_kvcache_4d; ModelParams m_model_params; ComputeParams m_compute_params; + int m_node_idx = -1; }; void print_tensor_address_map(const ggml_cgraph * cgraph); diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp index 0257e23dbf32..16325720d1dc 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.cpp @@ -129,8 +129,10 @@ void ggml_openvino_device_config::init() { "GGML_OPENVINO_ENABLE_CACHE", "GGML_OPENVINO_DISABLE_CACHE", "GGML_OPENVINO_DISABLE_KV_SLICE", + "GGML_OPENVINO_DISABLE_KV_4D", "GGML_OPENVINO_ENABLE_FALLBACK", "GGML_OPENVINO_MANUAL_GQA_ATTN", + "GGML_OPENVINO_DISABLE_ELTWISE_RANK_ALIGN", "GGML_OPENVINO_MOE_OP", "GGML_OPENVINO_MEMORY_OPTIMIZE", "GGML_OPENVINO_RELEASE_WEIGHTS", @@ -141,6 +143,7 @@ void ggml_openvino_device_config::init() { "GGML_OPENVINO_DISABLE_REMOTE_OUTPUTS", "GGML_OPENVINO_REQUANT_KQUANT", "GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT", + "GGML_OPENVINO_DISABLE_SHAPE_FROM_MASK", // Build the precise (but O(n_nodes)) graph cache key. Needed by op tests. "GGML_OPENVINO_FULL_GRAPH_KEY", }; @@ -321,6 +324,17 @@ bool ggml_openvino_is_gpu() { return has_prefix(ggml_openvino_get_device_name(), "GPU"); } +bool ggml_openvino_manual_gqa_attn_enabled() { + if (ggml_openvino_getenv_str("GGML_OPENVINO_MANUAL_GQA_ATTN") != nullptr) { + return ggml_openvino_getenv_int("GGML_OPENVINO_MANUAL_GQA_ATTN") > 0; + } + return ggml_openvino_is_gpu(); +} + +bool ggml_openvino_shape_from_mask_enabled() { + return ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_SHAPE_FROM_MASK") == 0; +} + size_t ggml_openvino_max_alloc_size() { return ggml_openvino_get_device_config().max_alloc_size; } diff --git a/ggml/src/ggml-openvino/ggml-openvino-extra.h b/ggml/src/ggml-openvino/ggml-openvino-extra.h index 3a58b2d02803..77947b9f54a3 100644 --- a/ggml/src/ggml-openvino/ggml-openvino-extra.h +++ b/ggml/src/ggml-openvino/ggml-openvino-extra.h @@ -120,6 +120,13 @@ bool ggml_openvino_is_npu(); // Check if running on a GPU (GPU, GPU.0, GPU.1, ...) bool ggml_openvino_is_gpu(); +// Explicit QK/softmax/V attention for grouped-query models instead of the fused SDPA op. Defaults to on for GPU; +// GGML_OPENVINO_MANUAL_GQA_ATTN=1/0 forces it on/off. Only stateless graphs use it. +bool ggml_openvino_manual_gqa_attn_enabled(); + +// Read n_seq_active and attention_size(_swa) from the KQ mask shape (off with GGML_OPENVINO_DISABLE_SHAPE_FROM_MASK=1) +bool ggml_openvino_shape_from_mask_enabled(); + // Largest single memory object the device can allocate, SIZE_MAX when there is no known limit size_t ggml_openvino_max_alloc_size(); diff --git a/ggml/src/ggml-openvino/ggml-openvino.cpp b/ggml/src/ggml-openvino/ggml-openvino.cpp index 87ee096fa547..98aed8a201b2 100644 --- a/ggml/src/ggml-openvino/ggml-openvino.cpp +++ b/ggml/src/ggml-openvino/ggml-openvino.cpp @@ -4,7 +4,6 @@ #include "ggml-backend.h" #include "ggml-impl.h" #include "ggml-openvino-extra.h" -#include "ggml-openvino/openvino/op_table.h" #include "ggml-openvino/utils.h" #include "ggml-quants.h" #include "ggml.h" @@ -31,6 +30,8 @@ #include #include #include +#include +#include #include #ifdef _WIN32 @@ -1163,18 +1164,6 @@ static ggml_backend_buffer_type_t ggml_backend_openvino_device_get_host_buffer_t return ggml_backend_openvino_host_buffer_type(ctx->device); } -static bool has_view_op_input(const ggml_tensor * op) { - for (int i = 0; i < GGML_MAX_SRC; i++) { - if (op->src[i] == nullptr) { - break; - } - if (op->src[i]->op == GGML_OP_VIEW) { - return true; - } - } - return false; -} - // OV slices whole elements per axis, so each stride must be a multiple of the next smaller one // (e.g. a batch stride of m*nb[1] + pad bytes cannot be expressed and would be read wrongly). static bool has_strides_on_element_grid(const ggml_tensor * t) { @@ -1192,552 +1181,41 @@ static bool has_strides_on_element_grid(const ggml_tensor * t) { } return true; } - -static bool has_non_contiguous_view_input(const ggml_tensor * op) { - for (int i = 0; i < GGML_MAX_SRC; i++) { - if (op->src[i] == nullptr) { - break; - } - if (op->src[i]->op == GGML_OP_VIEW && !ggml_is_contiguous(op->src[i])) { - return true; - } - } - return false; -} - -static bool is_supported_flash_attn_pattern(const ggml_tensor * op) { - // Each Q/K/V input must follow one of: - // PERMUTE -> VIEW -> base (view_src==nullptr) (llama KV-cache path) - // PERMUTE -> RESHAPE -> base (view_src==nullptr) (whisper Q) - // VIEW -> base (view_src==nullptr) (whisper K/V from kv_pad) - for (int i = 0; i < 3; i++) { - const ggml_tensor * src = op->src[i]; - if (src->op == GGML_OP_PERMUTE) { - if (src->src[0] == nullptr) { - return false; - } - if (src->src[0]->op != GGML_OP_VIEW && src->src[0]->op != GGML_OP_RESHAPE) { - return false; - } - if (src->src[0]->src[0] == nullptr || src->src[0]->src[0]->view_src != nullptr) { - return false; - } - } else if (src->op == GGML_OP_VIEW) { - if (src->src[0] == nullptr || src->src[0]->view_src != nullptr) { - return false; - } - } else if (src->op == GGML_OP_CPY) { - if (src->src[0] == nullptr || src->src[0]->op != GGML_OP_PERMUTE || src->src[0]->src[0] == nullptr) { - return false; - } - } else { - return false; - } - } - return true; -} - -static bool is_gemma3n_flash_attn_pattern(const ggml_tensor * op) { - if (!is_supported_flash_attn_pattern(op)) { - return false; - } - - const ggml_tensor * q_base = - op->src[0] != nullptr && op->src[0]->src[0] != nullptr ? op->src[0]->src[0]->src[0] : nullptr; - const ggml_tensor * k_base = - op->src[1] != nullptr && op->src[1]->src[0] != nullptr ? op->src[1]->src[0]->src[0] : nullptr; - const ggml_tensor * v_base = - op->src[2] != nullptr && op->src[2]->src[0] != nullptr ? op->src[2]->src[0]->src[0] : nullptr; - - if (q_base == nullptr || q_base->op != GGML_OP_ROPE) { - return false; - } - - // gemma3n direct attention path (no KV cache): q=ROPE, k=ROPE, v=RMS_NORM - // Only match this specific pattern to avoid falsely catching other models - // (e.g. Gemma4) that also use scale=1.0 with KV-cache backed attention. - const bool is_qkv_direct = - k_base != nullptr && v_base != nullptr && k_base->op == GGML_OP_ROPE && v_base->op == GGML_OP_RMS_NORM; - - return is_qkv_direct; -} - -static bool checked_mul_size(size_t a, size_t b, size_t & out) { - if (a == 0 || b == 0) { - out = 0; - return true; - } - if (a > SIZE_MAX / b) { - return false; - } - out = a * b; - return true; -} - -static bool tensor_view_fits_src_buffer(const ggml_tensor * tensor) { - if (tensor->view_src == nullptr) { - return true; - } - - const size_t src_nbytes = ggml_nbytes(tensor->view_src); - if (tensor->view_offs > src_nbytes) { - return false; - } - - const size_t tensor_nbytes = ggml_nbytes(tensor); - return tensor_nbytes <= src_nbytes - tensor->view_offs; -} - -static bool cpy_output_view_is_supported(const ggml_tensor * op) { - if (op->view_src == nullptr) { - return true; - } - - if (!tensor_view_fits_src_buffer(op)) { - return false; - } - - return ggml_nbytes(op) == 0 || ggml_is_contiguous(op) || GgmlOvDecoder::is_conv_state_writeback(op); -} - -static bool mul_mat_id_requires_large_tmp(const ggml_tensor * op) { - const ggml_tensor * as = op->src[0]; - const ggml_tensor * ids = op->src[2]; - if (as == nullptr || ids == nullptr) { - return true; - } - - // The MXFP4 MUL_MAT_ID translation (translate_mul_mat_id_mxfp4_packed in mul_mat_id.cpp) - // materializes selected expert weights with shape [n_tokens, n_used, rows, k]. Skip cases that - // would create a very large temporary and let the scheduler fall back instead. Every other weight - // type goes through GatherMatmul, which never materializes this temporary. - size_t tmp_elems = 1; - if (!checked_mul_size(tmp_elems, static_cast(ids->ne[1]), tmp_elems) || - !checked_mul_size(tmp_elems, static_cast(ids->ne[0]), tmp_elems) || - !checked_mul_size(tmp_elems, static_cast(as->ne[1]), tmp_elems) || - !checked_mul_size(tmp_elems, static_cast(as->ne[0]), tmp_elems)) { - return true; - } - - size_t tmp_bytes = 0; - if (!checked_mul_size(tmp_elems, sizeof(float), tmp_bytes)) { - return true; - } - - static constexpr size_t mul_mat_id_tmp_limit = 1ULL << 30; // 1 GiB - return tmp_bytes > mul_mat_id_tmp_limit; -} - -static bool tensor_name_starts_with(const ggml_tensor * tensor, const char * prefix) { - return tensor != nullptr && strncmp(tensor->name, prefix, strlen(prefix)) == 0; -} - -static bool is_msa_block_mask_expansion(const ggml_tensor * op) { - if (tensor_name_starts_with(op, "msa_")) { - return true; - } - - const ggml_tensor * src = op->src[0]; - while (src != nullptr && (src->op == GGML_OP_RESHAPE || src->op == GGML_OP_REPEAT)) { - if (tensor_name_starts_with(src, "msa_block_mask")) { - return true; - } - src = src->src[0]; - } - - return tensor_name_starts_with(src, "msa_block_mask"); -} - -namespace { struct ggml_openvino_op_support { bool is_supported = true; std::string reason; - - operator bool() const { - return is_supported; - } }; -} // namespace - -static ggml_openvino_op_support is_op_supported_case(const ggml_tensor * op) { - if (is_msa_block_mask_expansion(op)) { - return {false, "MSA block mask expansion is not supported"}; - } - - switch (op->op) { - case GGML_OP_CONCAT: { - if (op->type == GGML_TYPE_I64) { - return {false, "CONCAT with I64 type is not supported"}; - } - if (ggml_openvino_is_gpu() && op->type == GGML_TYPE_BF16 && has_view_op_input(op)) { - return {false, "CONCAT with BF16 type and VIEW input is not supported on GPU"}; - } - break; - } - case GGML_OP_SET: { - const auto nb1 = static_cast(op->op_params[0]); - const auto nb2 = static_cast(op->op_params[1]); - const auto nb3 = static_cast(op->op_params[2]); - - // OpenVINO SET translation currently supports dst layouts that match src0 strides. - if (op->src[0] == nullptr || nb1 != op->src[0]->nb[1] || nb2 != op->src[0]->nb[2] || nb3 != op->src[0]->nb[3]) { - return {false, "SET op with dst nb1=" + std::to_string(nb1) + ", nb2=" + std::to_string(nb2) + ", nb3=" + std::to_string(nb3) + - " that does not match src0 strides nb[1]=" + (op->src[0] != nullptr ? std::to_string(op->src[0]->nb[1]) : "null") + - ", nb[2]=" + (op->src[0] != nullptr ? std::to_string(op->src[0]->nb[2]) : "null") + - ", nb[3]=" + (op->src[0] != nullptr ? std::to_string(op->src[0]->nb[3]) : "null")}; - } - break; - } - case GGML_OP_GET_ROWS: - case GGML_OP_SET_ROWS: { - if (op->ne[3] != 1) { - return {false, "GET_ROWS/SET_ROWS with ne[3] != 1 (ne[3]=" + std::to_string(op->ne[3]) + ") is not supported"}; - } - if (op->op == GGML_OP_GET_ROWS && ggml_openvino_is_gpu() && - op->src[0]->type == GGML_TYPE_BF16) { - return {false, "GET_ROWS with BF16 src0 is not supported on GPU"}; - } - if (op->ne[0] == 256 && (op->src[0]->type == GGML_TYPE_Q4_K || op->src[0]->type == GGML_TYPE_Q5_K || - op->src[0]->type == GGML_TYPE_Q4_1 || op->src[0]->type == GGML_TYPE_Q5_1)) { - // These are all f16-arithmetic dequant rounding errors that intermittently exceed the - // tight 1e-7 NMSE threshold depending on the random test data (see ggml-quants.cpp - // make_int8_weights/make_int4_weights: dequant is done in f16, not f32, to keep the - // Convert/Subtract/Multiply chain fusable into GatherMatmulCompressed/FullyConnectedCompressed - // for the shared non-test code paths). - return {false, "GET_ROWS/SET_ROWS with ne[0] == 256 and type " + std::string(ggml_type_name(op->src[0]->type)) + - " rejected due to f16-arithmetic dequant rounding errors that intermittently exceed 1e-7 NMSE threshold"}; - } - break; - } - case GGML_OP_RESHAPE: { - if (strncmp(op->name, "ffn_norm_exps", sizeof("ffn_norm_exps") - 1) == 0) { - return {false, "RESHAPE for ffn_norm_exps is not supported"}; - } - break; - } - case GGML_OP_ADD: - case GGML_OP_MUL: - case GGML_OP_SUB: { - if (op->src[1]->op == GGML_OP_PERMUTE) { - return {false, "ADD/MUL/SUB with PERMUTE src1 is not supported"}; - } - if (op->src[0]->type != op->src[1]->type && - (op->src[0]->type == GGML_TYPE_BF16 || op->src[1]->type == GGML_TYPE_BF16)) { - return {false, "ADD/MUL/SUB with BF16 and a different src1 type is not supported"}; - } - // >8-expert MoE ReduceSum drifts past the 1e-7 tolerance (f32 order vs CPU); intermittent. - if (op->op == GGML_OP_ADD && is_moe_expert_sum_add(op) && op->src[1]->src[0]->ne[1] > 8) { - return {false, "MoE expert-plane sum with more than 8 experts is not supported"}; - } - for (int i = 0; i < 4; i++) { - if (op->src[0]->ne[i] != op->src[1]->ne[i] && (op->src[0]->ne[i] != 1 && op->src[1]->ne[i] != 1)) { - return {false, "ADD/MUL/SUB with incompatible broadcast shapes: src0->ne[" + std::to_string(i) + "]=" + - std::to_string(op->src[0]->ne[i]) + ", src1->ne[" + std::to_string(i) + "]=" + - std::to_string(op->src[1]->ne[i])}; - } - } - break; - } - case GGML_OP_SCALE: { - if (op->type == GGML_TYPE_BF16) { - return {false, "SCALE with BF16 type is not supported"}; - } - break; - } - case GGML_OP_ADD_ID: { - // Keep support aligned with the CPU backend implementation, which only handles f32 inputs/output and i32 ids. - if (op->type != GGML_TYPE_F32 || op->src[0]->type != GGML_TYPE_F32 || op->src[1]->type != GGML_TYPE_F32 || - op->src[2]->type != GGML_TYPE_I32) { - return {false, "ADD_ID only supports F32 inputs/output and I32 ids"}; - } - break; - } - case GGML_OP_DIV: { - // The GPU plugin can fuse broadcast DIV into the preceding FFN GEMM path - // and produce infs for per-channel scale vectors. Keep those DIVs on CPU - // until the fused GPU kernel is reliable. (falied case llama-arch-test mpt) - if (ggml_openvino_is_gpu() && op->src[1]->ne[0] == op->ne[0] && - op->src[1]->ne[1] == 1 && op->src[1]->ne[2] == 1 && op->src[1]->ne[3] == 1) { - return {false, "DIV per-channel scale broadcast is not supported on GPU"}; - } - break; - } - case GGML_OP_POOL_2D: { - if (ggml_openvino_is_gpu()) { - const int32_t * params = op->op_params; - const int k0 = params[1]; - const int k1 = params[2]; - const int p0 = params[5]; - const int p1 = params[6]; - if ((p0 > 0 || p1 > 0) && (k0 < 3 || k1 < 3)) { - return {false, "POOL_2D with padding and kernel size < 3 is not supported on " + ggml_openvino_get_device_name()}; - } - } - break; - } - case GGML_OP_SUM: { - if (op->src[0]->op == GGML_OP_PERMUTE) { - return {false, "SUM with PERMUTE input is not supported"}; - } - break; - } - case GGML_OP_MEAN: { - if (op->src[0]->op == GGML_OP_PERMUTE && op->src[0]->src[0] != nullptr && op->src[0]->src[0]->op == GGML_OP_VIEW) { - return {false, "MEAN with PERMUTE of VIEW input is not supported"}; - } - break; - } - case GGML_OP_SUM_ROWS: { - if (op->src[0]->op == GGML_OP_PERMUTE) { - return {false, "SUM_ROWS with PERMUTE input is not supported"}; - } - break; - } - case GGML_OP_FLASH_ATTN_EXT: { - float scale = 1.0f; - float max_bias = 0.0f; - float logit_softcap = 0.0f; - const auto * op_params = op->op_params; - memcpy(&scale, (const float *) op_params + 0, sizeof(float)); - memcpy(&max_bias, (const float *) op_params + 1, sizeof(float)); - memcpy(&logit_softcap, (const float *) op_params + 2, sizeof(float)); - - // Keep gemma3n flash-attn pattern on CPU for GPU runs to avoid - // accuracy drift in the OpenVINO path. Restrict by scale=1.0 to avoid - // affecting non-gemma3n models such as Llama-3.2. - if (fabsf(scale - 1.0f) < 1e-6f && is_gemma3n_flash_attn_pattern(op)) { - return {false, "FLASH_ATTN_EXT gemma3n pattern on GPU is not supported"}; - } - if (op->src[4] != nullptr) { - return {false, "FLASH_ATTN_EXT with sinks is not supported"}; - } - if (!is_supported_flash_attn_pattern(op)) { - return {false, "FLASH_ATTN_EXT unsupported attention pattern"}; - } - if (max_bias > 0) { - return {false, "FLASH_ATTN_EXT with max_bias > 0 (max_bias=" + std::to_string(max_bias) + ") is not supported"}; - } - if (logit_softcap != 0) { - return {false, "FLASH_ATTN_EXT with logit_softcap != 0 (logit_softcap=" + std::to_string(logit_softcap) + ") is not supported"}; - } - break; - } - case GGML_OP_PERMUTE: { - if (op->type == GGML_TYPE_BF16 && ggml_openvino_is_gpu()) { - return {false, "PERMUTE with BF16 type is not supported on GPU"}; - } - break; - } - case GGML_OP_CPY: { - if (op->src[0]->type != GGML_TYPE_BF16 && op->src[1]->type == GGML_TYPE_BF16) { - return {false, "CPY with BF16 src[1] type is not supported"}; - } - if (ggml_openvino_is_npu() && (op->src[0]->type == GGML_TYPE_BF16 || op->src[1]->type == GGML_TYPE_BF16)) { - return {false, "CPY with BF16 is not supported is not supported on NPU"}; - } - // CPY to a quantized destination (e.g. f32 -> q4_0) is numerically unstable with OpenVINO backend. - if (ggml_is_quantized(op->type)) { - return {false, "CPY to quantized destination (e.g. f32 -> q4_0) is numerically unstable"}; - } - if (ggml_nelements(op->src[0]) != ggml_nelements(op->src[1])) { - return {false, "CPY with mismatched element counts is not supported: src0=" + std::to_string(ggml_nelements(op->src[0])) + - " != src1=" + std::to_string(ggml_nelements(op->src[1]))}; - } - // op test case with non-contiguous src or dst - if ((op->ne[0] == 3 && op->ne[1] == 4 && op->ne[2] == 3 && op->ne[3] == 2) || - (op->ne[0] == 1 && op->ne[1] == 4 && op->ne[2] == 3 && op->ne[3] == 2) || - (op->ne[0] == 2 && op->ne[1] == 4 && op->ne[2] == 3 && op->ne[3] == 2)) { - return {false, "CPY with non-contiguous shape [" + std::to_string(op->ne[0]) + ", " + - std::to_string(op->ne[1]) + ", " + std::to_string(op->ne[2]) + ", " + - std::to_string(op->ne[3]) + "] is not supported"}; - } - if (!cpy_output_view_is_supported(op)) { - return {false, "CPY with non-contiguous output view is not supported"}; - } - break; - } - case GGML_OP_MUL_MAT: { - if (ggml_openvino_is_gpu() && op->src[0] != nullptr && op->src[1] != nullptr && - ggml_is_quantized(op->src[0]->type) && strcmp(op->src[0]->name, "a") == 0 && - strcmp(op->src[1]->name, "b") == 0 && op->src[0]->ne[1] == 1 && op->src[1]->ne[1] == 64 && - op->src[0]->ne[0] == 256 && op->src[1]->ne[0] == 256) { - return {false, "MUL_MAT quantized benchmark test case on GPU is not supported"}; - } - if (ggml_openvino_is_gpu() && op->type == GGML_TYPE_F32 && op->ne[0] == 1 && op->ne[1] == 1 && - (op->src[0]->buffer == nullptr || op->src[0]->buffer->usage != GGML_BACKEND_BUFFER_USAGE_WEIGHTS)) { - return {false, "MUL_MAT scalar dot product with non-weight src[0] on GPU is not supported"}; - } - if (op->src[0]->ne[3] != op->src[1]->ne[3] && op->src[0]->ne[3] != 1 && op->src[1]->ne[3] != 1) { - return {false, "MUL_MAT with incompatible broadcast on ne[3]: src0->ne[3]=" + std::to_string(op->src[0]->ne[3]) + - ", src1->ne[3]=" + std::to_string(op->src[1]->ne[3])}; - } - if (op->src[0]->op == GGML_OP_VIEW && op->src[1]->op == GGML_OP_VIEW) { - return {false, "MUL_MAT with both inputs as VIEW is not supported"}; - } - break; - } - case GGML_OP_MUL_MAT_ID: { - // Single-expert (or empty) MUL_MAT_ID is a degenerate shape that stresses GatherMatmul edge - // cases and never occurs in real MoE; let it fall back to CPU. - if (op->src[0] != nullptr && op->src[0]->ne[2] <= 1) { - return {false, "MUL_MAT_ID with single-expert or empty ne[2] <= 1 (ne[2]=" + - std::to_string(op->src[0]->ne[2]) + ") is not supported"}; - } - if (ggml_openvino_is_gpu() && op->src[0] != nullptr && !ggml_is_quantized(op->src[0]->type)) { - return {false, "MUL_MAT_ID with non-quantized weights on GPU is not supported"}; - } - // The GPU plugin's GatherMatmul returns wrong values for the layouts test-backend-ops - // produces: it builds a rank-4 input layout ([n_used, n_tokens, k, 1]) instead of rank 3 - // and the kernel misreads it, silently returning garbage (NMSE ~86) rather than asserting. - // The same graph is correct on the CPU plugin, and correct on GPU for every real model, - // which always feeds experts from a bound tensor buffer. Standalone op-test tensors have - // no buffer at all, so use that to exclude them and let the scheduler run them on CPU. - if (ggml_openvino_is_gpu() && op->src[0] != nullptr && op->src[0]->buffer == nullptr) { - return {false, "MUL_MAT_ID with unbound expert tensors on GPU is not supported"}; - } - // Only MXFP4 still needs the large-temporary guard; every other quantized type goes - // through GatherMatmul, which never materializes the selected expert weights. - if (ggml_openvino_is_gpu() && op->src[0] != nullptr && op->src[0]->type == GGML_TYPE_MXFP4 && - mul_mat_id_requires_large_tmp(op)) { - return {false, "MUL_MAT_ID with MXFP4 weights requires large temporary on GPU"}; - } - break; - } - case GGML_OP_ROPE: { - if (op->view_src != nullptr && !ggml_is_contiguous(op->src[0])) { - return {false, "ROPE on VIEW / non-contiguous input is not supported"}; - } - break; - } - case GGML_OP_TRANSPOSE: { - if (op->type == GGML_TYPE_BF16) { - return {false, "TRANSPOSE with BF16 type is not supported"}; - } - break; - } - case GGML_OP_REPEAT: { - if (ggml_openvino_is_gpu() && op->type == GGML_TYPE_BF16) { - return {false, "REPEAT with BF16 type is not supported on GPU"}; - } - break; - } - case GGML_OP_GATED_DELTA_NET: { - // enable after https://github.com/openvinotoolkit/openvino/pull/35917 is included in OV release - // return true; - // if (ggml_openvino_is_gpu() && op->src[0]->ne[2] > 1) { - // // CVS-186471 - // return true; - // } - if (op->src[2]->op == GGML_OP_PERMUTE) { - return {false, "GATED_DELTA_NET with PERMUTE src2 is not supported"}; - } - // kda (per-key-dimension gating) not supported by fused GatedDeltaNet op - if (op->src[3]->ne[0] != 1) { - return {false, "GATED_DELTA_NET with kda (per-key-dimension gating) is not supported"}; - } - // K > 1 (multiple state snapshots) not supported by fused op - if (((const int32_t *) op->op_params)[0] > 1) { - return {false, "GATED_DELTA_NET with K > 1 (multiple state snapshots) is not supported"}; - } - break; - } - case GGML_OP_SSM_CONV: { - // qwen3next is numerically unstable with OpenVINO SSM_CONV. - // Keep this op on CPU until the OpenVINO implementation is fixed. - // return true; - break; - } - case GGML_OP_VIEW: { - // Skip TOPK_MOE fused tests until it is fully supported. - // The argsort_top_k VIEW wrapping ARGSORT is named "selected_experts" in test_topk_moe. - if (strcmp(op->name, "selected_experts") == 0) { - return {false, "VIEW for selected_experts (argsort_top_k) is not supported"}; - } - break; - } - case GGML_OP_CONV_2D: - case GGML_OP_CONV_2D_DW: { - if (op->src[0]->ne[0] <= 0 || op->src[0]->ne[1] <= 0) { - return {false, "CONV_2D kernel size must be positive"}; - } - if (op->src[0]->op == GGML_OP_PERMUTE || op->src[1]->op == GGML_OP_PERMUTE) { - return {false, "CONV_2D with PERMUTE input is not supported"}; - } - if (has_non_contiguous_view_input(op)) { - return {false, "CONV_2D with non-contiguous view input is not supported"}; - } - const int32_t * params = op->op_params; - const int p0 = params[2]; - const int p1 = params[3]; - const int d0 = params[4]; - const int d1 = params[5]; - const int64_t dilated_kw = (int64_t) d0 * (op->src[0]->ne[0] - 1) + 1; - const int64_t dilated_kh = (int64_t) d1 * (op->src[0]->ne[1] - 1) + 1; - const int64_t padded_w = op->src[1]->ne[0] + 2 * p0; - const int64_t padded_h = op->src[1]->ne[1] + 2 * p1; - if (padded_w < dilated_kw || padded_h < dilated_kh) { - return {false, "CONV_2D padded input is smaller than kernel"}; - } - break; - } - case GGML_OP_CONV_3D: { - if (op->src[0]->ne[0] <= 0 || op->src[0]->ne[1] <= 0 || op->src[0]->ne[2] <= 0) { - return {false, "CONV_3D kernel size must be positive"}; - } - if (op->src[0]->op == GGML_OP_PERMUTE || op->src[1]->op == GGML_OP_PERMUTE) { - return {false, "CONV_3D with PERMUTE input is not supported"}; - } - if (has_non_contiguous_view_input(op)) { - return {false, "CONV_3D with non-contiguous view input is not supported"}; - } - const int32_t * params = op->op_params; - const int p0 = params[3]; - const int p1 = params[4]; - const int p2 = params[5]; - const int d0 = params[6]; - const int d1 = params[7]; - const int d2 = params[8]; - const int64_t dilated_kw = (int64_t) d0 * (op->src[0]->ne[0] - 1) + 1; - const int64_t dilated_kh = (int64_t) d1 * (op->src[0]->ne[1] - 1) + 1; - const int64_t dilated_kd = (int64_t) d2 * (op->src[0]->ne[2] - 1) + 1; - const int64_t padded_w = op->src[1]->ne[0] + 2 * p0; - const int64_t padded_h = op->src[1]->ne[1] + 2 * p1; - const int64_t padded_d = op->src[1]->ne[2] + 2 * p2; - if (padded_w < dilated_kw || padded_h < dilated_kh || padded_d < dilated_kd) { - return {false, "CONV_3D padded input is smaller than kernel"}; - } - break; - } - case GGML_OP_CONV_TRANSPOSE_1D: - case GGML_OP_CONV_TRANSPOSE_2D: { - if (op->src[0]->ne[0] <= 0 || op->src[0]->ne[1] <= 0) { - return {false, "CONV_TRANSPOSE kernel size must be positive"}; - } - if (op->src[0]->op == GGML_OP_PERMUTE || op->src[1]->op == GGML_OP_PERMUTE) { - return {false, "CONV_TRANSPOSE with PERMUTE input is not supported"}; - } - if (has_non_contiguous_view_input(op)) { - return {false, "CONV_TRANSPOSE with non-contiguous view input is not supported"}; - } - break; - } - case GGML_OP_IM2COL: { - if (op->src[0]->ne[0] <= 0 || op->src[0]->ne[1] <= 0) { - return {false, "IM2COL kernel size must be positive"}; - } - break; - } - case GGML_OP_IM2COL_3D: { - if (op->src[0]->ne[0] <= 0 || op->src[0]->ne[1] <= 0 || op->src[0]->ne[2] <= 0) { - return {false, "IM2COL_3D kernel size must be positive"}; - } - break; - } - default: - break; +// ggml operations the OpenVINO GGUF frontend translates. The frontend owns every translation; this list +// only answers the scheduler's supports_op queries, so keep it in step with the frontend's op table. +static ggml_openvino_op_support get_op_support_status(const ggml_tensor * op) { + static const std::unordered_set supported = { + // clang-format off + "GGML_OP_NONE", + "GGML_OP_ADD", "GGML_OP_ADD1", "GGML_OP_ADD_ID", "GGML_OP_ARGSORT", "GGML_OP_CLAMP", "GGML_OP_CONCAT", + "GGML_OP_CONT", "GGML_OP_CONV_2D", "GGML_OP_CONV_2D_DW", "GGML_OP_CONV_3D", "GGML_OP_CONV_TRANSPOSE_1D", + "GGML_OP_CONV_TRANSPOSE_2D", "GGML_OP_COS", "GGML_OP_CPY", "GGML_OP_CUMSUM", "GGML_OP_DIAG", + "GGML_OP_DIV", "GGML_OP_DUP", "GGML_OP_FILL", "GGML_OP_FLASH_ATTN_EXT", "GGML_OP_GATED_DELTA_NET", "GGML_OP_GET_REL_POS", + "GGML_OP_GET_ROWS", "GGML_OP_IM2COL", "GGML_OP_IM2COL_3D", "GGML_OP_L2_NORM", "GGML_OP_LOG", "GGML_OP_MEAN", "GGML_OP_MUL", "GGML_OP_MUL_MAT", + "GGML_OP_MUL_MAT_ID", "GGML_OP_NORM", "GGML_OP_PAD", "GGML_OP_PERMUTE", "GGML_OP_POOL_2D", "GGML_OP_REPEAT", + "GGML_OP_RESHAPE", "GGML_OP_RMS_NORM", "GGML_OP_ROLL", "GGML_OP_ROPE", "GGML_OP_SCALE", "GGML_OP_SET", + "GGML_OP_SET_ROWS", "GGML_OP_SIN", "GGML_OP_SOFT_MAX", "GGML_OP_SOLVE_TRI", "GGML_OP_SQR", "GGML_OP_SQRT", + "GGML_OP_SSM_CONV", "GGML_OP_SSM_SCAN", "GGML_OP_SUB", "GGML_OP_SUM", "GGML_OP_SUM_ROWS", "GGML_OP_TOP_K", + "GGML_OP_TRANSPOSE", "GGML_OP_TRI", "GGML_OP_UPSCALE", "GGML_OP_VIEW", "GGML_OP_WIN_PART", + "GGML_OP_WIN_UNPART", + "GGML_UNARY_OP_ELU", "GGML_UNARY_OP_EXP", "GGML_UNARY_OP_EXPM1", "GGML_UNARY_OP_GELU", "GGML_UNARY_OP_GELU_ERF", + "GGML_UNARY_OP_GELU_QUICK", "GGML_UNARY_OP_HARDSIGMOID", "GGML_UNARY_OP_NEG", "GGML_UNARY_OP_RELU", "GGML_UNARY_OP_ROUND", + "GGML_UNARY_OP_SIGMOID", "GGML_UNARY_OP_SILU", "GGML_UNARY_OP_SOFTPLUS", "GGML_UNARY_OP_STEP", + "GGML_UNARY_OP_TANH", + "GGML_GLU_OP_GEGLU", "GGML_GLU_OP_GEGLU_QUICK", "GGML_GLU_OP_SWIGLU", "GGML_GLU_OP_SWIGLU_CLAMP", + "GGML_GLU_OP_SWIGLU_OAI", + // clang-format on + }; + const std::string op_type = GgmlOvDecoder::compute_op_type(op); + if (supported.count(op_type) == 0) { + return {false, "op " + op_type + " is not supported"}; } - return {true, ""}; + return {}; } static ggml_openvino_op_support ggml_backend_openvino_device_supports_op_impl(ggml_backend_dev_t dev, const ggml_tensor * op) { @@ -1766,80 +1244,9 @@ static ggml_openvino_op_support ggml_backend_openvino_device_supports_op_impl(gg GGML_TYPE_Q4_1, GGML_TYPE_Q4_K, GGML_TYPE_Q5_1, GGML_TYPE_Q5_K, GGML_TYPE_Q8_0, GGML_TYPE_Q6_K, GGML_TYPE_MXFP4}; - // derive supported op sets from the op_table map, keys in - // the map use the full macro name (e.g. "GGML_OP_ADD"), while - // the ggml_*_op_name() helpers return only the trailing part (e.g. "ADD"). - // each set is built once and cached. - static const auto build_supported_sets = [] { - const auto & table = ov::frontend::ggml::get_supported_ops(); - std::unordered_set ops; - std::unordered_set unary_ops; - std::unordered_set glu_ops; - - // GGML_OP_NONE has no translator but is always safe to add to the supported set. - ops.insert(GGML_OP_NONE); - - for (int i = 0; i < GGML_OP_COUNT; ++i) { - const std::string key = std::string("GGML_OP_") + ggml_op_name(static_cast(i)); - if (table.count(key)) { - ops.insert(static_cast(i)); - } - } - for (int i = 0; i < GGML_UNARY_OP_COUNT; ++i) { - const std::string key = std::string("GGML_UNARY_OP_") + ggml_unary_op_name(static_cast(i)); - if (table.count(key)) { - unary_ops.insert(static_cast(i)); - } - } - for (int i = 0; i < GGML_GLU_OP_COUNT; ++i) { - const std::string key = std::string("GGML_GLU_OP_") + ggml_glu_op_name(static_cast(i)); - if (table.count(key)) { - glu_ops.insert(static_cast(i)); - } - } - return std::make_tuple(ops, unary_ops, glu_ops); - }; - static const auto supported_sets = build_supported_sets(); - static const auto & supported_ops = std::get<0>(supported_sets); - static const auto & supported_unary_ops = std::get<1>(supported_sets); - static const auto & supported_glu_ops = std::get<2>(supported_sets); - - switch (op->op) { - case GGML_OP_UNARY: { - auto supported = supported_unary_ops.find(ggml_get_unary_op(op)) != supported_unary_ops.end(); - if (!supported) { - return {false, "unary op " + std::string(ggml_unary_op_name(ggml_get_unary_op(op))) + " has no op translator"}; - } - if (op->type == GGML_TYPE_F32 && (ggml_get_unary_op(op) == GGML_UNARY_OP_EXP || - ggml_get_unary_op(op) == GGML_UNARY_OP_EXPM1)) { - return {false, "UNARY_EXP / UNARY_EXPM1 with F32 type is not supported"}; - } - break; - } - case GGML_OP_GLU: { - auto supported = supported_glu_ops.find(ggml_get_glu_op(op)) != supported_glu_ops.end(); - if (!supported) { - return {false, "GLU op " + std::string(ggml_glu_op_name(ggml_get_glu_op(op))) + " has no op translator"}; - } - // if (has_view_op_input(op)) { - // return {false, "GLU op " + std::string(ggml_glu_op_name(ggml_get_glu_op(op))) + " with view input is not supported"}; - // } - if (op->src[1] == nullptr && op->src[0]->ne[0] % 2 != 0) { - // triggers bug in ov gpu - return {false, "GLU op with odd src0 ne[0] and null src1 is not supported"}; - } - break; - } - default: { - auto supported = supported_ops.find(op->op) != supported_ops.end(); - if (!supported) { - return {false, "op " + std::string(ggml_op_name(op->op)) + " has no op translator"}; - } - static std::set ops_not_support_view_input{}; - if (ops_not_support_view_input.find(op->op) != ops_not_support_view_input.end() && has_view_op_input(op)) { - return {false, "op " + std::string(ggml_op_name(op->op)) + " with VIEW input is not supported"}; - } - } + auto op_status = get_op_support_status(op); + if (!op_status.is_supported) { + return op_status; } if (supported_types.find(op->type) == supported_types.end()) { @@ -1863,10 +1270,6 @@ static ggml_openvino_op_support ggml_backend_openvino_device_supports_op_impl(gg } } - auto op_support_case = is_op_supported_case(op); - if (!op_support_case.is_supported) { - return op_support_case; - } return {true, ""}; } diff --git a/ggml/src/ggml-openvino/kv-state-seq-axis.cpp b/ggml/src/ggml-openvino/kv-state-seq-axis.cpp new file mode 100644 index 000000000000..4bdc014c8f44 --- /dev/null +++ b/ggml/src/ggml-openvino/kv-state-seq-axis.cpp @@ -0,0 +1,145 @@ +#include "kv-state-seq-axis.h" + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +namespace ggml_openvino { + +namespace { + +const std::vector & seq_axis_perm() { + // [b, seq, n_heads_kv, head_size] <-> [b, n_heads_kv, seq, head_size] + static const std::vector perm{0, 2, 1, 3}; + return perm; +} + +// True when the state still has the frontend's stateful KV layout, so the sequence axis can be moved: +// rank 4, both head dims static, and the sequence at dim 1 and dynamic. The batch dim may be dynamic. +// Any KV head count is fine. With a single head the rewrite is pure metadata ([1, seq, 1, head] and +// [1, 1, seq, head] are the same memory); with several heads it also drops the reader-side transpose +// of the whole accumulated state, which is where most of the gain comes from at depth. +bool can_move_seq_axis(const ov::PartialShape & shape) { + return shape.rank().is_static() && shape.rank().get_length() == 4 && shape[1].is_dynamic() && + shape[2].is_static() && shape[3].is_static(); +} + +struct KVAppend { + std::shared_ptr concat; + std::shared_ptr read_value; + std::shared_ptr beam_gather; // reorders the past by beam; absent in other builders +}; + +// Assign(Concat([Gather(]ReadValue[, beam_idx, 0)], new_rows), axis 1) on one variable. +bool match_kv_append(const std::shared_ptr & assign, KVAppend & match) { + auto concat = ov::as_type_ptr(assign->get_input_node_shared_ptr(0)); + if (!concat || concat->get_input_size() != 2 || concat->get_axis() != 1) { + return false; + } + auto past = concat->get_input_node_shared_ptr(0); + auto gather = ov::as_type_ptr(past); + auto read_value = ov::as_type_ptr(gather ? gather->get_input_node_shared_ptr(0) : past); + if (!read_value || read_value->get_variable() != assign->get_variable() || + !can_move_seq_axis(read_value->get_output_partial_shape(0))) { + return false; + } + match = {concat, read_value, gather}; + return true; +} + +} // namespace + +bool KVStateSeqAxis::run_on_model(const std::shared_ptr & model) { + std::vector> assigns; + for (const auto & op : model->get_ops()) { + if (auto assign = ov::as_type_ptr(op)) { + assigns.push_back(assign); + } + } + + bool changed = false; + for (const auto & assign : assigns) { + KVAppend kv; + if (!match_kv_append(assign, kv)) { + continue; + } + const auto & concat = kv.concat; + const auto & read_value = kv.read_value; + const ov::Output past = kv.beam_gather ? kv.beam_gather->output(0) : read_value->output(0); + + // Consumers of the past other than the append (reads that must still observe the previous cache). + std::vector> past_readers; + for (const auto & reader : past.get_target_inputs()) { + if (reader.get_node() != concat.get()) { + past_readers.push_back(reader); + } + } + auto concat_readers = concat->output(0).get_target_inputs(); + + auto variable = read_value->get_variable(); + auto info = variable->get_info(); + const auto & shape = info.data_shape; + info.data_shape = ov::PartialShape{shape[0], shape[2], shape[1], shape[3]}; + variable->update(info); + + // The empty initial state carries the old layout too. + auto init = ov::as_type_ptr(read_value->get_input_node_shared_ptr(0)); + if (!init) { + return changed; + } + const auto & init_shape = init->get_shape(); + read_value->set_argument( + 0, + ov::op::v0::Constant::create( + init->get_element_type(), ov::Shape{init_shape[0], init_shape[2], init_shape[1], init_shape[3]}, std::vector{})); + read_value->validate_and_infer_types(); + if (kv.beam_gather) { + kv.beam_gather->validate_and_infer_types(); + } + + auto new_rows = concat->input_value(1); + auto perm_in = ov::op::v0::Constant::create(ov::element::i64, {4}, seq_axis_perm()); + concat->set_argument(1, std::make_shared(new_rows, perm_in)); + concat->set_axis(2); + concat->validate_and_infer_types(); + + // Other readers of the past still expect the sequence at dim 1. + for (auto & reader : past_readers) { + auto perm_out = ov::op::v0::Constant::create(ov::element::i64, {4}, seq_axis_perm()); + reader.replace_source_output(std::make_shared(past, perm_out)); + } + + // Readers of the grown cache still expect seq at dim 1. A reader that is itself the inverse + // Transpose wanted seq at dim 2 all along, so drop it; give anything else the inverse + // Transpose so its input is unchanged. + for (const auto & reader : concat_readers) { + auto * node = reader.get_node(); + if (ov::is_type(node)) { + continue; + } + bool dropped = false; + if (auto * transpose = ov::as_type(node)) { + auto order = ov::as_type_ptr(transpose->get_input_node_shared_ptr(1)); + if (order && order->cast_vector() == seq_axis_perm()) { + ov::replace_output_update_name(transpose->output(0), concat->output(0)); + dropped = true; + } + } + if (!dropped) { + auto perm_out = ov::op::v0::Constant::create(ov::element::i64, {4}, seq_axis_perm()); + reader.replace_source_output(std::make_shared(concat->output(0), perm_out)); + } + } + changed = true; + } + + return changed; +} + +} // namespace ggml_openvino diff --git a/ggml/src/ggml-openvino/openvino/pass/kv_state_seq_axis.h b/ggml/src/ggml-openvino/kv-state-seq-axis.h similarity index 74% rename from ggml/src/ggml-openvino/openvino/pass/kv_state_seq_axis.h rename to ggml/src/ggml-openvino/kv-state-seq-axis.h index 579022c45c59..79c3deac88b2 100644 --- a/ggml/src/ggml-openvino/openvino/pass/kv_state_seq_axis.h +++ b/ggml/src/ggml-openvino/kv-state-seq-axis.h @@ -1,9 +1,8 @@ +#pragma once + #include "openvino/pass/pass.hpp" -namespace ov { -namespace frontend { -namespace ggml { -namespace pass { +namespace ggml_openvino { // Moves the sequence axis of the stateful KV cache from dim 1 to dim 2, i.e. from // [1, seq, n_heads_kv, head_size] to [1, n_heads_kv, seq, head_size], and updates the @@ -11,14 +10,13 @@ namespace pass { // when the growing axis is a spatial axis, and the reader no longer has to transpose the // whole accumulated state every token (that cost grows with context length, so it is the // larger win at depth for a model with several KV heads). Only rewrites states that still -// match the frontend layout, so it no-ops if that layout ever changes. +// match the frontend layout, so it no-ops if that layout ever changes. Must run after +// the pass that creates the ReadValue/Assign pairs; disabled by +// GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT. class KVStateSeqAxis : public ov::pass::ModelPass { public: - OPENVINO_MODEL_PASS_RTTI("ov::frontend::ggml::pass::KVStateSeqAxis") + OPENVINO_MODEL_PASS_RTTI("ggml_openvino::KVStateSeqAxis") bool run_on_model(const std::shared_ptr & model) override; }; -} // namespace pass -} // namespace ggml -} // namespace frontend -} // namespace ov +} // namespace ggml_openvino diff --git a/ggml/src/ggml-openvino/model-cache.cpp b/ggml/src/ggml-openvino/model-cache.cpp index c8c5fb5adb8e..278f62a25a52 100644 --- a/ggml/src/ggml-openvino/model-cache.cpp +++ b/ggml/src/ggml-openvino/model-cache.cpp @@ -175,6 +175,7 @@ static const char * cache_settings[] = { "GGML_OPENVINO_REDUCE_COMPILE_MEM", "GGML_OPENVINO_MEMORY_OPTIMIZE", "GGML_OPENVINO_PROFILING", + "GGML_OPENVINO_DISABLE_SHAPE_FROM_MASK", }; void ggml_openvino_model_cache_init() { diff --git a/ggml/src/ggml-openvino/openvino/decoder.h b/ggml/src/ggml-openvino/openvino/decoder.h deleted file mode 100644 index ec6975282a54..000000000000 --- a/ggml/src/ggml-openvino/openvino/decoder.h +++ /dev/null @@ -1,133 +0,0 @@ -#pragma once - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { - -struct ModelInputInfo { - element::Type type; - PartialShape shape; -}; - -struct ModelExtraInputInfo { - element::Type type; - Shape shape; - int64_t value; - bool is_parameter; -}; - -class GgmlDecoder : public DecoderBase { -public: - virtual ov::Any get_attribute(const std::string & name) const = 0; - - virtual PartialShape get_input_shape(int node_idx, const std::string & name) const = 0; - - virtual std::vector get_input_stride(int node_idx, const std::string & name) const = 0; - - virtual size_t get_view_input_size(int node_idx, const std::string & name) const = 0; - - virtual size_t get_view_input_offset(int node_idx, const std::string & name, size_t view_index) const = 0; - - virtual size_t get_view_input_src_offset(int node_idx, const std::string & name, size_t view_index) const = 0; - - virtual std::vector get_view_input_stride(int node_idx, - const std::string & name, - size_t view_index) const = 0; - - virtual std::vector get_view_input_src_stride(int node_idx, - const std::string & name, - size_t view_index) const = 0; - - virtual Shape get_view_input_ggml_shape(int node_idx, const std::string & name, size_t view_index) const = 0; - - virtual Shape get_view_input_src_ggml_shape(int node_idx, const std::string & name, size_t view_index) const = 0; - - virtual PartialShape get_view_input_ov_shape(int node_idx, const std::string & name, size_t view_index) const = 0; - - virtual PartialShape get_view_input_src_ov_shape(int node_idx, - const std::string & name, - size_t view_index) const = 0; - - virtual std::string get_view_input_name(int node_idx, const std::string & name, size_t view_index) const = 0; - - virtual std::string get_view_input_src_name(int node_idx, const std::string & name, size_t view_index) const = 0; - - virtual element::Type get_input_type(int node_idx, const std::string & name) const = 0; - - virtual size_t get_input_size() const = 0; - - virtual size_t get_input_size(int node_idx) const = 0; - - virtual void get_input_node(size_t input_port_idx, - std::string & producer_name, - std::string & producer_output_port_name, - size_t & producer_output_port_index) const = 0; - - virtual std::vector get_input_names(int node_idx) const = 0; - - virtual PartialShape get_output_shape(int node_idx) const = 0; - - virtual element::Type get_output_type(const int node_idx) const = 0; - - virtual std::vector get_output_stride(int node_idx) const = 0; - - virtual int32_t * get_input_op_params(int node_idx, const std::string & name) const = 0; - - virtual int32_t * get_output_op_params(int node_idx) const = 0; - - virtual size_t get_output_op_offset(int node_idx) const = 0; - - virtual std::vector get_output_names(int node_idx) const = 0; - - virtual std::string get_inplace_op_src(int node_idx) const = 0; - - virtual bool is_view_like_alias_of(int node_idx, const std::string & view_src_name) const = 0; - - virtual const std::string & get_op_type() const = 0; - - virtual const std::string & get_op_type(int node_idx) const = 0; - - virtual const std::string & get_op_name() const = 0; - - virtual const std::string & get_op_name(int node_idx) const = 0; - - virtual void visit_subgraph(std::function, int node_idx)> node_visitor) const = 0; - - virtual int get_op_case(int node_idx) const = 0; - - virtual const std::map & get_model_inputs() const = 0; - virtual const std::map & get_model_extra_inputs() const = 0; - virtual const std::map> & get_model_weights() const = 0; - virtual std::set get_model_output_names() const = 0; - - virtual int32_t * get_rope_params() const = 0; - - virtual bool has_mixed_rope_params() const = 0; - - virtual int get_ssm_state_size() const = 0; - - virtual std::map get_kv_param_res_names() const = 0; - - virtual bool is_static() const = 0; - - virtual bool is_stateful() const = 0; - - virtual bool is_splited_model() const = 0; - - virtual int is_swa_layer(int layer) const = 0; - - virtual int32_t get_op_dynamic_dim(int node_idx) const = 0; -}; - -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/frontend.cpp b/ggml/src/ggml-openvino/openvino/frontend.cpp deleted file mode 100644 index 88de86feacae..000000000000 --- a/ggml/src/ggml-openvino/openvino/frontend.cpp +++ /dev/null @@ -1,28 +0,0 @@ -#include "frontend.h" - -#include "input_model.h" -#include "op_table.h" -#include "translate_session.h" -#include - -namespace ov { -namespace frontend { -namespace ggml { - -FrontEnd::FrontEnd() {} - -std::shared_ptr FrontEnd::convert(const InputModel::Ptr & model, bool naive) { - auto ggml_model = ov::as_type_ptr(model); - FRONT_END_GENERAL_CHECK(ggml_model, "Invalid input model"); - std::shared_ptr converted_model; - const auto & supported_ops = get_supported_ops(); - { - TranslateSession translate_session(model, supported_ops, naive); - converted_model = translate_session.get_converted_model(); - } - return converted_model; -} - -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/frontend.h b/ggml/src/ggml-openvino/openvino/frontend.h deleted file mode 100644 index 4e301d32e07a..000000000000 --- a/ggml/src/ggml-openvino/openvino/frontend.h +++ /dev/null @@ -1,22 +0,0 @@ -// Copyright (C) 2018-2024 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include - -namespace ov { -namespace frontend { -namespace ggml { - -class FrontEnd { -public: - FrontEnd(); - - static std::shared_ptr convert(const InputModel::Ptr & model, bool naive = false); -}; - -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/input_model.cpp b/ggml/src/ggml-openvino/openvino/input_model.cpp deleted file mode 100644 index 39b004c93176..000000000000 --- a/ggml/src/ggml-openvino/openvino/input_model.cpp +++ /dev/null @@ -1,17 +0,0 @@ -#include "input_model.h" - -#include "decoder.h" - -namespace ov { -namespace frontend { -namespace ggml { - -InputModel::InputModel(const std::shared_ptr & gdecoder) : m_decoder(gdecoder) {} - -const std::shared_ptr & InputModel::get_model_decoder() const { - return m_decoder; -} - -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/input_model.h b/ggml/src/ggml-openvino/openvino/input_model.h deleted file mode 100644 index 6ddcea996f03..000000000000 --- a/ggml/src/ggml-openvino/openvino/input_model.h +++ /dev/null @@ -1,29 +0,0 @@ -#pragma once - -#include "decoder.h" - -#include - -namespace ov { -namespace frontend { -namespace ggml { - -class FrontEnd; -class GgmlDecoder; -using ov::frontend::ggml::GgmlDecoder; - -class InputModel : public ov::frontend::InputModel { - friend class ::ov::frontend::ggml::FrontEnd; - -public: - explicit InputModel(const std::shared_ptr & gdecoder); - - const std::shared_ptr & get_model_decoder() const; - -private: - std::shared_ptr m_decoder; -}; - -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/node_context.h b/ggml/src/ggml-openvino/openvino/node_context.h deleted file mode 100644 index c653bc0e6947..000000000000 --- a/ggml/src/ggml-openvino/openvino/node_context.h +++ /dev/null @@ -1,180 +0,0 @@ -#pragma once - -#include "decoder.h" - -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { - -class TranslateSession; - -typedef std::map> TensorMap; - -class NodeContext : public frontend::NodeContext { -public: - NodeContext(const std::shared_ptr & decoder, - std::shared_ptr & tensor_map, - int node_idx, - TranslateSession * translate_session = nullptr) : - ov::frontend::NodeContext(decoder->get_op_type(node_idx)), - m_decoder(decoder), - m_tensor_map(tensor_map), - m_node_idx(node_idx), - m_translate_session(translate_session) { - m_input_names = decoder->get_input_names(m_node_idx); - m_output_names = decoder->get_output_names(m_node_idx); - } - - TranslateSession * get_translate_session() const { return m_translate_session; } - - const std::vector & get_input_names() const { return m_input_names; } - - const std::vector & get_output_names() const { return m_output_names; } - - size_t get_input_size() const override { return m_decoder->get_input_size(m_node_idx); } - - ov::element::Type get_input_type(size_t index) const { - return m_decoder->get_input_type(m_node_idx, m_input_names[index]); - } - - PartialShape get_input_shape(size_t input_index) const { - return m_decoder->get_input_shape(m_node_idx, m_input_names[input_index]); - } - - std::vector get_input_stride(size_t index) const { - return m_decoder->get_input_stride(m_node_idx, m_input_names[index]); - } - - std::string get_output_name() const { return m_output_names[0]; } - - PartialShape get_output_shape() const { return m_decoder->get_output_shape(m_node_idx); } - - int32_t * get_input_op_params(size_t index) const { - return m_decoder->get_input_op_params(m_node_idx, m_input_names[index]); - } - - size_t get_view_input_size(size_t index) const { - return m_decoder->get_view_input_size(m_node_idx, m_input_names[index]); - } - - size_t get_view_input_offset(size_t index, size_t view_index) const { - return m_decoder->get_view_input_offset(m_node_idx, m_input_names[index], view_index); - } - - size_t get_view_input_src_offset(size_t index, size_t view_index) const { - return m_decoder->get_view_input_src_offset(m_node_idx, m_input_names[index], view_index); - } - - std::vector get_view_input_stride(size_t index, size_t view_index) const { - return m_decoder->get_view_input_stride(m_node_idx, m_input_names[index], view_index); - } - - std::vector get_view_input_src_stride(size_t index, size_t view_index) const { - return m_decoder->get_view_input_src_stride(m_node_idx, m_input_names[index], view_index); - } - - ov::Shape get_view_input_ggml_shape(size_t index, size_t view_index) const { - return m_decoder->get_view_input_ggml_shape(m_node_idx, m_input_names[index], view_index); - } - - ov::Shape get_view_input_src_ggml_shape(size_t index, size_t view_index) const { - return m_decoder->get_view_input_src_ggml_shape(m_node_idx, m_input_names[index], view_index); - } - - ov::PartialShape get_view_input_ov_shape(size_t index, size_t view_index) const { - return m_decoder->get_view_input_ov_shape(m_node_idx, m_input_names[index], view_index); - } - - ov::PartialShape get_view_input_src_ov_shape(size_t index, size_t view_index) const { - return m_decoder->get_view_input_src_ov_shape(m_node_idx, m_input_names[index], view_index); - } - - std::string get_view_input_name(size_t index, size_t view_index) const { - return m_decoder->get_view_input_name(m_node_idx, m_input_names[index], view_index); - } - - std::string get_view_input_src_name(size_t index, size_t view_index) const { - return m_decoder->get_view_input_src_name(m_node_idx, m_input_names[index], view_index); - } - - int32_t get_op_dynamic_dim() const { return m_decoder->get_op_dynamic_dim(m_node_idx); } - - int32_t * get_output_op_params() const { return m_decoder->get_output_op_params(m_node_idx); } - - size_t get_output_op_offset() const { return m_decoder->get_output_op_offset(m_node_idx); } - - ov::element::Type get_output_type() const { return m_decoder->get_output_type(m_node_idx); } - - std::vector get_output_stride() const { return m_decoder->get_output_stride(m_node_idx); } - - Output get_input(int idx) const override { - // Check if this input is a VIEW - size_t view_input_size = m_decoder->get_view_input_size(m_node_idx, m_input_names[idx]); - if (view_input_size > 0) { - // This is a VIEW input, get the base tensor name (last element in the chain) - std::string base_name = - m_decoder->get_view_input_src_name(m_node_idx, m_input_names[idx], view_input_size - 1); - // Check if the VIEW has been resolved (translate_view produced a Slice) - auto view_it = m_tensor_map->find(m_input_names[idx]); - if (!base_name.empty() && view_it != m_tensor_map->end()) { - auto base_it = m_tensor_map->find(base_name); - // A multi-output translator can publish a VIEW directly without materializing - // its packed parent (GatedDeltaNet attention/state). In that case the VIEW is the - // authoritative value. The node comparison retains the existing resolved-VIEW path. - if (base_it == m_tensor_map->end() || - view_it->second.get_node_shared_ptr() != base_it->second.get_node_shared_ptr()) { - return view_it->second; - } - return base_it->second; - } - if (!base_name.empty()) { - return m_tensor_map->at(base_name); - } - } - // Not a VIEW or failed to get base name, use the original logic - return m_tensor_map->at(m_input_names[idx]); - } - - Output get_input(const std::string & name) const override { - if (m_tensor_map->find(name) == m_tensor_map->end()) { - throw std::runtime_error("'" + name + "' not found in tensor map."); - } - return m_tensor_map->at(name); - } - - bool has_input(const std::string & name) const { return m_tensor_map->find(name) != m_tensor_map->end(); } - - void put_shared(const std::string & name, const Output & value) const { - m_tensor_map->insert({name, value}); - } - - const std::string & get_name() const override { return m_decoder->get_op_name(m_node_idx); } - - ov::Any get_attribute_as_any(const std::string & name) const override { return m_decoder->get_attribute(name); } - - int get_op_case() const { return m_decoder->get_op_case(m_node_idx); } - - bool is_static() const { return m_decoder->is_static(); } - - bool is_stateful() const { return m_decoder->is_stateful(); } - - int get_ssm_state_size() const { return m_decoder->get_ssm_state_size(); } - -private: - std::shared_ptr m_decoder; - std::shared_ptr & m_tensor_map; - int m_node_idx; - TranslateSession * m_translate_session; - std::vector m_input_names; - std::vector m_output_names; -}; - -using CreatorFunction = std::function; - -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/add.cpp b/ggml/src/ggml-openvino/openvino/op/add.cpp deleted file mode 100644 index 84bc85618a3b..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/add.cpp +++ /dev/null @@ -1,67 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -OutputVector translate_add(const NodeContext & context) { - num_inputs_check(context, 2, 2); - - if (context.get_op_case() == 1) { - // MoE expert-plane sum (see is_moe_expert_sum_add): input 1 is a VIEW plane of the - // shared base tensor `experts` = [n_embd, n_expert_used, n_tokens, 1] (ggml order) -> - // [1, n_tokens, n_expert_used, n_embd] (OV order). The whole ADD chain is equivalent to - // reducing the expert axis (OV axis 2) of that base, so bypass the chain and the - // per-plane Slices entirely. - size_t view_size = context.get_view_input_size(1); - auto base_name = context.get_view_input_src_name(1, view_size - 1); - auto base = context.get_input(base_name); - - // Stateful models drop the leading batch dim, so the base is rank 3 and both axes - // below shift down by one. Take them from the actual rank: the expert axis is always - // second from last, and the token axis is re-added just before it. - const auto base_rank = base.get_partial_shape().rank(); - FRONT_END_OP_CONVERSION_CHECK(base_rank.is_static() && base_rank.get_length() >= 3, - "MoE expert sum needs a static rank of at least 3"); - const int64_t rank = base_rank.get_length(); - - auto reduced = std::make_shared( - base, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {rank - 2}), false); - auto res = std::make_shared( - reduced, ov::op::v0::Constant::create(ov::element::i64, {1}, {rank - 3})); - return rename_outputs_with_suffix({res}, context.get_name()); - } - - auto input_0 = process_view_input_new(context, 0); - auto input_1 = process_view_input_new(context, 1); - // opset1::Add needs matching types (e.g. fused ADD_ADD mixes f16/f32); add in f32, cast once. - auto output_type = context.get_output_type(); - if (input_0.get_element_type() != input_1.get_element_type()) { - if (input_0.get_element_type() != ov::element::f32) { - input_0 = std::make_shared(input_0, ov::element::f32); - } - if (input_1.get_element_type() != ov::element::f32) { - input_1 = std::make_shared(input_1, ov::element::f32); - } - } - ov::Output res = std::make_shared(input_0, input_1); - if (res.get_element_type() != output_type) { - res = std::make_shared(res, output_type); - } - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/add_id.cpp b/ggml/src/ggml-openvino/openvino/op/add_id.cpp deleted file mode 100644 index 79bdbe87731e..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/add_id.cpp +++ /dev/null @@ -1,76 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -static ov::Output reshape_add_id_input_to_2d(const ov::Output & input, - const ov::PartialShape & input_shape, - const std::vector & dims) { - const auto & actual_shape = input.get_partial_shape(); - if (actual_shape.rank().is_static() && actual_shape.rank().get_length() == 2) { - return input; - } - - if (input_shape.rank().is_static() && input_shape.rank().get_length() == 2) { - return input; - } - - auto shape = std::make_shared(input, ov::element::i64); - return std::make_shared(input, get_dimensions(shape, dims), false); -} - -OutputVector translate_add_id(const NodeContext & context) { - num_inputs_check(context, 3, 3); - - auto input = process_view_input_new(context, 0); - auto bias = process_view_input_new(context, 1); - auto ids = process_view_input_new(context, 2); - - // OpenVINO uses reversed GGML dimensions: - // input: [1, n_token, n_used, n_embd] - // bias: [1, 1, n_expert, n_embd] - // ids: [1, 1, n_token, n_used] - // Model bias constants may already be stored as [n_expert, n_embd]. - bias = reshape_add_id_input_to_2d(bias, context.get_input_shape(1), {2, 3}); - ids = reshape_add_id_input_to_2d(ids, context.get_input_shape(2), {2, 3}); - - if (ids.get_element_type() != ov::element::i32 && ids.get_element_type() != ov::element::i64) { - ids = std::make_shared(ids, ov::element::i32); - } - - auto gather_axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {0}); - ov::Output selected_bias = std::make_shared(bias, ids, gather_axis); - selected_bias = std::make_shared( - selected_bias, std::make_shared(input, ov::element::i64), false); - - if (selected_bias.get_element_type() != input.get_element_type()) { - selected_bias = std::make_shared(selected_bias, input.get_element_type()); - } - - ov::Output res = std::make_shared(input, selected_bias); - const auto output_type = context.get_output_type(); - if (res.get_element_type() != output_type) { - res = std::make_shared(res, output_type); - } - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/argsort.cpp b/ggml/src/ggml-openvino/openvino/op/argsort.cpp deleted file mode 100644 index dd0872516758..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/argsort.cpp +++ /dev/null @@ -1,51 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" -#include "ggml.h" - -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -OutputVector translate_argsort(const NodeContext & context) { - num_inputs_check(context, 1, 1); - - auto input = process_view_input_new(context, 0); - - const int32_t order = context.get_output_op_params()[0]; - - ov::op::v11::TopK::Mode mode; - switch (order) { - case GGML_SORT_ORDER_ASC: - mode = ov::op::v11::TopK::Mode::MIN; - break; - case GGML_SORT_ORDER_DESC: - mode = ov::op::v11::TopK::Mode::MAX; - break; - default: - FRONT_END_OP_CONVERSION_CHECK(false, "Unsupported GGML_OP_ARGSORT order: ", order); - } - - // Stateful models drop the leading size-1 batch dim, so the expert axis is 2 there - // instead of 3 (same rank-3-vs-rank-4 split as get_rows.cpp / process_view_input). - const int axis = (context.is_stateful() && input.get_partial_shape().rank() == 3) ? 2 : 3; - - auto k = std::make_shared(get_dimensions(input.get_node_shared_ptr(), {axis}), - ov::op::v0::Constant::create(ov::element::i64, {1}, {0})); - - auto topk = std::make_shared(input, k, axis, mode, ov::op::v11::TopK::SortType::SORT_VALUES, - context.get_output_type(), false); - - return rename_outputs_with_suffix({topk->output(1)}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/clamp.cpp b/ggml/src/ggml-openvino/openvino/op/clamp.cpp deleted file mode 100644 index 070ad33b7794..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/clamp.cpp +++ /dev/null @@ -1,33 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -OutputVector translate_clamp(const NodeContext & context) { - num_inputs_check(context, 1, 1); - - auto input = process_view_input_new(context, 0); - - const int32_t * op_params = context.get_output_op_params(); - FRONT_END_CHECK_IMPLEMENTED(op_params != nullptr, "CLAMP requires output op params"); - - float min; - float max; - std::memcpy(&min, reinterpret_cast(op_params) + 0, sizeof(float)); - std::memcpy(&max, reinterpret_cast(op_params) + 1, sizeof(float)); - - auto res = std::make_shared(input, min, max); - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/concat.cpp b/ggml/src/ggml-openvino/openvino/op/concat.cpp deleted file mode 100644 index 4d36a666b5e5..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/concat.cpp +++ /dev/null @@ -1,48 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -OutputVector translate_concat(const NodeContext & context) { - num_inputs_check(context, 2, 2); - - const int32_t * op_params = context.get_output_op_params(); - FRONT_END_CHECK_IMPLEMENTED(op_params != nullptr, "CONCAT requires output op params"); - - const auto output_shape = context.get_output_shape(); - FRONT_END_CHECK_IMPLEMENTED(output_shape.rank().is_static(), "CONCAT requires static output rank"); - - const auto rank = output_shape.rank().get_length(); - const int32_t ggml_dim = op_params[0]; - FRONT_END_CHECK_IMPLEMENTED(ggml_dim >= 0 && ggml_dim < rank, "CONCAT axis is out of range"); - - auto input_0 = process_view_input_new(context, 0); - auto input_1 = process_view_input_new(context, 1); - const auto output_type = context.get_output_type(); - - if (input_0.get_element_type() != output_type) { - input_0 = std::make_shared(input_0, output_type); - } - if (input_1.get_element_type() != output_type) { - input_1 = std::make_shared(input_1, output_type); - } - - const auto axis = static_cast(rank - 1 - ggml_dim); - auto res = std::make_shared(OutputVector{input_0, input_1}, axis); - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/cont.cpp b/ggml/src/ggml-openvino/openvino/op/cont.cpp deleted file mode 100644 index 9888f6b93fd3..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/cont.cpp +++ /dev/null @@ -1,37 +0,0 @@ - -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -OutputVector translate_cont(const NodeContext & context) { - num_inputs_check(context, 1, 1); - - auto src_shape = context.get_input_shape(0).to_shape(); - auto dst_shape = context.get_output_shape().to_shape(); - - if (context.get_op_dynamic_dim() != -1) { - dst_shape[3 - context.get_op_dynamic_dim()] = -1; - } - - auto input = process_view_input_new(context, 0); - - ov::Output res; - res = std::make_shared( - input, ov::op::v0::Constant::create(ov::element::i64, {dst_shape.size()}, dst_shape), false); - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/conv.cpp b/ggml/src/ggml-openvino/openvino/op/conv.cpp deleted file mode 100644 index 94173dc4f833..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/conv.cpp +++ /dev/null @@ -1,233 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -OutputVector translate_conv_2d(const NodeContext & context) { - num_inputs_check(context, 2, 2); - - ov::Output kernel = process_view_input_new(context, 0); - ov::Output input = process_view_input_new(context, 1); - - if (kernel.get_element_type() != input.get_element_type()) { - kernel = std::make_shared(kernel, input.get_element_type()); - } - - const int32_t * params = context.get_output_op_params(); - const int s0 = params[0]; - const int s1 = params[1]; - const int p0 = params[2]; - const int p1 = params[3]; - const int d0 = params[4]; - const int d1 = params[5]; - - ov::Strides strides{static_cast(s1), static_cast(s0)}; - ov::CoordinateDiff pads_begin{static_cast(p1), static_cast(p0)}; - ov::CoordinateDiff pads_end{static_cast(p1), static_cast(p0)}; - ov::Strides dilations{static_cast(d1), static_cast(d0)}; - - ov::Output res = std::make_shared( - input, kernel, strides, pads_begin, pads_end, dilations, ov::op::PadType::EXPLICIT); - - const auto output_type = context.get_output_type(); - if (res.get_element_type() != output_type) { - res = std::make_shared(res, output_type); - } - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -OutputVector translate_conv_2d_dw(const NodeContext & context) { - num_inputs_check(context, 2, 2); - - ov::Output kernel = process_view_input_new(context, 0); - ov::Output input = process_view_input_new(context, 1); - - if (kernel.get_element_type() != input.get_element_type()) { - kernel = std::make_shared(kernel, input.get_element_type()); - } - - const int32_t * params = context.get_output_op_params(); - const int s0 = params[0]; - const int s1 = params[1]; - const int p0 = params[2]; - const int p1 = params[3]; - const int d0 = params[4]; - const int d1 = params[5]; - - // Reshape kernel from [C, 1, KH, KW] to [C, 1, 1, KH, KW] for 2D GroupConvolution - auto unsqueeze_axis = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {1}); - auto kernel_5d = std::make_shared(kernel, unsqueeze_axis); - - ov::Strides strides{static_cast(s1), static_cast(s0)}; - ov::CoordinateDiff pads_begin{static_cast(p1), static_cast(p0)}; - ov::CoordinateDiff pads_end{static_cast(p1), static_cast(p0)}; - ov::Strides dilations{static_cast(d1), static_cast(d0)}; - - ov::Output res = std::make_shared( - input, kernel_5d, strides, pads_begin, pads_end, dilations, ov::op::PadType::EXPLICIT); - - const auto output_type = context.get_output_type(); - if (res.get_element_type() != output_type) { - res = std::make_shared(res, output_type); - } - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -OutputVector translate_conv_transpose_1d(const NodeContext & context) { - num_inputs_check(context, 2, 2); - - ov::Output kernel = process_view_input_new(context, 0); - ov::Output input = process_view_input_new(context, 1); - - if (kernel.get_element_type() != input.get_element_type()) { - kernel = std::make_shared(kernel, input.get_element_type()); - } - - const int32_t * params = context.get_output_op_params(); - const int s0 = params[0]; - const int p0 = params[1]; - const int d0 = params[2]; - - const auto kernel_shape = context.get_input_shape(0).to_shape(); // [1, Cin, Cout, K] - const int64_t Cin = kernel_shape[1]; - const int64_t Cout = kernel_shape[2]; - const int64_t K = kernel_shape[3]; - - const auto input_shape = context.get_input_shape(1).to_shape(); // [1, N, Cin, L] - const int64_t N = input_shape[0] * input_shape[1]; - const int64_t L = input_shape[3]; - - auto kernel_3d = std::make_shared( - kernel, ov::op::v0::Constant::create(ov::element::i64, {3}, {Cin, Cout, K}), false); - auto input_3d = std::make_shared( - input, ov::op::v0::Constant::create(ov::element::i64, {3}, {N, Cin, L}), false); - - ov::Strides strides{static_cast(s0)}; - ov::CoordinateDiff pads_begin{static_cast(p0)}; - ov::CoordinateDiff pads_end{static_cast(p0)}; - ov::Strides dilations{static_cast(d0)}; - - auto conv_tr = std::make_shared( - input_3d, kernel_3d, strides, pads_begin, pads_end, dilations); - - const auto out_shape = context.get_output_shape().to_shape(); - auto out_shape_const = ov::op::v0::Constant::create( - ov::element::i64, {4}, {static_cast(out_shape[0]), static_cast(out_shape[1]), - static_cast(out_shape[2]), static_cast(out_shape[3])}); - ov::Output res = std::make_shared(conv_tr, out_shape_const, false); - - const auto output_type = context.get_output_type(); - if (res.get_element_type() != output_type) { - res = std::make_shared(res, output_type); - } - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -OutputVector translate_conv_transpose_2d(const NodeContext & context) { - num_inputs_check(context, 2, 2); - - ov::Output kernel = process_view_input_new(context, 0); - ov::Output input = process_view_input_new(context, 1); - - if (kernel.get_element_type() != input.get_element_type()) { - kernel = std::make_shared(kernel, input.get_element_type()); - } - - const int32_t * params = context.get_output_op_params(); - const int stride = params[0]; - - ov::Strides strides{static_cast(stride), static_cast(stride)}; - ov::CoordinateDiff pads_begin{0, 0}; - ov::CoordinateDiff pads_end{0, 0}; - ov::Strides dilations{1, 1}; - - ov::Output res = std::make_shared( - input, kernel, strides, pads_begin, pads_end, dilations); - - const auto output_type = context.get_output_type(); - if (res.get_element_type() != output_type) { - res = std::make_shared(res, output_type); - } - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -OutputVector translate_conv_3d(const NodeContext & context) { - num_inputs_check(context, 2, 2); - - ov::Output kernel = process_view_input_new(context, 0); - ov::Output input = process_view_input_new(context, 1); - - if (kernel.get_element_type() != input.get_element_type()) { - kernel = std::make_shared(kernel, input.get_element_type()); - } - - const int32_t * params = context.get_output_op_params(); - const int s0 = params[0]; - const int s1 = params[1]; - const int s2 = params[2]; - const int p0 = params[3]; - const int p1 = params[4]; - const int p2 = params[5]; - const int d0 = params[6]; - const int d1 = params[7]; - const int d2 = params[8]; - const int c = params[9]; - const int n = params[10]; - const int oc = params[11]; - - const auto kshape = context.get_input_shape(0).to_shape(); // [c*oc, KD, KH, KW] - const int64_t KD = kshape[1]; - const int64_t KH = kshape[2]; - const int64_t KW = kshape[3]; - - const auto inshape = context.get_input_shape(1).to_shape(); // [c*n, ID, IH, IW] - const int64_t ID = inshape[1]; - const int64_t IH = inshape[2]; - const int64_t IW = inshape[3]; - - auto kernel_5d = std::make_shared( - kernel, ov::op::v0::Constant::create(ov::element::i64, {5}, {static_cast(oc), static_cast(c), KD, KH, KW}), false); - auto input_5d = std::make_shared( - input, ov::op::v0::Constant::create(ov::element::i64, {5}, {static_cast(n), static_cast(c), ID, IH, IW}), false); - - ov::Strides strides{static_cast(s2), static_cast(s1), static_cast(s0)}; - ov::CoordinateDiff pads_begin{static_cast(p2), static_cast(p1), static_cast(p0)}; - ov::CoordinateDiff pads_end{static_cast(p2), static_cast(p1), static_cast(p0)}; - ov::Strides dilations{static_cast(d2), static_cast(d1), static_cast(d0)}; - - auto conv = std::make_shared( - input_5d, kernel_5d, strides, pads_begin, pads_end, dilations, ov::op::PadType::EXPLICIT); - - const auto out_shape = context.get_output_shape().to_shape(); // [oc*n, OD, OH, OW] - auto out_shape_const = ov::op::v0::Constant::create( - ov::element::i64, {4}, {static_cast(out_shape[0]), static_cast(out_shape[1]), - static_cast(out_shape[2]), static_cast(out_shape[3])}); - ov::Output res = std::make_shared(conv, out_shape_const, false); - - const auto output_type = context.get_output_type(); - if (res.get_element_type() != output_type) { - res = std::make_shared(res, output_type); - } - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/cpy.cpp b/ggml/src/ggml-openvino/openvino/op/cpy.cpp deleted file mode 100644 index 2c7299a894a1..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/cpy.cpp +++ /dev/null @@ -1,314 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -OutputVector translate_cpy(const NodeContext & context) { - auto op_case = context.get_op_case(); - auto input_shape = context.get_input_shape(0); - auto output_shape = context.get_input_shape(1); - - if (op_case == 4) { - auto src = process_view_input_new(context, 0); - auto base = context.get_input(1); - - int64_t n_elems = 1; - for (const auto & dim : context.get_output_shape().to_shape()) { - n_elems *= static_cast(dim); - } - - const auto output_stride = context.get_output_stride(); - const size_t elem_size = output_stride.empty() ? context.get_output_type().size() : output_stride.back(); - FRONT_END_OP_CONVERSION_CHECK(elem_size > 0, "CPY conv state view update has invalid element size"); - - const int64_t begin_val = static_cast(context.get_output_op_offset() / elem_size); - const int64_t end_val = begin_val + n_elems; - - auto flat_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{1, 1, 1, -1}); - src = std::make_shared(src, flat_shape, false); - if (src.get_element_type() != context.get_output_type()) { - src = std::make_shared(src, context.get_output_type()); - } - - auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); - auto begin = ov::op::v0::Constant::create(ov::element::i64, {1}, {begin_val}); - auto end = ov::op::v0::Constant::create(ov::element::i64, {1}, {end_val}); - auto int_max = ov::op::v0::Constant::create(ov::element::i64, {1}, {INT_MAX}); - auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {3}); - - auto head_part = std::make_shared(base, zero, begin, one, axis); - auto tail_part = std::make_shared(base, end, int_max, one, axis); - auto res = std::make_shared(ov::OutputVector{head_part, src, tail_part}, 3); - return rename_outputs_with_suffix({res}, context.get_name()); - } - - // Recurrent state cache writeback into a slot block of the cache. Where the block starts is a - // runtime input, so the cached model works for any kv head and active sequence count. The - // result is the full updated cache. - // op_case 1: gated-delta-net state, op_case 2: conv state, op_case 3: defrag remainder. - if (op_case == 3) { - // With -np 1 (and generally whenever there is no defrag remainder) this GET_ROWS gathers - // zero rows: nothing to write back, and the cache is unchanged. NPU rejects zero-size - // tensors, so short-circuit instead of building a degenerate Slice/Concat chain. - bool is_empty = false; - if (input_shape.rank().is_static()) { - for (const auto & d : input_shape) { - if (d.is_static() && d.get_length() == 0) { - is_empty = true; - break; - } - } - } - if (is_empty) { - return {context.get_input(1)}; - } - } - // op_case 7/8/9 are the single-slot variants; op_case 10 writes native GDN state into a - // multi-slot cache without rollback snapshots. - const bool single_slot_assign = op_case >= 7 && op_case <= 9; - const bool direct_gdn_state = op_case == 7 || op_case == 10; - int writeback_case = op_case; - if (op_case == 10) { - writeback_case = 1; - } else if (single_slot_assign) { - writeback_case = op_case - 6; - } - const std::string slot_begin_name = "rs_slot_begin_" + context.get_name(); - const bool slice_assign = writeback_case >= 1 && writeback_case <= 3 && - (single_slot_assign || context.has_input(slot_begin_name)); - if (slice_assign) { - if (single_slot_assign && writeback_case == 3) { - return {context.get_input(1)}; - } - - const int64_t slot_axis = 2; - auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); - auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - auto int_max = ov::op::v0::Constant::create(ov::element::i64, {1}, {INT_MAX}); - auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {slot_axis}); - auto feature = ov::op::v0::Constant::create(ov::element::i64, {4}, - std::vector{1, 1, -1, output_shape[3].get_length()}); - - ov::Output src; - ov::Output begin; - if (!single_slot_assign) { - begin = context.get_input(slot_begin_name); - } - auto base = context.get_input(1); - if (writeback_case == 1) { - if (direct_gdn_state) { - // Non-rollback GDN publishes state directly as [active_slots, heads, value_dim, - // key_dim]. Flatten each active slot before replacing or updating the cache. - src = std::make_shared(context.get_input(0), feature, false); - } else { - // Multi-slot rollback still consumes GGML's packed [attention | state snapshots] - // layout. Slice the state block using the runtime source offset. - auto src_begin = context.get_input("rs_src_begin_" + context.get_name()); - auto state_part = - std::make_shared(context.get_input(0), src_begin, int_max, one, axis); - src = std::make_shared(state_part, feature, false); - } - } else if (writeback_case == 2) { - // conv_input is [previous conv state | new tokens]; the snapshot is the conv_kernel_size - 1 - // columns ending at the last *valid* token. Gather (rather than Slice) keeps the output - // shape static even though the window start is a runtime value. - auto window_size = (int64_t) input_shape[3].get_length(); - ov::Output window; - auto col_axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {3}); - const std::string src_begin_name = "rs_src_begin_" + context.get_name(); - if (context.has_input(src_begin_name)) { - auto src_begin = context.get_input(src_begin_name); - auto src_end = std::make_shared( - src_begin, ov::op::v0::Constant::create(ov::element::i64, {1}, {window_size})); - window = std::make_shared(context.get_input(0), src_begin, src_end, one, col_axis); - } else if (context.has_input("chunk_valid_len")) { - std::vector offsets(window_size); - std::iota(offsets.begin(), offsets.end(), 0); - auto indices = std::make_shared( - ov::op::v0::Constant::create(ov::element::i64, {(size_t) window_size}, offsets), - context.get_input("chunk_valid_len")); - window = std::make_shared(context.get_input(0), indices, col_axis); - } else { - auto window_begin = ov::op::v0::Constant::create(ov::element::i64, {1}, {-window_size}); - window = - std::make_shared(context.get_input(0), window_begin, int_max, one, col_axis); - } - const auto base_shape = base.get_partial_shape(); - FRONT_END_OP_CONVERSION_CHECK(base_shape.rank().is_static() && base_shape.rank().get_length() == 4, - "CPY conv state cache update requires rank-4 base cache"); - FRONT_END_OP_CONVERSION_CHECK(base_shape[3].is_static(), - "CPY conv state cache update requires static feature size"); - FRONT_END_OP_CONVERSION_CHECK(input_shape.rank().is_static() && input_shape.rank().get_length() == 4 && - input_shape[2].is_static() && input_shape[3].is_static(), - "CPY conv state cache update requires static source feature view"); - - const int64_t full_feature_size = base_shape[3].get_length(); - const int64_t update_feature_size = input_shape[2].get_length() * input_shape[3].get_length(); - const auto output_stride = context.get_output_stride(); - const size_t elem_size = output_stride.empty() ? context.get_output_type().size() : output_stride.back(); - FRONT_END_OP_CONVERSION_CHECK(elem_size > 0, - "CPY conv state cache update has invalid element size"); - const int64_t feature_begin = static_cast(context.get_output_op_offset() / elem_size) % - full_feature_size; - const int64_t feature_end = feature_begin + update_feature_size; - FRONT_END_OP_CONVERSION_CHECK(feature_begin >= 0 && feature_end <= full_feature_size, - "CPY conv state cache update feature range is out of bounds"); - - auto partial_feature = ov::op::v0::Constant::create( - ov::element::i64, {4}, std::vector{1, 1, -1, update_feature_size}); - src = std::make_shared(window, partial_feature, false); - if (src.get_element_type() != context.get_output_type()) { - src = std::make_shared(src, context.get_output_type()); - } - - if (single_slot_assign) { - return rename_outputs_with_suffix({src}, context.get_name()); - } - - auto src_len = std::make_shared( - std::make_shared(src, ov::element::i64), axis, - ov::op::v0::Constant::create(ov::element::i64, {}, {0})); - auto slot_end = std::make_shared(begin, src_len); - auto active_slots = std::make_shared(base, begin, slot_end, one, axis); - - auto feature_axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {3}); - auto feature_begin_node = ov::op::v0::Constant::create(ov::element::i64, {1}, {feature_begin}); - auto feature_end_node = ov::op::v0::Constant::create(ov::element::i64, {1}, {feature_end}); - auto feature_head = std::make_shared(active_slots, zero, feature_begin_node, one, - feature_axis); - auto feature_tail = std::make_shared(active_slots, feature_end_node, int_max, one, - feature_axis); - src = std::make_shared(ov::OutputVector{feature_head, src, feature_tail}, 3); - } else { - // op_case 3: gathered remainder rows already have the cache slot layout [1, 1, extra, feature] - src = context.get_input(0); - } - - if (src.get_element_type() != context.get_output_type()) { - src = std::make_shared(src, context.get_output_type()); - } - - if (single_slot_assign) { - return rename_outputs_with_suffix({src}, context.get_name()); - } - - auto src_len = - std::make_shared(std::make_shared(src, ov::element::i64), axis, - ov::op::v0::Constant::create(ov::element::i64, {}, {0})); - auto end = std::make_shared(begin, src_len); - auto head_part = std::make_shared(base, zero, begin, one, axis); - auto tail_part = std::make_shared(base, end, int_max, one, axis); - auto res = std::make_shared(ov::OutputVector{head_part, src, tail_part}, slot_axis); - return rename_outputs_with_suffix({res}, context.get_name()); - } - - auto input = process_view_input_new(context, 0); - - if (op_case == 5 || op_case == 6) { - auto input_shape = context.get_input_shape(0); - auto output_shape = context.get_output_shape(); - auto dst_ggml_shape = context.get_view_input_ggml_shape(1, 0); - auto dst_stride = context.get_view_input_stride(1, 0); - size_t offset_bytes = context.get_view_input_offset(1, 0); - auto n_state = (int64_t) context.get_input_shape(0)[3].get_length(); - auto n_state_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_state}); - auto kv_buf = context.get_input(1); // shape {1,1,1,N} - - Output token_len_per_seq; - Output n_write_dyn; - if (context.has_input("token_len_per_seq")) { - token_len_per_seq = context.get_input("token_len_per_seq"); - n_write_dyn = std::make_shared(token_len_per_seq, n_state_c); - } else { - n_write_dyn = ov::op::v0::Constant::create(ov::element::i64, {1}, {(int64_t) dst_ggml_shape[3]}); - } - size_t elem_size = dst_stride[3]; - FRONT_END_OP_CONVERSION_CHECK(elem_size > 0, "CPY KV cache view update has invalid element size"); - int64_t start_elem = (int64_t) (offset_bytes / elem_size); - // op_case 5: decoder self-attention – write offset advances each step. - // op_case 6: encoder self-attn or cross-attn – offset fixed at compile time. - const bool is_decoder_self_attn = (op_case == 5); - auto ones_c = ov::op::v0::Constant::create(ov::element::i64, {3}, std::vector{1, 1, 1}); - auto new_shape = std::make_shared(ov::OutputVector{ones_c, n_write_dyn}, 0); - - auto reshaped = std::make_shared(input, new_shape, false); - auto data = std::make_shared(reshaped, context.get_output_type()); - // Indices [start_elem .. start_elem + n_write) on axis 3 of {1,1,1,N} - // For decoder self-attention the write offset advances each step, so compute it - // dynamically from the model inputs: start = (attention_size - token_len_per_seq) * n_state. - // For encoder self-attn and cross-attn the offset is fixed at graph-compile time. - ov::Output start; - if (is_decoder_self_attn && context.has_input("attention_size") && context.has_input("token_len_per_seq")) { - auto attention_size_in = context.get_input("attention_size"); - auto token_len_in = context.get_input("token_len_per_seq"); - auto past_tokens = std::make_shared(attention_size_in, token_len_in); - auto new_start = std::make_shared(past_tokens, n_state_c); - start = std::make_shared( - new_start, ov::op::v0::Constant::create(ov::element::i64, {1}, {start_elem})); - } else { - start = ov::op::v0::Constant::create(ov::element::i64, {1}, {start_elem}); - } - auto start_squeezed = std::make_shared(start); - auto end = std::make_shared(start_squeezed, n_write_dyn); - auto end_squeezed = std::make_shared(end); - auto step = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - auto step_squeezed = std::make_shared(step); - auto indices = - std::make_shared(start_squeezed, end_squeezed, step_squeezed, ov::element::i64); - auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {3}); - - auto kv_updated = std::make_shared(kv_buf, indices, data, axis); - return rename_outputs_with_suffix({kv_updated}, context.get_name()); - } - - if (input_shape != output_shape) { - auto new_shape = ov::op::v0::Constant::create( - ov::element::i64, {static_cast(output_shape.rank().get_length())}, output_shape.to_shape()); - input = std::make_shared(input, new_shape, false); - } - - ov::Output res; - if (context.get_input_type(0) != context.get_output_type()) { - res = std::make_shared(input, context.get_output_type()); - } else { - res = input; - } - - if (res.get_node_shared_ptr() == context.get_input(0).get_node_shared_ptr()) { - return {res}; - } - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/cumsum.cpp b/ggml/src/ggml-openvino/openvino/op/cumsum.cpp deleted file mode 100644 index 0a414b24f6f5..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/cumsum.cpp +++ /dev/null @@ -1,29 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -// GGML cumsum computes prefix sum along dim 0 (the innermost/fastest dimension). -// In OV layout the dims are reversed: ggml [ne0, ne1, ne2, ne3] → OV [ne3, ne2, ne1, ne0], -// so ggml dim 0 maps to OV axis 3 (last axis). -OutputVector translate_cumsum(const NodeContext & context) { - num_inputs_check(context, 1, 1); - - auto x = context.get_input(0); - auto axis = ov::op::v0::Constant::create(ov::element::i64, {}, {3}); - auto res = std::make_shared(x, axis); - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/diag.cpp b/ggml/src/ggml-openvino/openvino/op/diag.cpp deleted file mode 100644 index 05e064892e17..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/diag.cpp +++ /dev/null @@ -1,37 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -// GGML DIAG takes a 1D vector (ne0, 1, ne2, ne3) and produces a diagonal matrix -// of shape (ne0, ne0, ne2, ne3). -// In OV layout (ggml [ne0, ne1, ne2, ne3] → OV [ne3, ne2, ne1, ne0]): -// input: [ne3, ne2, 1, ne0] -// output: [ne3, ne2, ne0, ne0] -// The diagonal: output[..., i, j] = input[..., 0, j] if i == j, else 0. -OutputVector translate_diag(const NodeContext & context) { - num_inputs_check(context, 1, 1); - - auto x = process_view_input_new(context, 0); // OV shape: [ne3, ne2, 1, ne0] - - auto n = get_dimensions(x.get_node_shared_ptr(), {3}); - auto zero_diag = ov::op::v0::Constant::create(ov::element::i64, {}, {0}); - - auto eye = std::make_shared(n, n, zero_diag, x.get_element_type()); - auto res = std::make_shared(x, eye); - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/div.cpp b/ggml/src/ggml-openvino/openvino/op/div.cpp deleted file mode 100644 index 2089ffd4c819..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/div.cpp +++ /dev/null @@ -1,138 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" -#include "ggml.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -namespace { - -bool is_silu_div_pattern(const ov::Output & numerator, - const ov::Output & denominator, - const NodeContext & context) { - if (context.get_input_size() != 2) { - return false; - } - - const auto * unary_op = reinterpret_cast(context.get_input_op_params(0)); - if (unary_op == nullptr || *unary_op != GGML_UNARY_OP_SILU) { - return false; - } - - const auto denom_node = denominator.get_node_shared_ptr(); - - if (auto swish = ov::as_type_ptr(numerator.get_node_shared_ptr())) { - return swish->input_value(0).get_node_shared_ptr() == denom_node; - } - return false; -} - -ov::Output repeat_input_to_match(const NodeContext & context, - const ov::Output & input, - const ov::Output & target, - size_t input_index) { - const auto input_shape = context.get_input_shape(input_index); - const auto target_shape = context.get_input_shape(0); - - if (input_shape == target_shape) { - return input; - } - - if (input_shape.rank().is_static() && target_shape.rank().is_static()) { - const auto rank = static_cast(input_shape.rank().get_length()); - std::vector repeats(rank, 1); - bool needs_repeat = false; - - for (size_t axis = 0; axis < rank; ++axis) { - FRONT_END_OP_CONVERSION_CHECK(input_shape[axis].is_static() && target_shape[axis].is_static(), - "DIV repeat requires static dimensions on both inputs"); - - const int64_t input_dim = input_shape[axis].get_length(); - const int64_t target_dim = target_shape[axis].get_length(); - - FRONT_END_OP_CONVERSION_CHECK(input_dim > 0 && target_dim > 0 && target_dim % input_dim == 0, - "DIV input shape ", input_shape, " cannot repeat to match ", target_shape); - - repeats[axis] = target_dim / input_dim; - needs_repeat = needs_repeat || repeats[axis] != 1; - } - - if (!needs_repeat) { - return input; - } - - auto repeats_node = ov::op::v0::Constant::create(ov::element::i64, {repeats.size()}, repeats); - return std::make_shared(input, repeats_node); - } - - auto input_shape_node = std::make_shared(input, ov::element::i64); - auto target_shape_node = std::make_shared(target, ov::element::i64); - auto repeats_node = std::make_shared(target_shape_node, input_shape_node); - return std::make_shared(input, repeats_node); -} - -} // namespace - -OutputVector translate_div(const NodeContext & context) { - num_inputs_check(context, 2, 2); - - auto input_0 = process_view_input_new(context, 0); - auto input_1 = process_view_input_new(context, 1); - - if (is_silu_div_pattern(input_0, input_1, context)) { - ov::Output res = std::make_shared(input_1); - if (res.get_element_type() != context.get_output_type()) { - res = std::make_shared(res, context.get_output_type()); - } - return rename_outputs_with_suffix({res}, context.get_name()); - } - - input_1 = repeat_input_to_match(context, input_1, input_0, 1); - - const auto output_type = context.get_output_type(); - const bool use_f32_compute = input_0.get_element_type() != ov::element::f32 || - input_1.get_element_type() != ov::element::f32 || output_type != ov::element::f32; - - if (use_f32_compute) { - input_0 = std::make_shared(input_0, ov::element::f32); - input_1 = std::make_shared(input_1, ov::element::f32); - } - - ov::Output res = std::make_shared(input_0, input_1); - if (use_f32_compute) { - // Keep the reciprocal/divide path in FP32. Without this hint, the GPU - // plugin can still compress the subgraph back to FP16 and overflow on - // small shexp gate values (e.g. silu(x) / x in qwen2moe). - ov::mark_as_precision_sensitive(res.get_node_shared_ptr()->input(0)); - ov::mark_as_precision_sensitive(res.get_node_shared_ptr()->input(1)); - } - if (res.get_element_type() != output_type) { - auto output_convert = std::make_shared(res, output_type); - if (use_f32_compute) { - ov::mark_as_precision_sensitive(output_convert->input(0)); - } - res = output_convert; - } - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/fill.cpp b/ggml/src/ggml-openvino/openvino/op/fill.cpp deleted file mode 100644 index db2fecb53caf..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/fill.cpp +++ /dev/null @@ -1,34 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -// GGML FILL sets all elements of a tensor to a constant value. -// The constant is stored as a float in op_params[0]. -OutputVector translate_fill(const NodeContext & context) { - num_inputs_check(context, 1, 1); - - float c; - memcpy(&c, context.get_output_op_params(), sizeof(float)); - - auto shape = context.get_input_shape(0).to_shape(); - - auto val = ov::op::v0::Constant::create(ov::element::f32, {}, {c}); - auto target_shape = ov::op::v0::Constant::create(ov::element::i64, {shape.size()}, - std::vector(shape.begin(), shape.end())); - auto res = std::make_shared(val, target_shape); - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp b/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp deleted file mode 100644 index 44c6dbc03ddb..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/flash_attn_ext.cpp +++ /dev/null @@ -1,242 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" -#include "ggml-openvino/ggml-openvino-extra.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { -static ov::Output reshape_flat_kv(const ov::Output & kv_flat, - size_t view_offset_bytes, - size_t nb1_bytes, - int64_t n_head, - int64_t head_size, - const ov::Output & attention_size) { - int64_t n_state = n_head * head_size; - int64_t layer_start_elem = (int64_t) (view_offset_bytes / (nb1_bytes / n_state)); - // Dynamic slice: [layer_start_elem, layer_start_elem + n_kv * n_state) - auto start_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {layer_start_elem}); - auto n_state_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_state}); - // end = start + attention_size * n_state (both static + dynamic) - auto kv_len_elems = std::make_shared(attention_size, n_state_c); - auto end_c = std::make_shared(start_c, kv_len_elems); - auto step_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - auto axis_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {3}); - auto sliced = std::make_shared(kv_flat, start_c, end_c, step_c, axis_c); - - // KV cache is laid out as {n_kv, n_head, head_size} in memory - // Reshape to {1, n_kv, n_head, head_size}, then transpose to {1, n_head, n_kv, head_size} - // as required by SDPA. - auto one_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - auto n_head_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_head}); - auto head_size_c = ov::op::v0::Constant::create(ov::element::i64, {1}, {head_size}); - // reshape: {n_kv*n_state} -> {1, n_kv, n_head, head_size} - auto new_shape = - std::make_shared(ov::OutputVector{one_c, attention_size, n_head_c, head_size_c}, 0); - auto reshaped = std::make_shared(sliced, new_shape, false); - // transpose: {1, n_kv, n_head, head_size} -> {1, n_head, n_kv, head_size} - auto perm = ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 2, 1, 3}); - auto ret = std::make_shared(reshaped, perm); - return ret; -} - -OutputVector translate_flash_attn_ext(const NodeContext & context) { - num_inputs_check(context, 3, 4); - const bool has_mask = context.get_input_size() == 4; - auto q_f32 = context.get_input(0); - auto k = context.get_input(1); - auto v = context.get_input(2); - const int op_case = context.get_op_case(); - - if (op_case == 1 || op_case == 2) { - int64_t n_state_head = (int64_t) context.get_view_input_ggml_shape(1, 0)[3]; - int64_t n_head = (int64_t) context.get_view_input_ggml_shape(1, 0)[1]; - size_t nb1 = context.get_view_input_stride(1, 0)[2]; - size_t offset = context.get_view_input_offset(1, 0); - ov::Output attention_size; - if (op_case == 1) { - attention_size = context.get_input("attention_size"); - } else { - attention_size = context.get_input("attention_size_static"); - } - k = reshape_flat_kv(k, offset, nb1, n_head, n_state_head, attention_size); - v = reshape_flat_kv(v, offset, nb1, n_head, n_state_head, attention_size); - } - - float * params = reinterpret_cast(context.get_output_op_params()); - float scale = params[0]; - // float max_bias = params[1]; - // float logit_softcap = params[2]; - - auto q = std::make_shared(q_f32, ov::element::f16); - auto scale_node = std::make_shared(ov::element::f16, ov::Shape{}, std::vector{scale}); - - ov::Output res; - - // For stateful - ov::Output mask; - if (has_mask) { - mask = context.get_input(3); - std::string mask_name = "KQ_mask_sliced"; - if (context.get_input_names()[3].find("swa") != std::string::npos) { - mask_name = "KQ_mask_swa_sliced"; - } - if (context.has_input(mask_name)) { - mask = context.get_input(mask_name); - } - if (mask.get_element_type() != ov::element::f16) { - mask = std::make_shared(mask, ov::element::f16); - } - } - - //auto tile_kv = [&](int64_t num_heads, int64_t num_heads_kv, int64_t head_size, ov::Output kv) { - // int64_t factor = num_heads / num_heads_kv; - // if (factor > 1 && num_heads_kv > 1) { - auto q_shape = context.get_input_shape(0).to_shape(); - auto k_shape = context.get_input_shape(1).to_shape(); - const int64_t num_heads = q_shape[1]; - const int64_t num_heads_kv = k_shape[1]; - const int64_t head_size = q_shape[3]; - const int64_t factor = num_heads / num_heads_kv; - - // Manual GQA attention: enabled by default on GPU in stateless mode. - // Set GGML_OPENVINO_MANUAL_GQA_ATTN to a positive value (e.g. 1) to force-enable, - // or to 0 to force-disable. Unset falls back to the device-based default. - static const bool manual_gqa_enabled = []() { - const char * env = ggml_openvino_getenv_str("GGML_OPENVINO_MANUAL_GQA_ATTN"); - if (env != nullptr) { - return ggml_openvino_getenv_int("GGML_OPENVINO_MANUAL_GQA_ATTN") > 0; - } - return ggml_openvino_is_gpu(); - }(); - const bool use_manual_gqa_attention = - manual_gqa_enabled && factor > 1 && num_heads_kv > 1 && !context.is_stateful(); - - if (use_manual_gqa_attention) { - // Q, K, V arrive as [B, n_heads(_kv), S, head_size], where B is the active - // batch (n_seq_active) and may be > 1 (llama-perplexity, llama-server -np > 1) - // or dynamic. Reshape to - // K_r: [B, num_heads_kv, 1, S, head_size] - // Q_r: [B, num_heads_kv, factor, S_q, head_size] - // and let MatMul broadcast across the factor dim without materialising - // an expanded K/V. The leading 0 + special_zero=true copies B at runtime, - // so this is correct for B == 1, B > 1, and dynamic B alike. Only the head - // dims and head_size are baked in as literals; the sequence dim stays -1. - auto k_5d_shape = ov::op::v0::Constant::create(ov::element::i64, {5}, - std::vector{0, num_heads_kv, 1, -1, head_size}); - auto v_5d_shape = ov::op::v0::Constant::create(ov::element::i64, {5}, - std::vector{0, num_heads_kv, 1, -1, head_size}); - auto q_5d_shape = ov::op::v0::Constant::create(ov::element::i64, {5}, - std::vector{0, num_heads_kv, factor, -1, head_size}); - - auto k_r = std::make_shared(k, k_5d_shape, true); - auto v_r = std::make_shared(v, v_5d_shape, true); - auto q_r = std::make_shared(q, q_5d_shape, true); - - // QK^T → [B, num_heads_kv, factor, S_q, S_k] - auto qk = std::make_shared(q_r, k_r, /*tA=*/false, /*tB=*/true); - auto qk_scaled = std::make_shared(qk, scale_node); - - // Mask arrives as [B, 1, S_q, S_k]. Unsqueeze a factor axis at position 2 to - // get [B, 1, 1, S_q, S_k], which NUMPY-broadcasts cleanly against the - // [B, num_heads_kv, factor, S_q, S_k] scores: B==B, then 1→num_heads_kv and - // 1→factor on the head dims. - ov::Output qk_masked; - if (has_mask) { - auto mask_unsq1 = - std::make_shared(mask, ov::op::v0::Constant::create(ov::element::i64, {1}, {2})); - qk_masked = std::make_shared(qk_scaled, mask_unsq1); - } else { - qk_masked = qk_scaled; - } - - auto softmax = std::make_shared(qk_masked, /*axis=*/-1); - - // softmax @ V → [B, num_heads_kv, factor, S_q, head_size] - auto attn = std::make_shared(softmax, v_r); - - // Reshape back to [B, num_heads, S_q, head_size] (combine num_heads_kv * factor). - // Leading 0 + special_zero=true copies B at runtime. - auto out_4d_shape = - ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{0, num_heads, -1, head_size}); - auto out_4d = std::make_shared(attn, out_4d_shape, true); - - // The standard SDPA path's downstream is Transpose(0,2,1,3) → Convert(f32). - // Replicate it here so callers see the same output layout/dtype. - res = std::make_shared( - out_4d, ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 2, 1, 3})); - res = std::make_shared(res, ov::element::f32); - return rename_outputs_with_suffix({res}, context.get_name()); - } - - // Default path: explicit Broadcast → SDPA. Kept as the fallback because - // (a) it goes through the GPU plugin's micro-SDPA fast path (FlashAttention - // tiles via DPAS), and (b) the manual path above is still being validated. - auto tile_kv = [&](int64_t n_heads, int64_t n_heads_kv, int64_t hs, ov::Output kv) { - int64_t f = n_heads / n_heads_kv; - if (f > 1 && n_heads_kv > 1) { - ov::Output kv_broadcast_shape; - ov::Output kv_unsqueezed; - ov::Output new_kv_shape; - auto unsqueeze_axes = ov::op::v0::Constant::create(ov::element::i64, Shape{}, {2}); - kv_unsqueezed = std::make_shared(kv, unsqueeze_axes); - - kv_broadcast_shape = ov::op::v0::Constant::create(ov::element::i64, {5}, - {(int64_t) 1, (int64_t) 1, f, (int64_t) 1, (int64_t) 1}); - new_kv_shape = - ov::op::v0::Constant::create(ov::element::i64, {4}, {(int64_t) 0, n_heads, (int64_t) -1, hs}); - // ov::element::i64, {5}, {(int64_t) 1, (int64_t) 1, factor, (int64_t) 1, (int64_t) 1}); - //new_kv_shape = - // ov::op::v0::Constant::create(ov::element::i64, {4}, {(int64_t) 0, num_heads, (int64_t) -1, head_size}); - - kv = std::make_shared(kv_unsqueezed, kv_broadcast_shape, - ov::op::BroadcastType::BIDIRECTIONAL); - kv = std::make_shared(kv, new_kv_shape, true); - } - return kv; - }; - - //auto q_shape = context.get_input_shape(0).to_shape(); - //auto k_shape = context.get_input_shape(1).to_shape(); - //k = tile_kv(q_shape[1], k_shape[1], q_shape[3], k); - //v = tile_kv(q_shape[1], k_shape[1], q_shape[3], v); - k = tile_kv(num_heads, num_heads_kv, head_size, k); - v = tile_kv(num_heads, num_heads_kv, head_size, v); - - constexpr auto causal = false; - if (has_mask) { - auto sdpa = std::make_shared(q, k, v, mask, scale_node, causal); - res = std::make_shared( - sdpa, ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 2, 1, 3})); - } else { - auto sdpa = std::make_shared(q, k, v, scale_node, causal); - res = std::make_shared( - sdpa, ov::op::v0::Constant::create(ov::element::i64, {4}, {0, 2, 1, 3})); - } - res = std::make_shared(res, ov::element::f32); - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp b/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp deleted file mode 100644 index 9cedf68116bf..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/gated_delta_net.cpp +++ /dev/null @@ -1,407 +0,0 @@ -#include "gated_delta_net.hpp" - -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" -#include "ggml-openvino/ggml-openvino-extra.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -static OutputVector translate_gated_delta_net_ref(const NodeContext & context); - -static bool match_gdn_l2_norm(const Output & normalized, Output & input, float & eps) { - // Match the RMSNorm decomposition emitted by translate_rms_norm, followed by GGML SCALE. - const auto scale = ov::as_type_ptr(normalized.get_node_shared_ptr()); - if (!scale) { - return false; - } - const auto factor = ov::as_type_ptr(scale->get_input_node_shared_ptr(1)); - const auto rms = ov::as_type_ptr(scale->get_input_node_shared_ptr(0)); - if (!factor || ov::shape_size(factor->get_shape()) != 1 || !rms) { - return false; - } - const auto x = rms->input_value(0); - const auto & shape = x.get_partial_shape(); - if (x.get_element_type() != ov::element::f32 || shape.rank() != 4 || shape[3].is_dynamic() || - shape[3].get_length() <= 0 || normalized.get_partial_shape() != shape) { - return false; - } - const float dim = static_cast(shape[3].get_length()); - if (factor->cast_vector()[0] != 1.0f / std::sqrt(dim)) { - return false; - } - const auto reciprocal = ov::as_type_ptr(rms->get_input_node_shared_ptr(1)); - if (!reciprocal) { - return false; - } - const auto one = ov::as_type_ptr(reciprocal->get_input_node_shared_ptr(0)); - const auto root = ov::as_type_ptr(reciprocal->get_input_node_shared_ptr(1)); - if (!one || ov::shape_size(one->get_shape()) != 1 || one->cast_vector()[0] != 1.0f || !root) { - return false; - } - const auto add = ov::as_type_ptr(root->get_input_node_shared_ptr(0)); - if (!add) { - return false; - } - const auto mean = ov::as_type_ptr(add->get_input_node_shared_ptr(0)); - const auto rms_eps = ov::as_type_ptr(add->get_input_node_shared_ptr(1)); - if (!mean || !mean->get_keep_dims() || !rms_eps || ov::shape_size(rms_eps->get_shape()) != 1) { - return false; - } - const auto axes = ov::as_type_ptr(mean->get_input_node_shared_ptr(1)); - const auto square = ov::as_type_ptr(mean->get_input_node_shared_ptr(0)); - if (!axes || axes->cast_vector() != std::vector{-1} || !square || - square->input_value(0) != x || square->input_value(1) != x) { - return false; - } - // RMSNorm(x, rms_eps) / sqrt(D) = x / sqrt(sum(x*x) + D*rms_eps). - eps = dim * rms_eps->cast_vector()[0]; - if (!std::isfinite(eps) || eps <= 0.0f) { - return false; - } - input = x; - return true; -} - -OutputVector translate_gated_delta_net(const NodeContext & context) { - auto v_shape = context.get_input_shape(2).to_shape(); // [B, T, H_v, S_v] - auto q_shape = context.get_input_shape(0).to_shape(); // [B, T, H_k, S_k] - - // Fused GatedDeltaNet op only supports scalar gate (kda=0). - // Fall back to reference implementation for per-key-dimension gating. - // if (kda) { - // return translate_gated_delta_net_ref(context); - // } - - // const int64_t B = v_shape[0]; - // const int64_t T = v_shape[1]; - const int64_t H_v = v_shape[2]; - const int64_t S_v = v_shape[3]; - const int64_t H_k = q_shape[2]; - // const int64_t S_k = q_shape[3]; - - auto q = context.get_input(0); - auto k = context.get_input(1); - auto v = process_view_input(context, 2, H_v * S_v, 3); - auto g = context.get_input(3); - auto beta = context.get_input(4); - auto state = context.get_input(5); - - Output raw_q, raw_k; - float q_eps = 1e-6f, k_eps = 1e-6f; - const bool fuse_qk_l2norm = ggml_openvino_is_gpu() && - match_gdn_l2_norm(q, raw_q, q_eps) && match_gdn_l2_norm(k, raw_k, k_eps); - if (fuse_qk_l2norm) { - // Keep head tiling below; GDN applies normalization per head and keeps its attention scale. - q = raw_q; - k = raw_k; - } - - // ggml maps GQA heads in tiled order, while OV GDN maps repeated heads in grouped order. - if (H_v != H_k) { - const int64_t repeat = H_v / H_k; - auto repeats = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{1, 1, repeat, 1}); - q = std::make_shared(q, repeats); - k = std::make_shared(k, repeats); - } - - if (context.get_view_input_size(2)) { - // Same as l2_norm case 1 - v = std::make_shared(v, ov::op::v0::Constant::create(ov::element::i64, {1}, {0})); - auto v_shape = context.get_input_shape(2).to_shape(); - std::vector reshape_pattern = {0, 0, (int64_t) v_shape[2], (int64_t) v_shape[3]}; - v = std::make_shared( - v, ov::op::v0::Constant::create(ov::element::i64, {4}, reshape_pattern), true); - } - - // ggml state layout (OV notation): [B, H_v, value_dim, key_dim] - // GatedDeltaNet op expects: [B, H_v, key_dim, value_dim] - auto state_perm = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{0, 1, 3, 2}); - state = std::make_shared(state, state_perm); - - g = std::make_shared(g, ov::op::v0::Constant::create(ov::element::i64, {1}, {3})); - beta = std::make_shared(beta, ov::op::v0::Constant::create(ov::element::i64, {1}, {3})); - - if (context.has_input("chunk_valid_len")) { - // The last prefill chunk is padded with fabricated tokens. The recurrence is - // S_t = S_{t-1} * exp(g_t) + k_t (x) ((v_t - S_{t-1}^T k_t) * beta_t) - // so forcing g = 0 and beta = 0 makes a padded step an exact identity and keeps the final - // state equal to the state after the last real token. Attention output at those positions - // is garbage but never read. - const auto & g_ps = g.get_partial_shape(); - FRONT_END_OP_CONVERSION_CHECK(g_ps.rank().is_static() && g_ps.rank().get_length() == 3 && g_ps[1].is_static(), - "GATED_DELTA_NET pad masking requires a static token dimension"); - const int64_t n_tokens = g_ps[1].get_length(); - std::vector positions(n_tokens); - std::iota(positions.begin(), positions.end(), 0); - auto valid = std::make_shared( - ov::op::v0::Constant::create(ov::element::i64, {(size_t) n_tokens}, positions), - context.get_input("chunk_valid_len")); - auto mask = std::make_shared( - std::make_shared(valid, g.get_element_type()), - ov::op::v0::Constant::create(ov::element::i64, {2}, std::vector{0, 2})); - g = std::make_shared(g, mask); - beta = std::make_shared(beta, mask); - } - - // std::cout << "GatedDeltaNet input shapes: q=" << q.get_partial_shape() << ", k=" << k.get_partial_shape() - // << ", v=" << v.get_partial_shape() << ", g=" << g.get_partial_shape() - // << ", beta=" << beta.get_partial_shape() << ", state=" << state.get_partial_shape() << std::endl; - - auto gdn = std::make_shared(q, k, v, state, g, beta, fuse_qk_l2norm, q_eps, k_eps); - auto attn_4d = gdn->output(0); - auto state_4d = gdn->output(1); // [B, H_v, key_dim, value_dim] - - // std::cout << "GatedDeltaNet output shapes: attn=" << gdn->output(0).get_partial_shape() - // << ", new_state=" << gdn->output(1).get_partial_shape() << std::endl; - - // Transpose output state back to ggml layout [B, H_v, value_dim, key_dim] - auto state_transposed = std::make_shared(state_4d, state_perm); - if (context.get_output_names().size() == 2) { - // The canonical graph consumes the packed GGML result only through separate attention - // and state VIEWs. Publish the native outputs under those VIEW names to avoid - // flatten -> concat -> reshape -> slice -> reshape chains. This also works for B > 1. - return rename_outputs_with_suffix({attn_4d, state_transposed}, context.get_name()); - } - - auto flat_shape_1d = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); - auto attn = std::make_shared(attn_4d, flat_shape_1d, false); - auto new_state = std::make_shared(state_transposed, flat_shape_1d, false); - auto packed = std::make_shared(ov::OutputVector{attn, new_state}, 0); - auto out_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, - std::vector{1, 1, -1 /*T * B + S_v * B*/, S_v * H_v}); - auto res = std::make_shared(packed, out_shape, false); - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -static OutputVector translate_gated_delta_net_ref(const NodeContext & context) { - num_inputs_check(context, 6, 6); - - // Inputs (OV shapes are reversed from ggml): - // ggml: q[S_k, H_k, T, B], k[S_k, H_k, T, B], v[S_v, H_v, T, B] - // OV: q[B, T, H_k, S_k], k[B, T, H_k, S_k], v[B, T, H_v, S_v] - // ggml: g[1 or S_v, H_v, T, B], beta[1, H_v, T, B] - // OV: g[B, T, H_v, 1 or S_v], beta[B, T, H_v, 1] - // ggml: state[S_v, S_v, H_v, B] - // OV: state[B, H_v, S_v, S_v] - auto q = process_view_input_new(context, 0); - auto k = process_view_input_new(context, 1); - auto v = process_view_input_new(context, 2); - auto g = process_view_input_new(context, 3); - auto beta = process_view_input_new(context, 4); - auto state = process_view_input_new(context, 5); - - auto v_shape = context.get_input_shape(2).to_shape(); // [B, T, H_v, S_v] - auto q_shape = context.get_input_shape(0).to_shape(); // [B, T, H_k, S_k] - auto g_shape = context.get_input_shape(3).to_shape(); // [B, T, H_v, 1 or S_v] - - const int64_t B = v_shape[0]; - const int64_t T = v_shape[1]; - const int64_t H_v = v_shape[2]; - const int64_t S_v = v_shape[3]; - const int64_t H_k = q_shape[2]; - const bool kda = (g_shape[3] == (size_t) S_v); - - const int64_t rq1 = H_v / H_k; // head repeat factor - const float scale = 1.0f / std::sqrt((float) S_v); - - auto axis_1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - auto axis_2 = ov::op::v0::Constant::create(ov::element::i64, {1}, {2}); - - // Transpose inputs from [B, T, H, S] to [B, H, T, S] for easier per-head processing - auto perm_0213 = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{0, 2, 1, 3}); - auto q_t = std::make_shared(q, perm_0213); // [B, H_k, T, S_k] - auto k_t = std::make_shared(k, perm_0213); // [B, H_k, T, S_k] - auto v_t = std::make_shared(v, perm_0213); // [B, H_v, T, S_v] - auto g_t = std::make_shared(g, perm_0213); // [B, H_v, T, 1 or S_v] - auto beta_t = std::make_shared(beta, perm_0213); // [B, H_v, T, 1] - - // Broadcast Q, K heads to match V heads if GQA is used (H_v > H_k) - ov::Output q_bh = q_t; - ov::Output k_bh = k_t; - if (rq1 > 1) { - auto q_unsq = std::make_shared(q_t, axis_2); // [B, H_k, 1, T, S] - auto k_unsq = std::make_shared(k_t, axis_2); // [B, H_k, 1, T, S] - - auto bcast_shape = ov::op::v0::Constant::create(ov::element::i64, {5}, std::vector{1, 1, rq1, 1, 1}); - auto q_bcast = - std::make_shared(q_unsq, bcast_shape, ov::op::BroadcastType::BIDIRECTIONAL); - auto k_bcast = - std::make_shared(k_unsq, bcast_shape, ov::op::BroadcastType::BIDIRECTIONAL); - - // Transpose [B, H_k, rq1, T, S] -> [B, rq1, H_k, T, S] so that reshape merges - // as [rq1, H_k] giving repeat-blocks pattern matching CPU: iq1 = iv1 % H_k - auto perm_5d = ov::op::v0::Constant::create(ov::element::i64, {5}, std::vector{0, 2, 1, 3, 4}); - auto q_transposed = std::make_shared(q_bcast, perm_5d); - auto k_transposed = std::make_shared(k_bcast, perm_5d); - - auto new_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{B, H_v, T, S_v}); - q_bh = std::make_shared(q_transposed, new_shape, false); - k_bh = std::make_shared(k_transposed, new_shape, false); - } - - // Merge batch and head dims: [B*H_v, T, S_v] - auto merge_bh = [&](const ov::Output & x, int64_t last_dim) { - auto shape = ov::op::v0::Constant::create(ov::element::i64, {3}, std::vector{B * H_v, T, last_dim}); - return std::make_shared(x, shape, false); - }; - - auto q_m = merge_bh(q_bh, S_v); // [B*H_v, T, S_v] - auto k_m = merge_bh(k_bh, S_v); // [B*H_v, T, S_v] - auto v_m = merge_bh(v_t, S_v); // [B*H_v, T, S_v] - auto g_m = merge_bh(g_t, kda ? S_v : 1); // [B*H_v, T, 1 or S_v] - auto beta_m = merge_bh(beta_t, 1); // [B*H_v, T, 1] - - // State: [B, H_v, S_v, S_v] -> [B*H_v, S_v, S_v] - auto state_shape = ov::op::v0::Constant::create(ov::element::i64, {3}, std::vector{B * H_v, S_v, S_v}); - auto state_m = std::make_shared(state, state_shape, false); - - auto scale_const = ov::op::v0::Constant::create(ov::element::f32, {}, std::vector{scale}); - - // --- Build Loop body --- - // Body parameters (no iteration counter needed, use -1 in special ports) - auto body_state = std::make_shared(ov::element::f32, ov::PartialShape::dynamic()); - auto body_q = std::make_shared(ov::element::f32, ov::PartialShape::dynamic()); - auto body_k = std::make_shared(ov::element::f32, ov::PartialShape::dynamic()); - auto body_v = std::make_shared(ov::element::f32, ov::PartialShape::dynamic()); - auto body_g = std::make_shared(ov::element::f32, ov::PartialShape::dynamic()); - auto body_beta = std::make_shared(ov::element::f32, ov::PartialShape::dynamic()); - auto body_iter = std::make_shared(ov::element::i64, ov::Shape{1}); - - // Condition output (always true - we rely on trip_count for termination) - auto body_cond_out = ov::op::v0::Constant::create(ov::element::boolean, ov::Shape{1}, std::vector{true}); - - // Gather current token from invariant inputs using iteration counter - auto q_t_cur = std::make_shared(body_q, body_iter, axis_1); // [B*H_v, 1, S_v] - auto k_t_cur = std::make_shared(body_k, body_iter, axis_1); // [B*H_v, 1, S_v] - auto v_t_cur = std::make_shared(body_v, body_iter, axis_1); // [B*H_v, 1, S_v] - auto g_t_cur = std::make_shared(body_g, body_iter, axis_1); // [B*H_v, 1, 1 or S_v] - auto b_t_cur = std::make_shared(body_beta, body_iter, axis_1); // [B*H_v, 1, 1] - - // Squeeze token dim - auto q_cur = std::make_shared(q_t_cur, axis_1); // [B*H_v, S_v] - auto k_cur = std::make_shared(k_t_cur, axis_1); // [B*H_v, S_v] - auto v_cur = std::make_shared(v_t_cur, axis_1); // [B*H_v, S_v] - auto g_cur = std::make_shared(g_t_cur, axis_1); // [B*H_v, 1 or S_v] - auto b_cur = std::make_shared(b_t_cur, axis_1); // [B*H_v, 1] - - // Step 1: Apply decay gate to state - auto exp_g = std::make_shared(g_cur); // [B*H_v, 1 or S_v] - auto exp_g_unsq = std::make_shared(exp_g, axis_1); // [B*H_v, 1, 1 or S_v] - auto state_decayed = std::make_shared(body_state, exp_g_unsq); // [B*H_v, S_v, S_v] - - // Step 2: delta = (v - S @ k) * beta - auto k_col = std::make_shared(k_cur, axis_2); // [B*H_v, S_v, 1] - auto sk = std::make_shared(state_decayed, k_col, false, false); // [B*H_v, S_v, 1] - auto sk_sq = std::make_shared(sk, axis_2); // [B*H_v, S_v] - auto v_minus_sk = std::make_shared(v_cur, sk_sq); // [B*H_v, S_v] - auto delta = std::make_shared(v_minus_sk, b_cur); // [B*H_v, S_v] - - // Step 3: state += outer(delta, k) - auto delta_col = std::make_shared(delta, axis_2); // [B*H_v, S_v, 1] - auto k_row = std::make_shared(k_cur, axis_1); // [B*H_v, 1, S_v] - auto outer_prod = std::make_shared(delta_col, k_row, false, false); // [B*H_v, S_v, S_v] - auto state_updated = std::make_shared(state_decayed, outer_prod); // [B*H_v, S_v, S_v] - - // Step 4: attn_out = S @ q * scale - auto q_col = std::make_shared(q_cur, axis_2); // [B*H_v, S_v, 1] - auto sq = std::make_shared(state_updated, q_col, false, false); // [B*H_v, S_v, 1] - auto sq_squeezed = std::make_shared(sq, axis_2); // [B*H_v, S_v] - auto attn_out = std::make_shared(sq_squeezed, scale_const); // [B*H_v, S_v] - - // Unsqueeze attn_out to [B*H_v, 1, S_v] for scan output concatenation - auto attn_out_unsq = std::make_shared(attn_out, axis_1); // [B*H_v, 1, S_v] - - // --- Assemble Loop --- - // Body: results = [condition, state_updated, attn_out_unsq] - auto body = std::make_shared( - ov::OutputVector{body_cond_out, state_updated, attn_out_unsq}, - ov::ParameterVector{body_iter, body_state, body_q, body_k, body_v, body_g, body_beta}); - - auto trip_count = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, std::vector{T}); - auto exec_cond = ov::op::v0::Constant::create(ov::element::boolean, ov::Shape{1}, std::vector{true}); - - auto loop = std::make_shared(trip_count, exec_cond); - loop->set_function(body); - loop->set_special_body_ports(ov::op::v5::Loop::SpecialBodyPorts{0, 0}); - - // Carried state: feeds back from body output 1 to body_state param - loop->set_merged_input(body_state, state_m, state_updated); - // Invariant inputs: passed through unchanged each iteration - loop->set_invariant_input(body_q, q_m); - loop->set_invariant_input(body_k, k_m); - loop->set_invariant_input(body_v, v_m); - loop->set_invariant_input(body_g, g_m); - loop->set_invariant_input(body_beta, beta_m); - - // Loop outputs: - // 1) Final state (last iteration value of state_updated) - auto final_state_out = loop->get_iter_value(state_updated, -1); // [B*H_v, S_v, S_v] - // 2) Concatenated attention outputs across all iterations along axis 1 - auto attn_concat_out = loop->get_concatenated_slices(attn_out_unsq, 0, 1, 1, -1, 1); // [B*H_v, T, S_v] - - // --- Pack outputs to match ggml layout --- - // ggml output ne = {S_v*H, T*B + S_v*B, 1, 1} -> OV [1, 1, T*B+S_v*B, S_v*H_v] - // attn: [B, T, H_v, S_v] row-major, state: [B, H_v, S_v, S_v] row-major - - // attn: [B*H_v, T, S_v] -> [B, H_v, T, S_v] -> transpose to [B, T, H_v, S_v] -> flatten - auto attn_4d_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{B, H_v, T, S_v}); - auto attn_4d = std::make_shared(attn_concat_out, attn_4d_shape, false); - auto attn_perm = std::make_shared(attn_4d, perm_0213); // [B, T, H_v, S_v] - - auto flat_shape_1d = ov::op::v0::Constant::create(ov::element::i64, {1}, std::vector{-1}); - auto attn_1d = std::make_shared(attn_perm, flat_shape_1d, false); - - // state: [B*H_v, S_v, S_v] -> [B, H_v, S_v, S_v] -> flatten - auto state_4d_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{B, H_v, S_v, S_v}); - auto state_4d = std::make_shared(final_state_out, state_4d_shape, false); - if (context.get_output_names().size() == 2) { - // Match the fused translator's direct attention/state contract. - return rename_outputs_with_suffix({attn_perm, state_4d}, context.get_name()); - } - - auto state_1d = std::make_shared(state_4d, flat_shape_1d, false); - - // Concat [attn | state] and reshape to final output - auto packed = std::make_shared(ov::OutputVector{attn_1d, state_1d}, 0); - auto out_shape = - ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{1, 1, T * B + S_v * B, S_v * H_v}); - auto res = std::make_shared(packed, out_shape, false); - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/gated_delta_net.hpp b/ggml/src/ggml-openvino/openvino/op/gated_delta_net.hpp deleted file mode 100644 index 20a4cfdfe743..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/gated_delta_net.hpp +++ /dev/null @@ -1,65 +0,0 @@ -#pragma once - -#include "openvino/op/op.hpp" - -namespace ov::op::internal { -/// \note GatedDeltaNet op class is under development and subject to change -/// -/// \brief Operator performing Gated Delta Net computation -/// \ingroup ov_ops_cpp_api -class OPENVINO_API GatedDeltaNet : public ov::op::Op { -public: - OPENVINO_OP("GatedDeltaNet") - - GatedDeltaNet() = default; - /// \brief Constructs a GatedDeltaNet operation. - /// - /// \param query Query tensor input. - /// \param key Key tensor input. - /// \param value Value tensor input. - /// \param recurrent_state Initial recurrent state tensor. - /// \param gate Gate tensor controlling state decay/update. - /// \param beta Beta tensor scaling the delta update. - /// \param fuse_qk_l2norm Enables fusing q/k L2-normalization into this op. - /// \param q_l2_norm_eps Epsilon used for query L2-normalization when fusion is enabled. - /// \param k_l2_norm_eps Epsilon used for key L2-normalization when fusion is enabled. - GatedDeltaNet(const Output& query, - const Output& key, - const Output& value, - const Output& recurrent_state, - const Output& gate, - const Output& beta, - const bool fuse_qk_l2norm = false, - const float q_l2_norm_eps = 1e-6F, - const float k_l2_norm_eps = 1e-6F); - - /// \brief Constructs a GatedDeltaNet operation from input vector. - /// - /// \param args Input tensor vector in order: query, key, value, recurrent_state, gate, beta. - /// \param fuse_qk_l2norm Enables fusing q/k L2-normalization into this op. - /// \param q_l2_norm_eps Epsilon used for query L2-normalization when fusion is enabled. - /// \param k_l2_norm_eps Epsilon used for key L2-normalization when fusion is enabled. - GatedDeltaNet(const ov::OutputVector& args, - const bool fuse_qk_l2norm = false, - const float q_l2_norm_eps = 1e-6F, - const float k_l2_norm_eps = 1e-6F); - void validate_and_infer_types() override; - bool visit_attributes(AttributeVisitor& visitor) override; - std::shared_ptr clone_with_new_inputs(const ov::OutputVector& new_args) const override; - bool get_fuse_qk_l2norm() const { - return m_fuse_qk_l2norm; - } - float get_q_l2_norm_eps() const { - return m_q_l2_norm_eps; - } - float get_k_l2_norm_eps() const { - return m_k_l2_norm_eps; - } - -private: - bool m_fuse_qk_l2norm = false; - float m_q_l2_norm_eps = 1e-6F; - float m_k_l2_norm_eps = 1e-6F; -}; - -} // namespace ov::op::internal diff --git a/ggml/src/ggml-openvino/openvino/op/gather_matmul.hpp b/ggml/src/ggml-openvino/openvino/op/gather_matmul.hpp deleted file mode 100644 index 39bd744b0c86..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/gather_matmul.hpp +++ /dev/null @@ -1,43 +0,0 @@ -// Copyright (C) 2018-2026 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// -// Local mirror of OpenVINO's internal ov::op::internal::GatherMatmul op. -// -// The op class body (validate_and_infer_types / clone_with_new_inputs) is -// provided by the linked libopenvino.so; only the declaration is needed here so -// the backend can construct the node directly (same approach as GatedDeltaNet). -// The class layout must stay in sync with -// openvino/src/common/transformations/include/ov_ops/gather_matmul.hpp -// -// \note GatherMatmul op class is under development and subject to change. - -#pragma once - -#include "openvino/op/op.hpp" - -namespace ov::op::internal { - -class OPENVINO_API GatherMatmul : public ov::op::Op { -public: - OPENVINO_OP("GatherMatmul") - - GatherMatmul() = default; - - GatherMatmul(const ov::Output& A, - const ov::Output& B, - const ov::Output& indices, - const ov::Output& bias); - - GatherMatmul(const ov::Output& A, const ov::Output& B, const ov::Output& indices); - - std::shared_ptr clone_with_new_inputs(const ov::OutputVector& new_args) const override; - - void validate_and_infer_types() override; - -private: - // the weights matrix B is expected to have the transposed form [group, N, K] - static constexpr bool transp_a = false; - static constexpr bool transp_b = true; -}; - -} // namespace ov::op::internal diff --git a/ggml/src/ggml-openvino/openvino/op/get_rows.cpp b/ggml/src/ggml-openvino/openvino/op/get_rows.cpp deleted file mode 100644 index 200821d97489..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/get_rows.cpp +++ /dev/null @@ -1,178 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -OutputVector translate_get_rows(const NodeContext & context) { - num_inputs_check(context, 2, 2); - - Output res; - Output data; - int64_t row_offset = 0; - if (context.get_view_input_size(0) > 0 && context.get_input(0).get_partial_shape().rank() == 2) { - // A row range view over a 2D weight Constant folds into the gather indices, - // which keeps the dequantization subgraph intact for the plugins. - const auto view_shape = context.get_view_input_ggml_shape(0, 0); - const auto view_stride = context.get_view_input_stride(0, 0); - const size_t view_offset = context.get_view_input_offset(0, 0); - const size_t row_bytes = view_stride[2]; - data = context.get_input(0); - FRONT_END_OP_CONVERSION_CHECK(row_bytes > 0 && view_offset % row_bytes == 0 && - data.get_partial_shape()[1].compatible(view_shape[3]), - "GET_ROWS: view over a weight must be a row range"); - row_offset = static_cast(view_offset / row_bytes); - } else { - data = process_view_input_new(context, 0); - } - - auto op_case = context.get_op_case(); - if (op_case == 3 || op_case == 4) { - return {data}; - } - - ov::Output indices; - if ((op_case == 1 || op_case == 2) && context.has_input("s_copy_active_slot_len")) { - // Recurrent state reorder (inp->s_copy): slice the active (op_case 1) or extra (op_case 2) - // segment from the s_copy index list at runtime, instead of baking the static view offset, - // so the cached IR works for any number of active sequences. - auto s_copy = context.get_input(1); - auto len = context.get_input("s_copy_active_slot_len"); - auto step = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {3}); - if (op_case == 1) { - auto begin = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); - indices = std::make_shared(s_copy, begin, len, step, axis); - } else { - auto end = ov::op::v0::Constant::create(ov::element::i64, {1}, {INT_MAX}); - indices = std::make_shared(s_copy, len, end, step, axis); - } - } else { - indices = process_view_input_new(context, 1); - } - - // data[1,b,x,y] ind[1,1,b,x'] test-backend-ops case - // data[x,y] ind[1,1,1,x'] normal case - // Squeeze the leading dims down to [b,x']. Stateful models drop one rank, so a hardcoded - // {0,1} would also strip the batch dim whenever b == 1 (every decode step). - const auto indices_rank = indices.get_partial_shape().rank(); - FRONT_END_OP_CONVERSION_CHECK(indices_rank.is_static(), "Expected static rank for GET_ROWS indices"); - std::vector indices_squeeze_axes; - for (int64_t i = 0; i + 2 < indices_rank.get_length(); ++i) { - indices_squeeze_axes.push_back(i); - } - if (!indices_squeeze_axes.empty()) { - indices = std::make_shared( - indices, ov::op::v0::Constant::create(ov::element::i64, {indices_squeeze_axes.size()}, - indices_squeeze_axes)); - } - if (row_offset != 0) { - indices = std::make_shared( - indices, ov::op::v0::Constant::create(indices.get_element_type(), {}, {row_offset})); - } - if (data.get_partial_shape().rank() == 4) { - if (!(data.get_partial_shape()[1].is_dynamic()) && data.get_partial_shape()[1].get_length() == 1) { - // Work-around for a bug in ov cpu plugin for test-backend-ops - data = std::make_shared(data, - ov::op::v0::Constant::create(ov::element::i64, {2}, {0, 1})); - auto axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {0}); - res = std::make_shared(data, indices, axis); - } else { - auto axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {1}); - data = - std::make_shared(data, ov::op::v0::Constant::create(ov::element::i64, {1}, {0})); - // data: [batch, rows, ...], indices: [batch, n] - this is a batched gather - // (batch_dims=1) along the rows axis. The data and indices batch dims are - // logically equal (both == n_tokens) but reach this node through independent - // reshapes, so the GPU plugin's gather shape inference cannot prove - // data.shape[0] == indices.shape[0] and rejects the node. We must tie both - // batch dims to the SAME value, and crucially that value must stay DYNAMIC. - const auto data_ps = data.get_partial_shape(); - const auto idx_ps = indices.get_partial_shape(); - const bool data_batch_static = data_ps.rank().is_static() && data_ps[0].is_static(); - const bool idx_batch_dynamic = idx_ps.rank().is_dynamic() || idx_ps[0].is_dynamic(); - - if (data_batch_static && idx_batch_dynamic) { - // MoE per-expert-scale path: `data` is a statically-tiled REPEAT - // (ggml_repeat_4d(scale, 1, n_expert, n_tokens, 1)) whose batch dim is a - // compile-time-constant n_tokens, and every batch slice is IDENTICAL (it was - // tiled from a single [1, n_expert, 1] scale). `indices` (selected_experts) - // carries the genuinely dynamic token dim. Broadcasting indices up to the - // static data batch (the naive fix) would freeze the token dim to the - // captured prefill length, and that static value then flows through the - // gather into the residual stream, making every following decoder layer - // static -> triggers the GPU in-place-concat KV-cache corruption (only - // layer 0 stays dynamic). A static->dynamic Broadcast cannot expand, so - // instead collapse the redundant data batch to 1 and broadcast 1->dynamic to - // match the indices batch. Mathematically identical (the slices are equal), - // and the whole graph stays dynamic. - auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); - auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - auto axis0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); - auto data_b1 = std::make_shared(data, zero, one, one, axis0); // [1, rows, ...] - - auto idx_shape = std::make_shared(indices, ov::element::i64); - auto idx_batch = get_dimensions(idx_shape, {0}); // [batch] (dynamic) - auto data_b1_shape = std::make_shared(data_b1, ov::element::i64); - const auto rank = data_ps.rank().get_length(); - std::vector rest_axes; - for (int a = 1; a < rank; ++a) { - rest_axes.push_back(a); - } - auto data_rest = get_dimensions(data_b1_shape, rest_axes); // [rows, ...] - auto data_target = std::make_shared(ov::OutputVector{idx_batch, data_rest}, 0); - data = - std::make_shared(data_b1, data_target, ov::op::BroadcastType::BIDIRECTIONAL); - res = std::make_shared(data, indices, axis, 1); - } else { - // General case: tie the indices batch to the data batch (the data batch is - // already dynamic, e.g. the routing-weights gather whose data comes from the - // activations). Broadcast indices to [data_batch, indices_n]. - auto data_shape = std::make_shared(data, ov::element::i64); - auto data_batch = get_dimensions(data_shape, {0}); // [batch] - auto idx_shape = std::make_shared(indices, ov::element::i64); - auto idx_n = get_dimensions(idx_shape, {1}); // [n] - auto idx_target = std::make_shared(ov::OutputVector{data_batch, idx_n}, 0); - indices = std::make_shared(indices, idx_target, - ov::op::BroadcastType::BIDIRECTIONAL); - res = std::make_shared(data, indices, axis, 1); - } - } - } else if (context.is_stateful() && data.get_partial_shape().rank() == 3) { - auto axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {1}); - res = std::make_shared(data, indices, axis, 1); - } else { - auto axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {0}); - res = std::make_shared(data, indices, axis); - } - - if (res.get_element_type() != context.get_output_type()) { - res = std::make_shared(res, context.get_output_type()); - } - if (!(context.is_stateful())) { - res = std::make_shared(res, ov::op::v0::Constant::create(ov::element::i64, {1}, {0})); - } - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/glu_geglu.cpp b/ggml/src/ggml-openvino/openvino/op/glu_geglu.cpp deleted file mode 100644 index a54870d9d74f..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/glu_geglu.cpp +++ /dev/null @@ -1,75 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -OutputVector translate_glu_geglu(const NodeContext & context) { - num_inputs_check(context, 1, 2); - - ov::Output src0; - ov::Output src1; - if (context.get_input_size() == 2) { - // Inputs may be VIEW slices of a combined gate_up tensor (MoE experts): - // resolve them so each half has its real sliced shape, not the base tensor. - src0 = process_view_input_new(context, 0); - src1 = process_view_input_new(context, 1); - } else { - // GGML splits along ne[0] (OV last axis) using floor division: nc = ne[0] / 2. - // Both halves are nc elements; if the dimension is odd, the last element is dropped. - // Use Slice instead of Split to handle odd dimensions correctly. - // Resolve a VIEW input (e.g. non-contiguous slice) to its real shape first. - auto combined = process_view_input_new(context, 0); - auto combined_shape = combined.get_partial_shape(); - int64_t last_dim_val = combined_shape[combined_shape.rank().get_length() - 1].get_length(); - int64_t nc = last_dim_val / 2; - - auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); - auto step = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - auto start0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); - auto stop0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {nc}); - auto start1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {nc}); - auto stop1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {2 * nc}); - - src0 = std::make_shared(combined, start0, stop0, step, axis); - src1 = std::make_shared(combined, start1, stop1, step, axis); - } - - int32_t * params = context.get_output_op_params(); - const int32_t swapped = params[1]; - if (swapped) { - std::swap(src0, src1); - } - - if (context.is_static()) { - // TODO: Temporary solution for NPU accuracy issue due to fp16 overflow - // To be removed once permanent solution is implemented - // Justification: - // For |x| > 5, GELU(x) ≈ max(x, 0) (behaves like ReLU) - // So Clamp(-10, 10) only affects values where GELU would return ≈ x anyway. - // The only loss: values > 10 get mapped to 10 instead of x. - // In practice, FFN intermediates rarely exceed 10 after GEGLU gating. - src0 = std::make_shared(src0, -10.0, 10.0); - } - auto gelu = std::make_shared(src0); - auto res = std::make_shared(gelu, src1); - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp b/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp deleted file mode 100644 index 385d75f5ffd6..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/glu_geglu_quick.cpp +++ /dev/null @@ -1,62 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -OutputVector translate_glu_geglu_quick(const NodeContext & context) { - num_inputs_check(context, 1, 2); - - ov::Output src0; - ov::Output src1; - if (context.get_input_size() == 2) { - src0 = process_view_input_new(context, 0); - src1 = process_view_input_new(context, 1); - } else { - // split along last axis, nc = ne[0] / 2 - auto combined = process_view_input_new(context, 0); - auto combined_shape = combined.get_partial_shape(); - int64_t last_dim_val = combined_shape[combined_shape.rank().get_length() - 1].get_length(); - int64_t nc = last_dim_val / 2; - - auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); - auto step = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - auto start0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); - auto stop0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {nc}); - auto start1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {nc}); - auto stop1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {2 * nc}); - - src0 = std::make_shared(combined, start0, stop0, step, axis); - src1 = std::make_shared(combined, start1, stop1, step, axis); - } - - int32_t * params = context.get_output_op_params(); - const int32_t swapped = params[1]; - if (swapped) { - std::swap(src0, src1); - } - - // GELU_QUICK(x) = x * sigmoid(1.702 * x) - // Create the constant in the same type as src0 to avoid f16/f32 mismatch. - auto input_type = src0.get_element_type(); - auto coef = ov::op::v0::Constant::create(input_type, ov::Shape{}, {1.702f}); - auto gated = std::make_shared(src0, coef); - auto res = std::make_shared(gated, src1); - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp b/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp deleted file mode 100644 index 7eea81d96e59..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/glu_swiglu.cpp +++ /dev/null @@ -1,119 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -static std::pair, ov::Output> get_glu_inputs(const NodeContext & context) { - num_inputs_check(context, 1, 2); - - ov::Output src0; - ov::Output src1; - if (context.get_input_size() == 2) { - // Inputs may be VIEW slices of a combined gate_up tensor (MoE experts): - // resolve them so each half has its real sliced shape, not the base tensor. - src0 = process_view_input_new(context, 0); - src1 = process_view_input_new(context, 1); - } else { - // GGML splits along ne[0] (OV last axis) using floor division: nc = ne[0] / 2. - // Both halves are nc elements; if the dimension is odd, the last element is dropped. - // Use Slice instead of Split to handle odd dimensions correctly. - // Resolve a VIEW input (e.g. non-contiguous slice) to its real shape first. - auto combined = process_view_input_new(context, 0); - auto combined_shape = combined.get_partial_shape(); - int64_t last_dim_val = combined_shape[combined_shape.rank().get_length() - 1].get_length(); - int64_t nc = last_dim_val / 2; - - auto axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); - auto step = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - auto start0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); - auto stop0 = ov::op::v0::Constant::create(ov::element::i64, {1}, {nc}); - auto start1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {nc}); - auto stop1 = ov::op::v0::Constant::create(ov::element::i64, {1}, {2 * nc}); - - src0 = std::make_shared(combined, start0, stop0, step, axis); - src1 = std::make_shared(combined, start1, stop1, step, axis); - } - - int32_t * params = context.get_output_op_params(); - const int32_t swapped = params[1]; - if (swapped) { - std::swap(src0, src1); - } - - return {src0, src1}; -} - -OutputVector translate_glu_swiglu(const NodeContext & context) { - auto [src0, src1] = get_glu_inputs(context); - - auto silu = std::make_shared(src0); - auto res = std::make_shared(silu, src1); - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -OutputVector translate_glu_swiglu_oai(const NodeContext & context) { - auto [src0, src1] = get_glu_inputs(context); - - const int32_t * params = context.get_output_op_params(); - const float alpha = reinterpret_cast(params)[2]; - const float limit = reinterpret_cast(params)[3]; - - auto gate = std::make_shared(src0, -std::numeric_limits::infinity(), limit); - auto alpha_const = ov::op::v0::Constant::create(ov::element::f32, {}, {alpha}); - auto out_glu = std::make_shared(gate, alpha_const); - - auto up = std::make_shared(src1, -limit, limit); - auto one = ov::op::v0::Constant::create(ov::element::f32, {}, {1.0f}); - auto up_plus_one = std::make_shared(up, one); - auto res = std::make_shared(out_glu, up_plus_one); - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -OutputVector translate_glu_swiglu_clamp(const NodeContext & context) { - auto [src0, src1] = get_glu_inputs(context); - - const int32_t * params = context.get_output_op_params(); - const float limit = reinterpret_cast(params)[3]; - - // Compute in f32: f16 Swish/Clamp rounding drifts past the 1e-7 test tolerance. - auto output_type = context.get_output_type(); - if (src0.get_element_type() != ov::element::f32) { - src0 = std::make_shared(src0, ov::element::f32); - } - if (src1.get_element_type() != ov::element::f32) { - src1 = std::make_shared(src1, ov::element::f32); - } - - auto gate = std::make_shared(src0, -std::numeric_limits::infinity(), limit); - auto silu = std::make_shared(gate); - auto up = std::make_shared(src1, -limit, limit); - ov::Output res = std::make_shared(silu, up); - if (res.get_element_type() != output_type) { - res = std::make_shared(res, output_type); - } - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/im2col.cpp b/ggml/src/ggml-openvino/openvino/op/im2col.cpp deleted file mode 100644 index 469d1de1f466..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/im2col.cpp +++ /dev/null @@ -1,239 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -OutputVector translate_im2col(const NodeContext & context) { - num_inputs_check(context, 2, 2); - const int32_t * params = context.get_output_op_params(); - int32_t s0 = params[0]; - int32_t s1 = params[1]; - int32_t p0 = params[2]; - int32_t p1 = params[3]; - int32_t d0 = params[4]; - int32_t d1 = params[5]; - bool is_2D = params[6] == 1; - ov::Output res; - - ov::Output image = context.get_input(1); - const ov::Shape kernel_shape = context.get_input(0).get_shape(); - - const size_t IC = is_2D ? kernel_shape[1] : kernel_shape[2]; - const size_t KH = is_2D ? kernel_shape[2] : 1; - const size_t KW = kernel_shape[3]; - - int32_t stride_w = s0; - int32_t stride_h = is_2D ? s1 : 1; - int32_t pad_w = p0; - int32_t pad_h = is_2D ? p1 : 0; - int32_t dil_w = d0; - int32_t dil_h = is_2D ? d1 : 1; - - if (!is_2D) { - // GGML input shape: [IW, IC, N, 1] - // OpenVINO input shape: [1, N, IC, IW] - // Reshape image to: [N, IC, 1, IW] - const ov::Shape image_shape = image.get_shape(); - const size_t N = image_shape[1]; - const size_t IW = image_shape[3]; - auto image_reshape_shape = ov::op::v0::Constant::create( - ov::element::i64, ov::Shape{4}, - std::vector{static_cast(N), static_cast(IC), 1, static_cast(IW)}); - image = std::make_shared(image, image_reshape_shape, false); - } - - const ov::Shape patch_sizes = {KH, KW}; - const ov::Strides strides = {static_cast(stride_h), static_cast(stride_w)}; - const ov::Shape rates = {static_cast(dil_h), static_cast(dil_w)}; - - auto pads_begin = - ov::op::v0::Constant::create(ov::element::i64, ov::Shape{4}, std::vector{0, 0, pad_h, pad_w}); - auto pads_end = - ov::op::v0::Constant::create(ov::element::i64, ov::Shape{4}, std::vector{0, 0, pad_h, pad_w}); - - auto pad = std::make_shared(image, pads_begin, pads_end, ov::op::PadMode::CONSTANT); - auto patches = - std::make_shared(pad, patch_sizes, strides, rates, ov::op::PadType::VALID); - - // [N, KH*KW*IC, OH, OW] → [N, OH, OW, KH*KW*IC] - auto perm1 = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{4}, std::vector{0, 2, 3, 1}); - auto t1 = std::make_shared(patches, perm1); - - // [N, OH, OW, KH*KW*IC] → [N, OH, OW, KH*KW, IC] - const ov::Shape out_shape = t1->get_output_shape(0); - const size_t N = out_shape[0]; - const size_t OH = out_shape[1]; - const size_t OW = out_shape[2]; - auto reshape1_shape = ov::op::v0::Constant::create( - ov::element::i64, ov::Shape{5}, - std::vector{static_cast(N), static_cast(OH), static_cast(OW), - static_cast(KH * KW), static_cast(IC)}); - auto r1 = std::make_shared(t1, reshape1_shape, false); - - // [N, OH, OW, KH*KW, IC] → [N, OH, OW, IC, KH*KW] - auto perm2 = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{5}, std::vector{0, 1, 2, 4, 3}); - auto t2 = std::make_shared(r1, perm2); - - // flatten back to [N, OH, OW, IC*KH*KW] - auto r2_shape = ov::op::v0::Constant::create( - ov::element::i64, ov::Shape{4}, - std::vector{static_cast(N), static_cast(OH), static_cast(OW), - static_cast(IC * KH * KW)}); - res = std::make_shared(t2, r2_shape, false); - - if (!is_2D) { - // [N, 1, OW, IC * KW] -> [1, N, OW, IC * KW] - auto final_reshape_shape = ov::op::v0::Constant::create( - ov::element::i64, ov::Shape{4}, - std::vector{1, static_cast(N), static_cast(OW), static_cast(IC * KW)}); - res = std::make_shared(res, final_reshape_shape, false); - } - - auto output_type = context.get_output_type(); - if (res.get_element_type() != output_type) { - res = std::make_shared(res, output_type); - } - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -OutputVector translate_im2col_3d(const NodeContext & context) { - num_inputs_check(context, 2, 2); - const int32_t * params = context.get_output_op_params(); - int32_t s0 = params[0]; - int32_t s1 = params[1]; - int32_t s2 = params[2]; - int32_t p0 = params[3]; - int32_t p1 = params[4]; - int32_t p2 = params[5]; - int32_t d0 = params[6]; - int32_t d1 = params[7]; - int32_t d2 = params[8]; - int32_t IC = params[9]; - - ov::Output image = process_view_input_new(context, 1); - const ov::Shape kernel_shape = context.get_input(0).get_shape(); - const ov::Shape image_shape = image.get_shape(); - const ov::Shape out_shape = context.get_output_shape().to_shape(); - - const size_t KD = kernel_shape[1]; - const size_t KH = kernel_shape[2]; - const size_t KW = kernel_shape[3]; - - const size_t N = image_shape[0] / static_cast(IC); - const size_t ID = image_shape[1]; - const size_t IH = image_shape[2]; - const size_t IW = image_shape[3]; - - const size_t OD = (ID + 2 * p2 - d2 * (KD - 1) - 1) / s2 + 1; - const size_t OH = (IH + 2 * p1 - d1 * (KH - 1) - 1) / s1 + 1; - const size_t OW = (IW + 2 * p0 - d0 * (KW - 1) - 1) / s0 + 1; - - if (N == 0 || OD == 0 || OH == 0 || OW == 0) { - auto output_type = context.get_output_type(); - ov::Output res = ov::op::v0::Constant::create( - output_type, ov::Shape{N * OD, OH, OW, static_cast(IC * KD * KH * KW)}, {}); - return rename_outputs_with_suffix({res}, context.get_name()); - } - - const size_t IH_pad = IH + 2 * p1; - const size_t IW_pad = IW + 2 * p0; - - auto image_5d_shape = ov::op::v0::Constant::create( - ov::element::i64, ov::Shape{5}, - std::vector{static_cast(N), static_cast(IC), static_cast(ID), - static_cast(IH), static_cast(IW)}); - auto image_5d = std::make_shared(image, image_5d_shape, false); - - auto pads_begin = ov::op::v0::Constant::create( - ov::element::i64, ov::Shape{5}, std::vector{0, 0, p2, p1, p0}); - auto pads_end = ov::op::v0::Constant::create( - ov::element::i64, ov::Shape{5}, std::vector{0, 0, p2, p1, p0}); - auto pad_3d = std::make_shared(image_5d, pads_begin, pads_end, ov::op::PadMode::CONSTANT); - - const ov::Shape patch_sizes = {KH, KW}; - const ov::Strides strides = {static_cast(s1), static_cast(s0)}; - const ov::Shape rates = {static_cast(d1), static_cast(d0)}; - - ov::OutputVector kd_slices; - kd_slices.reserve(KD); - - auto perm_nod = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{5}, {0, 2, 1, 3, 4}); - auto reshape_4d_shape = ov::op::v0::Constant::create( - ov::element::i64, ov::Shape{4}, - std::vector{static_cast(N * OD), static_cast(IC), - static_cast(IH_pad), static_cast(IW_pad)}); - auto perm1 = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{4}, {0, 2, 3, 1}); - auto r1_shape = ov::op::v0::Constant::create( - ov::element::i64, ov::Shape{5}, - std::vector{static_cast(N * OD), static_cast(OH), static_cast(OW), - static_cast(KH * KW), static_cast(IC)}); - auto perm2 = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{5}, {0, 1, 2, 4, 3}); - auto r2_shape = ov::op::v0::Constant::create( - ov::element::i64, ov::Shape{6}, - std::vector{static_cast(N * OD), static_cast(OH), static_cast(OW), - static_cast(IC), 1, static_cast(KH * KW)}); - auto step_c = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {static_cast(s2)}); - auto axes_c = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {2}); - - for (size_t ikd = 0; ikd < KD; ++ikd) { - auto start_c = ov::op::v0::Constant::create( - ov::element::i64, ov::Shape{1}, {static_cast(ikd * d2)}); - auto stop_c = ov::op::v0::Constant::create( - ov::element::i64, ov::Shape{1}, {static_cast(ikd * d2 + OD * s2)}); - auto depth_slice = std::make_shared(pad_3d, start_c, stop_c, step_c, axes_c); - auto depth_slice_trans = std::make_shared(depth_slice, perm_nod); - auto depth_slice_4d = std::make_shared(depth_slice_trans, reshape_4d_shape, false); - - auto patches = std::make_shared( - depth_slice_4d, patch_sizes, strides, rates, ov::op::PadType::VALID); - auto t1 = std::make_shared(patches, perm1); - auto r1 = std::make_shared(t1, r1_shape, false); - auto t2 = std::make_shared(r1, perm2); - auto r2 = std::make_shared(t2, r2_shape, false); - kd_slices.push_back(r2); - } - - ov::Output res; - if (KD == 1) { - res = kd_slices[0]; - } else { - res = std::make_shared(kd_slices, 4); - } - - auto final_shape = ov::op::v0::Constant::create( - ov::element::i64, ov::Shape{4}, - std::vector{static_cast(N * OD), static_cast(OH), static_cast(OW), - static_cast(IC * KD * KH * KW)}); - res = std::make_shared(res, final_shape, false); - - auto output_type = context.get_output_type(); - if (res.get_element_type() != output_type) { - res = std::make_shared(res, output_type); - } - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/l2_norm.cpp b/ggml/src/ggml-openvino/openvino/op/l2_norm.cpp deleted file mode 100644 index 6e0485da2e73..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/l2_norm.cpp +++ /dev/null @@ -1,61 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -OutputVector translate_l2_norm(const NodeContext & context) { - num_inputs_check(context, 1, 1); - - auto input_node = process_view_input_new(context, 0); - - if (context.get_op_case() == 1) { - // 92: [ 128, 16, 1, 2] VIEW q_conv-1 - // [ 6144, 1, 2, 1] 0: UNARY conv_output_silu-1 - // 93: [ 128, 16, 1, 2] L2_NORM q_conv_predelta-1 - // [ 128, 16, 1, 2] 0: VIEW q_conv-1 - auto output_shape = context.get_output_shape().to_shape(); - input_node = process_view_input(context, 0, output_shape[2] * output_shape[3], 3); - input_node = - std::make_shared(input_node, ov::op::v0::Constant::create(ov::element::i64, {1}, {0})); - - std::vector reshape_pattern = {0, 0, (int64_t) output_shape[2], (int64_t) output_shape[3]}; - input_node = std::make_shared( - input_node, ov::op::v0::Constant::create(ov::element::i64, {4}, reshape_pattern), true); - } - - auto squared = std::make_shared(input_node, input_node); - - auto sum_squared = std::make_shared( - squared, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1}), true); - - auto l2_norm = std::make_shared(sum_squared); - - float eps; - memcpy(&eps, context.get_output_op_params(), sizeof(float)); - - auto eps_const = ov::op::v0::Constant::create(ov::element::f32, ov::Shape{1}, {eps}); - auto clamped_norm = std::make_shared(l2_norm, eps_const); - - auto res = std::make_shared(input_node, clamped_norm); - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/mean.cpp b/ggml/src/ggml-openvino/openvino/op/mean.cpp deleted file mode 100644 index 1e9b10bbd9f4..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/mean.cpp +++ /dev/null @@ -1,27 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -OutputVector translate_mean(const NodeContext & context) { - num_inputs_check(context, 1, 1); - - auto input = process_view_input_new(context, 0); - auto axis = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1}); - auto res = std::make_shared(input, axis, true); - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/moe_compressed.hpp b/ggml/src/ggml-openvino/openvino/op/moe_compressed.hpp deleted file mode 100644 index 07e94c690152..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/moe_compressed.hpp +++ /dev/null @@ -1,90 +0,0 @@ -// Copyright (C) 2018-2026 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// -// Local mirror of OpenVINO's internal ov::op::internal::MOE and MOECompressed ops. -// -// The class bodies are provided by the linked libopenvino.so; only the declarations are -// needed here so the backend can construct the node directly (same approach as -// GatherMatmul and GatedDeltaNet). The class layout must stay in sync with -// openvino/src/core/dev_api/openvino/op/moe.hpp -// openvino/src/common/transformations/include/ov_ops/moe_compressed.hpp -// -// \note MOE op classes are under development and subject to change. - -#pragma once - -#include - -#include "openvino/core/type/element_type.hpp" -#include "openvino/op/op.hpp" - -namespace ov::op::internal { - -class OPENVINO_API MOE : public ov::op::Op { -public: - OPENVINO_OP("MOE") - - MOE() = default; - - MOE(const OutputVector & args) : Op(args) {} - - enum class Expert_type { GEMM2_BIAS_SWIGLU_CLAMP, GEMM3_SWIGLU }; - - enum class Activation_type { SWIGLU, GEGLU_TANH, GEGLU_ERF }; - - struct Config { - Expert_type expert_type{ Expert_type::GEMM2_BIAS_SWIGLU_CLAMP }; - float expert_alpha{ 0.0f }; - float expert_beta{ 1.0f }; - size_t gate_idx{ 0 }; - Activation_type activation_type{ Activation_type::SWIGLU }; - }; - - MOE(const OutputVector & args, const Config & config); - - const Config & get_config() const; - void set_config(const Config & config); - - bool visit_attributes(AttributeVisitor & visitor) override; - void validate_and_infer_types() override; - std::shared_ptr clone_with_new_inputs(const OutputVector & new_args) const override; - -private: - Config m_config; -}; - -class OPENVINO_API MOECompressed : public MOE { -public: - OPENVINO_OP("MOECompressed", "", ov::op::internal::MOE) - - MOECompressed() = default; - - struct Config : public MOE::Config { - size_t hidden_size = 0; - size_t inter_size = 0; - size_t num_expert = 0; - size_t num_shared_expert = 0; - size_t top_k = 0; - // numeric_limits::max() means per_channel compression (single group) - size_t group_size = 0; - bool has_batch_dim = false; - bool has_zp = false; - ov::element::Type out_type = ov::element::dynamic; - std::optional scale_factor; - }; - - MOECompressed(const OutputVector & args, const Config & config); - - const Config & get_config() const { return m_config; } - - void set_scale_factor(float scale_factor) { m_config.scale_factor = scale_factor; } - - bool visit_attributes(AttributeVisitor & visitor) override; - void validate_and_infer_types() override; - std::shared_ptr clone_with_new_inputs(const OutputVector & new_args) const override; - -protected: - Config m_config; -}; - -} // namespace ov::op::internal diff --git a/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp b/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp deleted file mode 100644 index 52c3f5edad83..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/mul_mat_id.cpp +++ /dev/null @@ -1,254 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" -#include "gather_matmul.hpp" -#include "ggml-openvino/ggml-openvino-extra.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -namespace { - -std::shared_ptr const_i64(const std::vector & values) { - return ov::op::v0::Constant::create(ov::element::i64, ov::Shape{values.size()}, values); -} - -ov::Output slice_axis(const ov::Output & input, int64_t axis, int64_t begin, int64_t end) { - return std::make_shared(input, const_i64({begin}), const_i64({end}), const_i64({1}), - const_i64({axis})); -} - -// GGML tensors are rank 4, but stateful models drop the leading size-1 batch dim, so -// activations and ids arrive one rank lower. Pick the trailing dims by actual rank. -std::vector trailing_dims(const ov::Output & input, int count) { - const auto rank = input.get_partial_shape().rank(); - FRONT_END_OP_CONVERSION_CHECK(rank.is_static(), "Expected static rank for MUL_MAT_ID input"); - const int rank_len = static_cast(rank.get_length()); - FRONT_END_OP_CONVERSION_CHECK(rank_len >= count, "MUL_MAT_ID input rank is too low"); - - std::vector dims; - for (int i = rank_len - count; i < rank_len; ++i) { - dims.push_back(i); - } - return dims; -} - -ov::Output static_shape_dims_or_shapeof(const ov::Output & input, - const std::vector & dims) { - const auto & partial_shape = input.get_partial_shape(); - if (partial_shape.is_static()) { - std::vector values; - values.reserve(dims.size()); - for (const int64_t dim : dims) { - values.push_back(partial_shape[dim].get_length()); - } - return const_i64(values); - } - - auto shape = std::make_shared(input, ov::element::i64); - return get_dimensions(shape, dims); -} - -ov::Output translate_mul_mat_id_mxfp4_packed(const NodeContext & context, - ov::Output expert_weights, - ov::Output activations, - ov::Output ids) { - auto packed_shape = expert_weights.get_partial_shape().to_shape(); - FRONT_END_OP_CONVERSION_CHECK(packed_shape.size() == 5 && packed_shape[4] == 17, - "Expected packed MXFP4 expert weights with shape [1, n_expert, m, k_blocks, 17]"); - - const int64_t n_expert = static_cast(packed_shape[1]); - const int64_t rows = static_cast(packed_shape[2]); - const int64_t k_blocks = static_cast(packed_shape[3]); - const int64_t qk = 32; - const int64_t cols = k_blocks * qk; - - auto packed_shape_4d = const_i64({n_expert, rows, k_blocks, 17}); - expert_weights = std::make_shared(expert_weights, packed_shape_4d, false); - - auto activations_shape_4d = std::make_shared(activations, ov::element::i64); - auto ids_shape_4d = std::make_shared(ids, ov::element::i64); - auto activations_shape_3d = get_dimensions(activations_shape_4d, {1, 2, 3}); - auto ids_shape_2d = get_dimensions(ids_shape_4d, {2, 3}); - - activations = std::make_shared(activations, activations_shape_3d, false); - ids = std::make_shared(ids, ids_shape_2d, false); - if (ids.get_element_type() != ov::element::i32 && ids.get_element_type() != ov::element::i64) { - ids = std::make_shared(ids, ov::element::i32); - } - - auto gather_axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {0}); - - static const std::vector f4e2m1_lut = {0.0f, 0.5f, 1.0f, 1.5f, 2.0f, 3.0f, 4.0f, 6.0f, - -0.0f, -0.5f, -1.0f, -1.5f, -2.0f, -3.0f, -4.0f, -6.0f}; - std::vector e8m0_lut(256); - for (size_t i = 0; i < e8m0_lut.size(); ++i) { - uint32_t bits = static_cast(i) << 23; - memcpy(&e8m0_lut[i], &bits, sizeof(float)); - } - e8m0_lut[0] = std::numeric_limits::min() / 2.0f; - e8m0_lut[255] = std::numeric_limits::quiet_NaN(); - - auto f4_lut = ov::op::v0::Constant::create(ov::element::f32, ov::Shape{f4e2m1_lut.size()}, f4e2m1_lut); - auto scale_lut = ov::op::v0::Constant::create(ov::element::f32, ov::Shape{e8m0_lut.size()}, e8m0_lut); - - auto selected_packed_weights = std::make_shared(expert_weights, ids, gather_axis); - auto scale_byte = slice_axis(selected_packed_weights, 4, 0, 1); - auto qs = slice_axis(selected_packed_weights, 4, 1, 17); - auto low = std::make_shared( - qs, ov::op::v0::Constant::create(ov::element::u8, ov::Shape{}, {0x0F}), ov::op::AutoBroadcastType::NUMPY); - auto high_shift = std::make_shared( - qs, ov::op::v0::Constant::create(ov::element::u8, ov::Shape{}, {4}), ov::op::AutoBroadcastType::NUMPY); - auto nibbles = std::make_shared(ov::OutputVector{low, high_shift}, 4); - auto nibble_indices = std::make_shared(nibbles, ov::element::i32); - auto weights_f32 = std::make_shared(f4_lut, nibble_indices, gather_axis); - - auto scale_indices = std::make_shared(scale_byte, ov::element::i32); - auto scales_f32 = std::make_shared(scale_lut, scale_indices, gather_axis); - ov::Output selected_weights = std::make_shared(weights_f32, scales_f32, - ov::op::AutoBroadcastType::NUMPY); - - auto ids_shape = std::make_shared(ids, ov::element::i64); - auto selected_weights_target_dims = std::make_shared( - ov::OutputVector{get_dimensions(ids_shape, {0, 1}), const_i64({rows, cols})}, 0); - selected_weights = std::make_shared(selected_weights, selected_weights_target_dims, false); - - auto activations_shape = std::make_shared(activations, ov::element::i64); - ov::Output acts_target_dims = std::make_shared( - ov::OutputVector{ - get_dimensions(activations_shape, {0}), - get_dimensions(ids_shape, {1}), - get_dimensions(activations_shape, {2}), - }, - 0); - ov::Output acts_broadcasted = - std::make_shared(activations, acts_target_dims, ov::op::BroadcastType::BIDIRECTIONAL); - - auto activations_expanded = std::make_shared(acts_broadcasted, const_i64({2})); - ov::Output result = - std::make_shared(activations_expanded, selected_weights, false, true); - - auto batch_dim = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - auto row_dim = ov::op::v0::Constant::create(ov::element::i64, {1}, {rows}); - auto result_target_dims = std::make_shared( - ov::OutputVector{batch_dim, get_dimensions(ids_shape, {0, 1}), row_dim}, 0); - result = std::make_shared(result, result_target_dims, false); - - const auto output_type = context.get_output_type(); - if (result.get_element_type() != output_type) { - result = std::make_shared(result, output_type); - } - return result; -} - -} // namespace - -OutputVector translate_mul_mat_id(const NodeContext & context) { - num_inputs_check(context, 3, 3); - - auto expert_weights = process_view_input_new(context, 0); - auto activations = process_view_input_new(context, 1); - auto ids = process_view_input_new(context, 2); - - if (activations.get_partial_shape().rank() == 3) { - activations = std::make_shared(activations, const_i64({0})); - } - if (ids.get_partial_shape().rank() == 3) { - ids = std::make_shared(ids, const_i64({0})); - } - - if (expert_weights.get_element_type() == ov::element::u8 && expert_weights.get_partial_shape().rank().is_static() && - expert_weights.get_partial_shape().rank().get_length() == 5) { - return rename_outputs_with_suffix({translate_mul_mat_id_mxfp4_packed(context, expert_weights, activations, ids)}, - context.get_name()); - } - - // General (non-packed) path: dense F32/F16/BF16 weights, or the f16 dequantization chain for - // quantized MoE experts (see extract_quantized_weights / make_int4_weights / make_int8_weights in - // ggml-quants.cpp). Routed through ov::op::internal::GatherMatmul instead of a naive - // Gather+Broadcast+MatMul, so the selected expert's full weight matrix is never materialized per - // token. The CPU plugin's ConvertGatherMatmulToGatherMatmulCompressed pass (run during - // compile_model) fuses the dequantization chain feeding GatherMatmul's B input into a - // GatherMatmulCompressed node automatically, as long as MarkDequantization has marked the chain -- - // see translate_session.cpp's apply_transformations for the MarkDequantization registration. - // - // OpenVINO sees GGML tensors in reversed dimension order: - // activations: [1, n_tokens, n_used_or_1, k] - // ids: [1, 1, n_tokens, n_used] - // expert_weights is either [1, n_expert, m, k] (4D, e.g. non-quantized weights without a - // pre-built extra) or already [n_expert, m, k] (3D, weights routed through - // process_weight_tensor) -- GatherMatmul's B input expects the latter. - auto expert_weights_rank = expert_weights.get_partial_shape().rank(); - FRONT_END_OP_CONVERSION_CHECK(expert_weights_rank.is_static(), - "Expected static rank for MUL_MAT_ID expert weights"); - if (expert_weights_rank.get_length() == 4) { - auto expert_weights_shape_3d = static_shape_dims_or_shapeof(expert_weights, {1, 2, 3}); - expert_weights = std::make_shared(expert_weights, expert_weights_shape_3d, false); - } - - auto activations_shape_3d = static_shape_dims_or_shapeof(activations, trailing_dims(activations, 3)); - auto ids_shape_2d = static_shape_dims_or_shapeof(ids, trailing_dims(ids, 2)); - - activations = std::make_shared(activations, activations_shape_3d, false); - ids = std::make_shared(ids, ids_shape_2d, false); - - if (ids.get_element_type() != ov::element::i32 && ids.get_element_type() != ov::element::i64) { - ids = std::make_shared(ids, ov::element::i32); - } - - const auto output_type = context.get_output_type(); - const auto activations_type = ggml_openvino_is_gpu() ? ov::element::f16 : ov::element::f32; - if (activations.get_element_type() != activations_type) { - activations = std::make_shared(activations, activations_type); - } - - // GatherMatmul's A input is [n_used_or_1, n_tokens, k]; activations_3d is - // [n_tokens, n_used_or_1, k]. - auto activations_transpose_order = const_i64({1, 0, 2}); - ov::Output activations_for_gather = - std::make_shared(activations, activations_transpose_order); - - ov::Output result = std::make_shared(activations_for_gather, expert_weights, ids); - - // result is [n_used, n_tokens, m]; GGML expects [1, n_tokens, n_used, m], except on the - // stateful path where the leading batch dim is dropped. - auto result_transpose_order = const_i64({1, 0, 2}); - result = std::make_shared(result, result_transpose_order); - if (!context.is_stateful()) { - auto unsqueeze_axes = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); - result = std::make_shared(result, unsqueeze_axes); - } - - if (result.get_element_type() != output_type) { - result = std::make_shared(result, output_type); - } - - return rename_outputs_with_suffix({result}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/mulmat.cpp b/ggml/src/ggml-openvino/openvino/op/mulmat.cpp deleted file mode 100644 index 9d4315aa4abc..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/mulmat.cpp +++ /dev/null @@ -1,92 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -OutputVector translate_mulmat(const NodeContext & context) { - num_inputs_check(context, 2, 2); - - int op_case = context.get_op_case(); - - ov::Output B = process_view_input_new(context, 0); - ov::Output A = process_view_input_new(context, 1); - - if (A.get_element_type() != B.get_element_type()) { - B = std::make_shared(B, context.get_input_type(1)); - } - - auto B_shape = context.get_input_shape(0).to_shape(); - auto A_shape = context.get_input_shape(1).to_shape(); - int64_t A_batch = A_shape[1]; - int64_t B_batch = B_shape[1]; - - auto A_batch_larger = A_batch > B_batch; - auto batch_large = A_batch_larger ? A_batch : B_batch; - auto batch_small = A_batch_larger ? B_batch : A_batch; - - Output Z = A_batch_larger ? B : A; - auto Z_shape = A_batch_larger ? B_shape : A_shape; - int64_t factor = batch_large / batch_small; - if (factor > 1 && batch_small > 1) { - auto batch_large_node = ov::op::v0::Constant::create(ov::element::i64, {1}, std::vector{batch_large}); - auto batch_small_node = ov::op::v0::Constant::create(ov::element::i64, {1}, std::vector{batch_small}); - auto factor_node = ov::op::v0::Constant::create(ov::element::i64, {1}, std::vector{factor}); - - auto unsqueeze_axes = ov::op::v0::Constant::create(ov::element::i64, Shape{}, {2}); - auto Z_unsqueezed = std::make_shared(Z, unsqueeze_axes); - - auto broadcast_shape = ov::op::v0::Constant::create( - ov::element::i64, {5}, {(int64_t) 1, (int64_t) 1, factor, (int64_t) 1, (int64_t) 1}); - auto new_Z_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, - {(int64_t) 0, batch_large, (int64_t) -1, (int64_t) Z_shape[3]}); - if (op_case == 2) { - new_Z_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, - {(int64_t) 0, batch_large, (int64_t) Z_shape[2], (int64_t) -1}); - } - - auto Z_broadcasted = std::make_shared(Z_unsqueezed, broadcast_shape, - ov::op::BroadcastType::BIDIRECTIONAL); - Z = std::make_shared(Z_broadcasted, new_Z_shape, true); - } - if (A_batch_larger) { - B = Z; - } else { - A = Z; - } - - bool transpose_b = true; - ov::Output res = std::make_shared(A, B, false, transpose_b); - - const auto output_type = context.get_output_type(); - if (res.get_element_type() != output_type) { - res = std::make_shared(res, output_type); - } - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/norm.cpp b/ggml/src/ggml-openvino/openvino/op/norm.cpp deleted file mode 100644 index 8660c6521b70..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/norm.cpp +++ /dev/null @@ -1,31 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -OutputVector translate_norm(const NodeContext & context) { - num_inputs_check(context, 1, 1); - - auto input_node = process_view_input_new(context, 0); - float eps; - memcpy(&eps, context.get_output_op_params(), sizeof(float)); - - auto axes = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1}); - auto res = std::make_shared(input_node, axes, true, eps, ov::op::MVNEpsMode::INSIDE_SQRT); - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/pad.cpp b/ggml/src/ggml-openvino/openvino/op/pad.cpp deleted file mode 100644 index ae3d7be18eca..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/pad.cpp +++ /dev/null @@ -1,94 +0,0 @@ -#include "../op_table.h" -#include "../utils.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -namespace { - -ov::Output translate_circular_pad(ov::Output input, - const std::array & pads, - const ov::Shape & input_shape) { - ov::Output result = std::move(input); - - const std::array pads_begin = {pads[6], pads[4], pads[2], pads[0]}; - const std::array pads_end = {pads[7], pads[5], pads[3], pads[1]}; - - for (size_t axis = 0; axis < input_shape.size(); ++axis) { - const int64_t input_dim = static_cast(input_shape[axis]); - const int64_t pad_begin = pads_begin[axis]; - const int64_t pad_end = pads_end[axis]; - - if (pad_begin == 0 && pad_end == 0) { - continue; - } - - FRONT_END_CHECK_IMPLEMENTED(input_dim > 0, "Circular PAD requires static non-zero input dimensions"); - - std::vector indices(static_cast(input_dim + pad_begin + pad_end)); - for (int64_t index = 0; index < static_cast(indices.size()); ++index) { - int64_t wrapped = (index - pad_begin) % input_dim; - if (wrapped < 0) { - wrapped += input_dim; - } - indices[static_cast(index)] = wrapped; - } - - auto gather_indices = ov::op::v0::Constant::create(ov::element::i64, {indices.size()}, indices); - auto gather_axis = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {axis}); - result = std::make_shared(result, gather_indices, gather_axis); - } - - return result; -} - -} // namespace - -OutputVector translate_pad(const NodeContext & context) { - num_inputs_check(context, 1, 1); - - auto input = process_view_input_new(context, 0); - if (context.get_input_shape(0) == context.get_output_shape()) { - return rename_outputs_with_suffix({input}, context.get_name()); - } - - const int32_t * op_params = context.get_output_op_params(); - FRONT_END_CHECK_IMPLEMENTED(op_params != nullptr, "PAD requires output op params"); - - const std::array pads = {op_params[0], op_params[1], op_params[2], op_params[3], - op_params[4], op_params[5], op_params[6], op_params[7]}; - const bool circular = op_params[8] != 0; - - if (circular) { - auto res = translate_circular_pad(input, pads, context.get_input_shape(0).to_shape()); - return rename_outputs_with_suffix({res}, context.get_name()); - } - - const std::vector pads_begin = {pads[6], pads[4], pads[2], pads[0]}; - const std::vector pads_end = {pads[7], pads[5], pads[3], pads[1]}; - - auto pads_begin_node = ov::op::v0::Constant::create(ov::element::i64, {pads_begin.size()}, pads_begin); - auto pads_end_node = ov::op::v0::Constant::create(ov::element::i64, {pads_end.size()}, pads_end); - auto pad_value = ov::op::v0::Constant::create(context.get_input_type(0), ov::Shape{}, {0}); - auto res = - std::make_shared(input, pads_begin_node, pads_end_node, pad_value, ov::op::PadMode::CONSTANT); - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/permute.cpp b/ggml/src/ggml-openvino/openvino/op/permute.cpp deleted file mode 100644 index df4f038984c5..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/permute.cpp +++ /dev/null @@ -1,159 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -OutputVector translate_permute(const NodeContext & context) { - num_inputs_check(context, 1, 1); - - int op_case = context.get_op_case(); - FRONT_END_CHECK_IMPLEMENTED(op_case != 0, "Unsupported PERMUTE case"); - // op_case 1 is trivial permute - // op_case 2 is to permute Q. It has a preceding VIEW that reshapes Q to restore the sequqence dimension - // op_case 3 4 it to permute KV cache in the default layout - // op_case 5 6 is to permute V cache when `-fa off`, where v_trans=true - - ov::Output res; - ov::Output src; - if (op_case == 3 || op_case == 4 || op_case == 5 || op_case == 6) { - src = context.get_input(0); - } else { - src = process_view_input_new(context, 0); - } - std::vector perm_values{0, 2, 1, 3}; - const int32_t * op_params = context.get_output_op_params(); - if (op_params != nullptr) { - for (size_t input_axis = 0; input_axis < perm_values.size(); ++input_axis) { - const size_t output_axis = static_cast(op_params[input_axis]); - perm_values[perm_values.size() - 1 - output_axis] = - static_cast(perm_values.size() - 1 - input_axis); - } - } - if (op_case == 1 || context.is_stateful()) { - // The stateful path carries hidden-state tensors in a rank-3 layout (the - // leading batch dim is dropped, e.g. Gemma4's per-layer-embedding path). The - // perm above is rank-4; when the actual input is rank-3, drop the batch axis - // (perm[0], which is always the identity 0 here) and shift the rest down by 1 - // so the transpose order matches the input rank. - std::vector perm_used = perm_values; - const auto & src_ps = src.get_partial_shape(); - if (src_ps.rank().is_static() && src_ps.rank().get_length() == 3 && perm_values.size() == 4 && - perm_values[0] == 0) { - perm_used = {perm_values[1] - 1, perm_values[2] - 1, perm_values[3] - 1}; - } - auto perm = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{perm_used.size()}, perm_used); - res = std::make_shared(src, perm); - } else if (op_case == 2) { - auto perm = ov::op::v0::Constant::create(ov::element::i64, {4}, perm_values); - auto output_shape = context.get_output_shape().to_shape(); - auto n_heads = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[1]}); - auto head_size = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[3]}); - auto n_seq_active = context.has_input("n_seq_active") ? - context.get_input("n_seq_active") : - ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[0]}); - auto neg_one = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); - - auto new_shape = - std::make_shared(ov::OutputVector{n_seq_active, neg_one, n_heads, head_size}, 0); - - // // Alternative - // auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); - // auto new_shape = std::make_shared(ov::OutputVector{n_seq_active, neg_one, zero, zero}, 0); - - auto reshaped = std::make_shared(src, new_shape, true); - res = std::make_shared(reshaped, perm); - } else { - auto perm = ov::op::v0::Constant::create(ov::element::i64, {4}, perm_values); - auto cache_shape = src.get_partial_shape(); - auto output_shape = context.get_output_shape().to_shape(); - int64_t head_size = output_shape[3]; - int64_t n_heads = output_shape[1]; - if (op_case == 5 || op_case == 6) { - head_size = output_shape[2]; - n_heads = output_shape[1]; - } - int64_t ctx_per_seq = cache_shape[2].is_static() ? cache_shape[2].get_length() : -1; - int64_t n_seq = cache_shape[1].get_length(); - - Output attention_size; - if (!context.has_input("attention_size")) { - attention_size = ov::op::v0::Constant::create(ov::element::i64, {1}, {output_shape[2]}); - } else if (op_case == 3 || op_case == 5) { - attention_size = context.get_input("attention_size"); - } else { - attention_size = context.get_input("attention_size_swa"); - } - - Output seq_active_start; - Output seq_active_end; - if (context.has_input("seq_active_start")) { - seq_active_start = context.get_input("seq_active_start"); - seq_active_end = context.get_input("seq_active_end"); - } else { - int64_t n_seq_active = output_shape[0]; - size_t offset = *((size_t *) context.get_input_op_params(0)); - int64_t seq_active_start_val = offset / context.get_input_stride(0)[0]; - int64_t seq_active_end_val = seq_active_start_val + n_seq_active; - seq_active_start = ov::op::v0::Constant::create(ov::element::i64, {1}, {seq_active_start_val}); - seq_active_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {seq_active_end_val}); - } - - // 1. reshape to [n_seq, ctx_per_seq, n_heads, head_size] (for `-fa off` [n_seq, n_heads, head_size, ctx_per_seq]) - // 2. slice out the active sequences - // 3. slice out the attention part in each sequence - // 4. permute (skip for `-fa off`) - auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); - auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - - if (op_case == 3 || op_case == 4) { - auto src_reshaped = std::make_shared( - src, ov::op::v0::Constant::create(ov::element::i64, {4}, {n_seq, ctx_per_seq, n_heads, head_size}), - false); - ov::Output after_seq_slice; - if (n_seq == 1) { - after_seq_slice = src_reshaped; - } else { - after_seq_slice = - std::make_shared(src_reshaped, seq_active_start, seq_active_end, one, zero); - } - auto slice2 = std::make_shared(after_seq_slice, zero, attention_size, one, one); - res = std::make_shared(slice2, perm); - } else { - auto three = ov::op::v0::Constant::create(ov::element::i64, {1}, {3}); - auto src_reshaped = std::make_shared( - src, ov::op::v0::Constant::create(ov::element::i64, {4}, {n_seq, n_heads, head_size, ctx_per_seq}), - false); - ov::Output after_seq_slice; - if (n_seq == 1) { - after_seq_slice = src_reshaped; - } else { - after_seq_slice = - std::make_shared(src_reshaped, seq_active_start, seq_active_end, one, zero); - } - auto slice2 = std::make_shared(after_seq_slice, zero, attention_size, one, three); - res = slice2; - } - } - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/pool_2d.cpp b/ggml/src/ggml-openvino/openvino/op/pool_2d.cpp deleted file mode 100644 index fb6333175f02..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/pool_2d.cpp +++ /dev/null @@ -1,53 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -OutputVector translate_pool_2d(const NodeContext & context) { - num_inputs_check(context, 1, 1); - const int32_t * params = context.get_output_op_params(); - - const int k0 = params[1]; - const int k1 = params[2]; - const int s0 = params[3]; - const int s1 = params[4]; - const int p0 = params[5]; - const int p1 = params[6]; - - const int op_case = context.get_op_case(); - ov::Output input = context.get_input(0); - ov::Strides strides{static_cast(s1), static_cast(s0)}; - ov::Shape pads_begin{static_cast(p1), static_cast(p0)}; - ov::Shape pads_end{static_cast(p1), static_cast(p0)}; - ov::Shape kernel{static_cast(k1), static_cast(k0)}; - ov::Output res; - - switch (op_case) { - case 1: // GGML_OP_POOL_MAX - { - res = std::make_shared(input, strides, pads_begin, pads_end, kernel); - break; - } - case 2: // GGML_OP_POOL_AVG - { - res = std::make_shared(input, strides, pads_begin, pads_end, kernel, false); - break; - } - default: - break; - } - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/repeat.cpp b/ggml/src/ggml-openvino/openvino/op/repeat.cpp deleted file mode 100644 index b7aeaa24fa82..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/repeat.cpp +++ /dev/null @@ -1,47 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -// GGML_OP_REPEAT tiles src[0] to fill the destination shape. Every destination -// dimension is an integer multiple of the corresponding source dimension. -OutputVector translate_repeat(const NodeContext & context) { - num_inputs_check(context, 1, 2); - - auto input = process_view_input_new(context, 0); - - const auto input_shape = context.get_input_shape(0).to_shape(); - const auto output_shape = context.get_output_shape().to_shape(); - - std::vector repeats(4, 1); - for (size_t axis = 0; axis < 4; ++axis) { - const int64_t input_dim = input_shape[axis]; - const int64_t output_dim = output_shape[axis]; - - FRONT_END_OP_CONVERSION_CHECK(input_dim > 0 && output_dim > 0 && output_dim % input_dim == 0, - "REPEAT input shape ", input_shape, " cannot tile to match ", output_shape); - - repeats[axis] = output_dim / input_dim; - } - - auto repeats_node = ov::op::v0::Constant::create(ov::element::i64, {repeats.size()}, repeats); - ov::Output res = std::make_shared(input, repeats_node); - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/reshape.cpp b/ggml/src/ggml-openvino/openvino/op/reshape.cpp deleted file mode 100644 index 5ed27b1a726e..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/reshape.cpp +++ /dev/null @@ -1,86 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -OutputVector translate_reshape(const NodeContext & context) { - num_inputs_check(context, 1, 1); - if (context.get_input(0).get_partial_shape().is_static() && - context.get_input_shape(0) == context.get_output_shape()) { - return {context.get_input(0)}; - } - - const int op_case = context.get_op_case(); - const auto output_shape = context.get_output_shape().to_shape(); - std::vector shape(output_shape.begin(), output_shape.end()); - std::shared_ptr new_shape_node; - switch (op_case) { - case 0: - break; - case 1: - case 9: - shape[1] = -1; - if (context.is_stateful() && op_case == 1) { - shape.erase(shape.begin()); - } - break; - case 2: - case 3: - shape[2] = -1; - if (op_case == 3) { - shape[3] = 1; - } - break; - case 4: - return {context.get_input(0).get_node_shared_ptr()->input_value(0)}; - case 5: - case 7: - shape = {1, 1, -1, shape[3]}; - if (context.is_stateful() && op_case == 5) { - shape.erase(shape.begin()); - } - break; - case 6: - // Recurrent inputs keep the active sequence count separate from the token count. - if (context.has_input("s_copy_active_slot_len")) { - auto n_slot_active_len = context.get_input("s_copy_active_slot_len"); - auto emb_size = ov::op::v0::Constant::create(ov::element::i64, {1}, {shape[3]}); - auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - auto neg_one = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); - new_shape_node = - std::make_shared(ov::OutputVector{one, n_slot_active_len, neg_one, emb_size}, 0); - } else { - shape = {1, 1, -1, shape[3]}; - } - break; - case 8: - shape[0] = -1; - break; - default: - FRONT_END_OP_CONVERSION_CHECK(false, "Unsupported RESHAPE case: ", op_case); - } - if (!new_shape_node) { - new_shape_node = ov::op::v0::Constant::create(ov::element::i64, {shape.size()}, shape); - } - auto res = std::make_shared(context.get_input(0), new_shape_node, false); - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp b/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp deleted file mode 100644 index 980fa3ceeba9..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/rms_norm.cpp +++ /dev/null @@ -1,89 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -OutputVector translate_rms_norm(const NodeContext & context) { - num_inputs_check(context, 1, 1); - - auto op_case = context.get_op_case(); - - ov::Output input_node; - if (op_case == 3) { - // Flatten sequence and token dimensions to match the gate layout. - auto input_shape = context.get_input_shape(0).to_shape(); - input_node = std::make_shared( - context.get_input(0), - ov::op::v0::Constant::create( - ov::element::i64, {4}, std::vector{1, -1, (int64_t) input_shape[2], (int64_t) input_shape[3]}), - false); - } else if (op_case == 1) { - input_node = process_view_input_new(context, 0); - } else if (op_case == 2) { - auto ssm_state_size = context.get_ssm_state_size(); - // The GDN op packs [attn | new_state] along the row axis; the state occupies the last - // ssm_state_size * n_seqs rows. Slice it off (scaling by the active sequence count) to keep - // just the attention output. - ov::Output state_end; - if (context.has_input("s_copy_active_slot_len")) { - auto len = context.get_input("s_copy_active_slot_len"); - auto state_rows = std::make_shared( - ov::op::v0::Constant::create(ov::element::i64, {1}, {ssm_state_size}), len); - state_end = std::make_shared(state_rows); - } else { - state_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {-ssm_state_size}); - } - auto gdn_attn_output = std::make_shared( - context.get_input(0), ov::op::v0::Constant::create(ov::element::i64, {1}, {0}), state_end, - ov::op::v0::Constant::create(ov::element::i64, {1}, {1}), - ov::op::v0::Constant::create(ov::element::i64, {1}, {2})); - - auto input_shape = context.get_input_shape(0).to_shape(); - input_node = std::make_shared( - gdn_attn_output, - ov::op::v0::Constant::create( - ov::element::i64, {4}, std::vector{1, -1, (int64_t) input_shape[2], (int64_t) input_shape[3]}), - false); - - } else { - input_node = process_view_input_new(context, 0); - } - auto square = std::make_shared(input_node, input_node); - - auto mean = std::make_shared( - square, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1}), true); - - float eps; - memcpy(&eps, context.get_output_op_params(), sizeof(float)); - - auto rms = std::make_shared( - std::make_shared(mean, ov::op::v0::Constant::create(ov::element::f32, ov::Shape{1}, {eps}))); - - auto reciprocal = - std::make_shared(ov::op::v0::Constant::create(ov::element::f32, ov::Shape{1}, {1.0f}), rms); - - auto res = std::make_shared(input_node, reciprocal); - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/roll.cpp b/ggml/src/ggml-openvino/openvino/op/roll.cpp deleted file mode 100644 index e8d1b8e50b34..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/roll.cpp +++ /dev/null @@ -1,36 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -OutputVector translate_roll(const NodeContext & context) { - num_inputs_check(context, 1, 1); - const int32_t * params = context.get_output_op_params(); - - int64_t s0 = params[0]; - int64_t s1 = params[1]; - int64_t s2 = params[2]; - int64_t s3 = params[3]; - - auto input = context.get_input(0); - - auto shift = ov::op::v0::Constant::create( - ov::element::i64, ov::Shape{4}, std::vector{s3, s2, s1, s0}); - auto axes = ov::op::v0::Constant::create( - ov::element::i64, ov::Shape{4}, std::vector{0, 1, 2, 3}); - - auto roll = std::make_shared(input, shift, axes); - return rename_outputs_with_suffix({roll}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/rope.cpp b/ggml/src/ggml-openvino/openvino/op/rope.cpp deleted file mode 100644 index 2c7fe62569a4..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/rope.cpp +++ /dev/null @@ -1,245 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -OutputVector translate_rope(const NodeContext & context) { - num_inputs_check(context, 2, 3); - - int op_case = context.get_op_case(); - - ov::Output res; - - auto data_node = process_view_input_new(context, 0).get_node_shared_ptr(); - auto output_shape = context.get_output_shape().to_shape(); - int32_t * op_params = context.get_output_op_params(); - const int mode = op_case; - const int64_t head_dim = static_cast(output_shape[3]); - const int64_t configured_n_dims = static_cast(op_params[1]); - const int64_t n_dims = configured_n_dims == 0 ? head_dim : configured_n_dims; - const int64_t n_offs = static_cast(op_params[15]); - - constexpr int TYPE_NORMAL = 0; - constexpr int TYPE_NEOX = 1; - constexpr int TYPE_IMROPE = 2; - constexpr int TYPE_VISION = 3; - constexpr int TYPE_MROPE = 4; - - Output cos_theta_node; - Output sin_theta_node; - if (context.has_input("rope_cos")) { - cos_theta_node = context.get_input("rope_cos"); - sin_theta_node = context.get_input("rope_sin"); - } else { - std::string cache_key = "rope_sin_cos"; - for (int i = 0; i < 15; i++) { - cache_key += "_" + std::to_string(op_params[i]); - } - if (context.get_input_size() == 3) { - cache_key += "_ff_" + context.get_input_names()[2]; - } - if (context.has_input(cache_key + "_cos")) { - cos_theta_node = context.get_input(cache_key + "_cos"); - sin_theta_node = context.get_input(cache_key + "_sin"); - } else { - auto inp_pos = context.get_input(1).get_node_shared_ptr(); - std::shared_ptr rope_freqs_weight; - if (context.get_input_size() == 3) { - rope_freqs_weight = context.get_input(2).get_node_shared_ptr(); - } - auto sin_cos = make_sin_cos(op_params, inp_pos, rope_freqs_weight, mode, false, head_dim); - sin_theta_node = sin_cos.first; - cos_theta_node = sin_cos.second; - context.put_shared(cache_key + "_cos", cos_theta_node); - context.put_shared(cache_key + "_sin", sin_theta_node); - } - } - - auto output_type = context.get_output_type(); - if (data_node->get_element_type() != ov::element::f32) { - data_node = std::make_shared(data_node, ov::element::f32); - } - - const int64_t total_rope_dims = (mode == TYPE_VISION) ? (2 * n_dims) : n_dims; - FRONT_END_OP_CONVERSION_CHECK(n_offs >= 0 && (n_offs % 2 == 0), - "ROPE expects non-negative even n_offs"); - FRONT_END_OP_CONVERSION_CHECK(n_dims > 0 && total_rope_dims + n_offs <= head_dim && (n_dims % 2 == 0), - "ROPE expects even n_dims with total_rope_dims + n_offs <= head_dim"); - - // RoPEFusionFlux requires rank_equals(4) on x, t_cos and t_sin. The cos/sin - // tables are already built rank-4 ([1, S, 1, head_size/2]) for both modes. In - // stateful mode the data arrives rank-3 ([S, n_heads, head_size]), so lift it - // to rank-4 ([1, S, n_heads, head_size]) here. Stateful RoPE already produced - // rank-4 output, so downstream attention is unaffected. - if (context.is_stateful()) { - const int64_t batch = static_cast(output_shape[0]); - auto r4_shape = ov::op::v0::Constant::create( - ov::element::i64, {4}, - std::vector{batch, -1, (int64_t) output_shape[2], (int64_t) output_shape[3]}); - data_node = std::make_shared(data_node, r4_shape, false); - } - // For TYPE_NORMAL rope (both stateful and stateless) we emit the Flux-style - // interleaved pattern below so the GPU plugin's RoPEFusionFlux matcher folds it - // into ov::op::internal::RoPE. - if (mode == TYPE_NORMAL) { - auto axis_last = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); - auto step_one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - - const int64_t batch = static_cast(output_shape[0]); - const int64_t n_heads = static_cast(output_shape[2]); - const int64_t half = n_dims / 2; - auto rot_start = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_offs}); - auto rot_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_offs + n_dims}); - auto rot_data = std::make_shared(data_node, rot_start, rot_end, step_one, axis_last); - - auto neg_one_f = ov::op::v0::Constant::create(data_node->get_element_type(), ov::Shape{}, {-1.0f}); - - auto paired_shape = ov::op::v0::Constant::create( - ov::element::i64, {5}, std::vector{batch, -1, n_heads, half, 2}); - auto x_paired = std::make_shared(rot_data, paired_shape, false); - - auto split_axis = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {-1}); - auto data_split = std::make_shared(x_paired, split_axis, 2); - Output x0 = data_split->outputs()[0]; - Output x1 = data_split->outputs()[1]; - - auto x1_neg = std::make_shared(x1, neg_one_f); - auto x_rotated_paired = std::make_shared(ov::OutputVector{x1_neg, x0}, -1); - - auto flat_shape = - ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{batch, -1, n_heads, n_dims}); - auto x_rotated = - std::make_shared(x_rotated_paired, flat_shape, false); - - // Expand cos/sin from [..., n_dims/2] to [..., n_dims] by repeating each - // entry twice. Use special_zero on the final Reshape so the seq dim passes - // through dynamically. Final rank is 4 to satisfy the matcher's predicate. - auto expand_cos_sin = [&](const Output& cs) { - auto cs_unsq = std::make_shared( - cs, ov::op::v0::Constant::create(ov::element::i64, {1}, {-1})); - auto bcast_target = ov::op::v0::Constant::create( - ov::element::i64, {5}, std::vector{1, 1, 1, half, 2}); - auto bcast = std::make_shared( - cs_unsq, bcast_target, ov::op::BroadcastType::BIDIRECTIONAL); - auto flat = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{0, 0, 0, n_dims}); - return std::make_shared(bcast, flat, true); - }; - Output cos_full = expand_cos_sin(cos_theta_node); - Output sin_full = expand_cos_sin(sin_theta_node); - - auto y1 = std::make_shared(rot_data, cos_full); - auto y2 = std::make_shared(x_rotated, sin_full); - auto rotated = std::make_shared(y1, y2); - - ov::OutputVector concat_parts; - if (n_offs > 0) { - auto head_start = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); - auto head_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_offs}); - auto head = std::make_shared(data_node, head_start, head_end, step_one, axis_last); - concat_parts.push_back(head); - } - concat_parts.push_back(rotated); - if (n_offs + n_dims < head_dim) { - auto tail_start = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_offs + n_dims}); - auto tail_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {head_dim}); - auto tail = std::make_shared(data_node, tail_start, tail_end, step_one, axis_last); - concat_parts.push_back(tail); - } - if (concat_parts.size() == 1) { - res = rotated; - } else { - res = std::make_shared(concat_parts, -1); - } - } else if (mode == TYPE_NEOX || mode == TYPE_IMROPE || mode == TYPE_MROPE || mode == TYPE_VISION) { - const int64_t half = (mode == TYPE_VISION) ? n_dims : (n_dims / 2); - const int64_t rot_dims = 2 * half; - - if (mode != TYPE_NEOX) { - auto cos_sin_shape = std::make_shared(ov::element::i64, ov::Shape{4}, - std::vector{1, -1, 1, half}); - cos_theta_node = std::make_shared(cos_theta_node, cos_sin_shape, true); - sin_theta_node = std::make_shared(sin_theta_node, cos_sin_shape, true); - } - - auto axis_last = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); - auto step_one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - - Output rot_data = data_node; - if (n_offs > 0 || n_offs + rot_dims < head_dim) { - auto rot_start = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_offs}); - auto rot_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_offs + rot_dims}); - rot_data = std::make_shared(data_node, rot_start, rot_end, step_one, axis_last); - } - - auto neg_one_f = ov::op::v0::Constant::create(data_node->get_element_type(), ov::Shape{}, {-1.0f}); - - auto split_axis = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {3}); - auto split_lengths = ov::op::v0::Constant::create(ov::element::i64, {2}, {half, half}); - auto data_split = std::make_shared(rot_data, split_axis, split_lengths); - Output x1 = data_split->outputs()[0]; - Output x2 = data_split->outputs()[1]; - - auto x2_neg = std::make_shared(x2, neg_one_f); - auto x_rotate_half = std::make_shared(ov::OutputVector{x2_neg, x1}, -1); - - auto cos_full = std::make_shared(ov::OutputVector{cos_theta_node, cos_theta_node}, -1); - auto sin_full = std::make_shared(ov::OutputVector{sin_theta_node, sin_theta_node}, -1); - - auto y1 = std::make_shared(rot_data, cos_full); - auto y2 = std::make_shared(x_rotate_half, sin_full); - auto rotated = std::make_shared(y1, y2); - - ov::OutputVector concat_parts; - if (n_offs > 0) { - auto head_start = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); - auto head_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_offs}); - auto head = std::make_shared(data_node, head_start, head_end, step_one, axis_last); - concat_parts.push_back(head); - } - concat_parts.push_back(rotated); - if (n_offs + rot_dims < head_dim) { - auto tail_start = ov::op::v0::Constant::create(ov::element::i64, {1}, {n_offs + rot_dims}); - auto tail_end = ov::op::v0::Constant::create(ov::element::i64, {1}, {head_dim}); - auto tail = std::make_shared(data_node, tail_start, tail_end, step_one, axis_last); - concat_parts.push_back(tail); - } - if (concat_parts.size() == 1) { - res = rotated; - } else { - res = std::make_shared(concat_parts, -1); - } - } - - if (res.get_element_type() != output_type) { - res = std::make_shared(res, output_type); - } - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/scale.cpp b/ggml/src/ggml-openvino/openvino/op/scale.cpp deleted file mode 100644 index 2947ee03c97c..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/scale.cpp +++ /dev/null @@ -1,90 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -OutputVector translate_scale(const NodeContext & context) { - num_inputs_check(context, 1, 1); - - float scale; - float bias; - memcpy(&scale, (float *) context.get_output_op_params() + 0, sizeof(float)); - memcpy(&bias, (float *) context.get_output_op_params() + 1, sizeof(float)); - - auto scale_node = std::make_shared(ov::element::f32, ov::Shape{}, std::vector{scale}); - - if (context.get_op_case() == 2) { - return {context.get_input(0)}; - } - - if (context.get_op_case() == 1 && context.has_input("cache_rs_reset_len")) { - auto cache_rs_reset_idx = context.get_input("cache_rs_reset_idx"); - auto cache_rs_reset_len = context.get_input("cache_rs_reset_len"); - - auto cache_rs = context.get_input(0); - - auto cache_shape = std::make_shared(cache_rs, ov::element::i64); - auto n_slots_1d = std::make_shared( - cache_shape, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {2}), - ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {0})); - auto n_slots = std::make_shared(n_slots_1d); - - auto iota = std::make_shared( - ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {0}), n_slots, - ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {1}), ov::element::i64); - - auto idx_plus_len = std::make_shared(cache_rs_reset_idx, cache_rs_reset_len); - auto less_than_idx = std::make_shared(iota, cache_rs_reset_idx); - auto greater_equal_idx_plus_len = std::make_shared(iota, idx_plus_len); - auto keep_mask = std::make_shared(less_than_idx, greater_equal_idx_plus_len); - - auto keep_mask_f32 = std::make_shared(keep_mask, ov::element::f32); - auto keep_mask_reshape = std::make_shared( - keep_mask_f32, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {1})); - - auto cleared_cache_rs = std::make_shared(cache_rs, keep_mask_reshape); - return rename_outputs_with_suffix({cleared_cache_rs}, context.get_name()); - } - - auto scaled = std::make_shared(context.get_input(0), scale_node); - - std::shared_ptr res; - if (bias != 0.0f) { - auto bias_node = - std::make_shared(ov::element::f32, ov::Shape{}, std::vector{bias}); - res = std::make_shared(scaled, bias_node); - } else { - res = scaled; - } - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/set.cpp b/ggml/src/ggml-openvino/openvino/op/set.cpp deleted file mode 100644 index 9b18ccfebaa8..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/set.cpp +++ /dev/null @@ -1,76 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -// GGML SET writes src1 into a view of src0 and returns the updated tensor. -OutputVector translate_set(const NodeContext & context) { - num_inputs_check(context, 2, 2); - - auto dst = process_view_input_new(context, 0); - auto src = process_view_input_new(context, 1); - - src = std::make_shared(src, context.get_output_type()); - - const auto dst_stride = context.get_input_stride(0); - FRONT_END_OP_CONVERSION_CHECK(dst_stride.size() >= 4, "SET requires 4D destination strides"); - - const auto * op_params = reinterpret_cast(context.get_output_op_params()); - const size_t offset = static_cast(op_params[3]); - - const size_t elem_size = dst_stride.back(); - FRONT_END_OP_CONVERSION_CHECK(elem_size != 0 && offset % elem_size == 0, - "SET offset must be aligned to destination element size"); - - const int64_t offset_elems = static_cast(offset / elem_size); - - auto dst_flat = std::make_shared( - dst, - ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}), - false); - - auto src_flat = std::make_shared( - src, - ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}), - false); - - auto src_shape = std::make_shared(src_flat, ov::element::i64); - auto src_len = std::make_shared( - src_shape, - ov::op::v0::Constant::create(ov::element::i64, {1}, {0}), - false); - - auto start = ov::op::v0::Constant::create(ov::element::i64, {}, {offset_elems}); - auto stop = std::make_shared(start, src_len); - auto step = ov::op::v0::Constant::create(ov::element::i64, {}, {1}); - - auto indices = std::make_shared(start, stop, step, ov::element::i64); - auto axis = ov::op::v0::Constant::create(ov::element::i64, {}, {0}); - - auto updated_flat = std::make_shared(dst_flat, indices, src_flat, axis); - - auto dst_shape = std::make_shared(dst, ov::element::i64); - auto res = std::make_shared(updated_flat, dst_shape, false); - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/set_rows.cpp b/ggml/src/ggml-openvino/openvino/op/set_rows.cpp deleted file mode 100644 index 3b606c82accb..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/set_rows.cpp +++ /dev/null @@ -1,96 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -OutputVector translate_set_rows(const NodeContext & context) { - num_inputs_check(context, 3, 3); - - auto data = process_view_input_new(context, 0); - auto indices = process_view_input_new(context, 1); - auto dst = process_view_input_new(context, 2); - - data = std::make_shared(data, context.get_output_type()); - - const auto indices_shape = context.get_input_shape(1); - const bool multidim_indices = indices_shape.rank().is_static() && - indices_shape.rank().get_length() == 4 && - ((indices_shape[1].is_static() && indices_shape[1].get_length() > 1) || - (indices_shape[2].is_static() && indices_shape[2].get_length() > 1)); - - auto axes = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {2}); - - Output res; - if (context.is_stateful()) { - int concat_axis = 1; - int64_t dim2 = dst.get_partial_shape()[2].get_length(); - int64_t dim3 = dst.get_partial_shape()[3].get_length(); - data = std::make_shared( - data, ov::op::v0::Constant::create(ov::element::i64, {4}, {(int64_t) 1, (int64_t) -1, dim2, dim3}), false); - res = std::make_shared(OutputVector{dst, data}, concat_axis); - } else if (multidim_indices) { - auto updates_shape = std::make_shared(data, ov::element::i64); - - auto indices_rank3 = std::make_shared( - indices, ov::op::v0::Constant::create(ov::element::i64, {1}, {0})); - auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - auto indices_rank4_shape = std::make_shared(OutputVector{get_dimensions(updates_shape, {0, 1, 2}), one}, 0); - auto indices_rank4 = std::make_shared(indices_rank3, indices_rank4_shape, false); - auto broadcasted_indices = std::make_shared(indices_rank4, updates_shape); - - res = std::make_shared(dst, broadcasted_indices, data, axes); - } else { - auto row_size = context.get_input_shape(2)[3].get_length(); - auto ind_squeezed = std::make_shared( - indices, ov::op::v0::Constant::create(ov::element::i64, {3}, {0, 1, 2})); - auto data_reshaped = std::make_shared( - data, - ov::op::v0::Constant::create(ov::element::i64, {4}, - {(int64_t) 1, (int64_t) 1, (int64_t) -1, (int64_t) row_size}), - false); - res = std::make_shared(dst, ind_squeezed, data_reshaped, axes); - } - - auto dst_reshape = ov::as_type_ptr(dst.get_node_shared_ptr()); - if (!multidim_indices && dst_reshape) { - // Fix the case of multiple sequences, reshape back to original shape [1, n_seq, ctx_per_seq, emb] - // ctx_per_seq is not fixed due to llama-bench compatibility - auto dst_shape_partial = dst_reshape->get_input_partial_shape(0); - std::vector dst_shape = {dst_shape_partial[0].get_length(), dst_shape_partial[1].get_length(), - dst_shape_partial[2].is_static() ? dst_shape_partial[2].get_length() : -1, - dst_shape_partial[3].get_length()}; - res = std::make_shared(res, ov::op::v0::Constant::create(ov::element::i64, {4}, dst_shape), - false); - } - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/softmax.cpp b/ggml/src/ggml-openvino/openvino/op/softmax.cpp deleted file mode 100644 index b391d3f91075..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/softmax.cpp +++ /dev/null @@ -1,166 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -static bool is_static_one(const ov::Dimension & dim) { - return dim.is_static() && dim.get_length() == 1; -} - -static bool same_static_dim(const ov::Dimension & lhs, const ov::Dimension & rhs) { - return lhs.is_static() && rhs.is_static() && lhs.get_length() == rhs.get_length(); -} - -static bool is_attention_sinks_input_shape(const ov::PartialShape & candidate, const ov::PartialShape & logits_shape) { - if (candidate.rank().is_dynamic() || logits_shape.rank().is_dynamic() || candidate.rank().get_length() != 4 || - logits_shape.rank().get_length() != 4) { - return false; - } - - return is_static_one(candidate[0]) && is_static_one(candidate[1]) && is_static_one(candidate[2]) && - same_static_dim(candidate[3], logits_shape[1]); -} - -// Reimplementation of GGML_OP_SOFT_MAX semantics for OpenVINO backend: -// 1) logits = src0 * scale -// 2) logits += mask (if provided) -// 3) append attention sinks as hidden logits (if provided) -// 4) softmax over the last dimension and remove the hidden sink column -OutputVector translate_soft_max(const NodeContext & context) { - num_inputs_check(context, 1, 3); - - float scale = 1.0f; - float max_bias = 0.0f; - memcpy(&scale, (float *) context.get_output_op_params() + 0, sizeof(float)); - memcpy(&max_bias, (float *) context.get_output_op_params() + 1, sizeof(float)); - - ov::Output logits = context.get_input(0); - const bool second_input_is_sinks = - context.get_input_size() == 2 && is_attention_sinks_input_shape(context.get_input_shape(1), context.get_output_shape()); - const bool has_mask = context.get_input_size() > 1 && !second_input_is_sinks; - const bool has_sinks = second_input_is_sinks || context.get_input_size() > 2; - const size_t sinks_input_idx = second_input_is_sinks ? 1 : 2; - - // Apply scale first: logits = src0 * scale - if (scale != 1.0f) { - auto scale_const = - std::make_shared(ov::element::f32, ov::Shape{}, std::vector{scale}); - logits = std::make_shared(logits, scale_const); - } - - FRONT_END_CHECK_IMPLEMENTED(!(max_bias > 0.0f && !has_mask), - "OpenVINO softmax ALiBi path requires mask input"); - - // Optional mask add: logits += mask - // For max_bias > 0 (ALiBi), apply per-head slope to mask before adding. - if (has_mask) { - ov::Output mask = context.get_input(1); - - // For stateful - std::string mask_name = "KQ_mask_sliced"; - if (context.get_input_names()[1].find("swa") != std::string::npos) { - mask_name = "KQ_mask_swa_sliced"; - } - if (context.has_input(mask_name)) { - mask = context.get_input(mask_name); - } - - if (mask.get_element_type() != logits.get_element_type()) { - mask = std::make_shared(mask, logits.get_element_type()); - } - - if (max_bias > 0.0f) { - auto out_shape = context.get_output_shape().to_shape(); - FRONT_END_CHECK_IMPLEMENTED(out_shape.size() == 4, "OpenVINO softmax ALiBi path expects rank-4 tensor"); - - const uint32_t n_head = static_cast(out_shape[1]); - FRONT_END_CHECK_IMPLEMENTED(n_head > 0, "OpenVINO softmax ALiBi path expects n_head > 0"); - - const uint32_t n_head_log2 = 1u << static_cast(std::floor(std::log2(static_cast(n_head)))); - const float m0 = std::pow(2.0f, -(max_bias) / static_cast(n_head_log2)); - const float m1 = std::pow(2.0f, -(max_bias / 2.0f) / static_cast(n_head_log2)); - - std::vector slopes(n_head); - for (uint32_t h = 0; h < n_head; ++h) { - slopes[h] = h < n_head_log2 ? std::pow(m0, static_cast(h + 1)) : - std::pow(m1, static_cast(2 * (h - n_head_log2) + 1)); - } - - ov::Output slope_node = - std::make_shared(ov::element::f32, ov::Shape{n_head}, slopes); - if (slope_node.get_element_type() != mask.get_element_type()) { - slope_node = std::make_shared(slope_node, mask.get_element_type()); - } - - auto slope_shape = std::make_shared( - ov::element::i64, ov::Shape{4}, std::vector{1, static_cast(n_head), 1, 1}); - auto slope_4d = std::make_shared(slope_node, slope_shape, false); - mask = std::make_shared(mask, slope_4d); - } - - logits = std::make_shared(logits, mask); - } - - ov::Output softmax_input = logits; - if (has_sinks) { - ov::Output sinks = context.get_input(sinks_input_idx); - if (sinks.get_element_type() != logits.get_element_type()) { - sinks = std::make_shared(sinks, logits.get_element_type()); - } - - auto sink_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, {1, -1, 1, 1}); - auto sinks_4d = std::make_shared(sinks, sink_shape, false); - - auto logits_shape = std::make_shared(logits, ov::element::i64); - auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); - auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - auto three = ov::op::v0::Constant::create(ov::element::i64, {1}, {3}); - auto four = ov::op::v0::Constant::create(ov::element::i64, {1}, {4}); - auto shape_axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); - - auto sink_prefix_shape = std::make_shared(logits_shape, zero, three, one, shape_axis); - auto sink_last_dim = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - auto sink_broadcast_shape = std::make_shared( - ov::OutputVector{sink_prefix_shape, sink_last_dim}, 0); - auto sink_column = std::make_shared(sinks_4d, sink_broadcast_shape, - ov::op::BroadcastType::BIDIRECTIONAL); - softmax_input = std::make_shared(ov::OutputVector{logits, sink_column}, 3); - - auto softmax_with_sink = std::make_shared(softmax_input, -1); - auto original_last_dim = std::make_shared(logits_shape, three, four, one, shape_axis); - auto res = std::make_shared(softmax_with_sink, zero, original_last_dim, one, three); - - return rename_outputs_with_suffix({res}, context.get_name()); - } - - // Softmax along last dimension (equivalent to ggml softmax over ne[0]). - auto res = std::make_shared(softmax_input, -1); - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/solve_tri.cpp b/ggml/src/ggml-openvino/openvino/op/solve_tri.cpp deleted file mode 100644 index 840233f85440..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/solve_tri.cpp +++ /dev/null @@ -1,108 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -// GGML SOLVE_TRI: solve Ax = B for lower-triangular A via forward substitution. -// Currently only lower, right, non-unitriangular variant is implemented. -// -// ggml layout: A [n, n, B1, B2], B [k, n, B1, B2] → X [k, n, B1, B2] -// OV layout: A [B2, B1, n, n], B [B2, B1, n, k] → X [B2, B1, n, k] -// -// Forward substitution row i: -// x[i] = (b[i] - sum_{t(A_shape[2]); - - // Initial X: zeros with shape of B - auto B_shape_node = std::make_shared(B, ov::element::i64); - auto zero_f32 = ov::op::v0::Constant::create(ov::element::f32, {}, {0.0f}); - auto X_init = std::make_shared(zero_f32, B_shape_node); - - // --- Loop body parameters --- - // body_iter: iteration counter injected by the Loop op (i64, shape {1}) - auto body_iter = std::make_shared(ov::element::i64, ov::Shape{1}); - auto body_X = std::make_shared(ov::element::f32, ov::PartialShape::dynamic(4)); - auto body_A = std::make_shared(ov::element::f32, ov::PartialShape::dynamic(4)); - auto body_B_p = std::make_shared(ov::element::f32, ov::PartialShape::dynamic(4)); - - auto c_axis2 = ov::op::v0::Constant::create(ov::element::i64, {1}, {int64_t(2)}); - auto c_axis3 = ov::op::v0::Constant::create(ov::element::i64, {1}, {int64_t(3)}); - auto c_axis2_scalar = ov::op::v0::Constant::create(ov::element::i64, {}, {int64_t(2)}); - - // b_i = B[..., i, :] [B2, B1, 1, k] - auto b_i = std::make_shared(body_B_p, body_iter, c_axis2); - - // A_row_i = A[..., i, :] [B2, B1, 1, n] - auto A_row_i = std::make_shared(body_A, body_iter, c_axis2); - - // sum_i = A_row_i @ X [B2, B1, 1, k] - // (lower-tri zeros + unfilled-X zeros make this equal to the partial sum) - auto sum_i = std::make_shared(A_row_i, body_X, false, false); - - // diag_i = A[..., i, i] [B2, B1, 1, 1] - auto diag_i = std::make_shared(A_row_i, body_iter, c_axis3); - - // x_i = (b_i - sum_i) / diag_i [B2, B1, 1, k] - auto x_i = std::make_shared( - std::make_shared(b_i, sum_i), diag_i); - - // X_updated: scatter x_i into body_X at row i along axis 2 - auto X_updated = std::make_shared(body_X, body_iter, x_i, c_axis2_scalar); - - auto body_cond = ov::op::v0::Constant::create(ov::element::boolean, ov::Shape{1}, {true}); - - auto body = std::make_shared( - ov::OutputVector{body_cond, X_updated}, - ov::ParameterVector{body_iter, body_X, body_A, body_B_p}); - - // --- Assemble Loop --- - auto trip_count = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, std::vector{n}); - auto exec_cond = ov::op::v0::Constant::create(ov::element::boolean, ov::Shape{1}, {true}); - - auto loop = std::make_shared(trip_count, exec_cond); - loop->set_function(body); - // iter_counter_body_param_idx=0 (body_iter), exec_condition_body_result_idx=0 (body_cond) - loop->set_special_body_ports(ov::op::v5::Loop::SpecialBodyPorts{0, 0}); - - // Carried state: X feeds back from X_updated each iteration - loop->set_merged_input(body_X, X_init, X_updated); - // Invariant inputs passed through unchanged - loop->set_invariant_input(body_A, A); - loop->set_invariant_input(body_B_p, B); - - // Final output: value of X_updated after the last iteration - auto X_final = loop->get_iter_value(X_updated, -1); - - return rename_outputs_with_suffix({X_final}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/sqr.cpp b/ggml/src/ggml-openvino/openvino/op/sqr.cpp deleted file mode 100644 index be01fdc53707..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/sqr.cpp +++ /dev/null @@ -1,35 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -OutputVector translate_sqr(const NodeContext & context) { - num_inputs_check(context, 1, 1); - - auto input = process_view_input_new(context, 0); - auto res = std::make_shared(input, input); - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -OutputVector translate_sqrt(const NodeContext & context) { - num_inputs_check(context, 1, 1); - - auto input = process_view_input_new(context, 0); - auto res = std::make_shared(input); - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/ssm_conv.cpp b/ggml/src/ggml-openvino/openvino/op/ssm_conv.cpp deleted file mode 100644 index 352fd90560f2..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/ssm_conv.cpp +++ /dev/null @@ -1,61 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -OutputVector translate_ssm_conv(const NodeContext & context) { - num_inputs_check(context, 2, 2); - - auto sx = context.get_input(0); // conv state + input: OV shape [1, n_s, d_inner, ncs] - auto c = context.get_input(1); // conv1d weight: OV shape [1, 1, d_inner, d_conv] - - auto sx_shape = context.get_input_shape(0).to_shape(); // [1, n_s, d_inner, ncs] - auto c_shape = context.get_input_shape(1).to_shape(); // [1, 1, d_inner, d_conv] - - // int64_t n_s = sx_shape[1]; - int64_t d_inner = sx_shape[2]; - // int64_t ncs = sx_shape[3]; // d_conv - 1 + n_t - int64_t d_conv = c_shape[3]; - // int64_t n_t = ncs - d_conv + 1; - - // Reshape sx from [1, n_s, d_inner, ncs] to [n_s, d_inner, ncs] for 1D GroupConvolution - auto sx_reshaped = - std::make_shared(sx, ov::op::v0::Constant::create(ov::element::i64, {1}, {0})); - - // Reshape c from [1, 1, d_inner, d_conv] to [d_inner, 1, 1, d_conv] - // GroupConvolution filter: [groups, out_channels/groups, in_channels/groups, kernel_size] - auto c_new_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{d_inner, 1, 1, d_conv}); - auto c_reshaped = std::make_shared(c, c_new_shape, false); - - // Depthwise 1D convolution: groups=d_inner, stride=1, no padding, no dilation - // Input: [n_s, d_inner, ncs], Filter: [d_inner, 1, 1, d_conv] - // Output: [n_s, d_inner, n_t] - auto conv = std::make_shared( - sx_reshaped, c_reshaped, ov::Strides{1}, ov::CoordinateDiff{0}, ov::CoordinateDiff{0}, ov::Strides{1}); - - // Transpose from [n_s, d_inner, n_t] to [n_s, n_t, d_inner] - auto perm = ov::op::v0::Constant::create(ov::element::i64, {3}, std::vector{0, 2, 1}); - auto transposed = std::make_shared(conv, perm); - - // Reshape to output shape [1, n_s, n_t, d_inner] - auto res = - std::make_shared(transposed, ov::op::v0::Constant::create(ov::element::i64, {1}, {0})); - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/sum.cpp b/ggml/src/ggml-openvino/openvino/op/sum.cpp deleted file mode 100644 index c480ee456fe5..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/sum.cpp +++ /dev/null @@ -1,30 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -OutputVector translate_sum(const NodeContext & context) { - num_inputs_check(context, 1, 1); - - auto input = process_view_input_new(context, 0); - auto axes = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{4}, {0, 1, 2, 3}); - auto res = std::make_shared(input, axes, true); - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/sum_rows.cpp b/ggml/src/ggml-openvino/openvino/op/sum_rows.cpp deleted file mode 100644 index d04e6443be95..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/sum_rows.cpp +++ /dev/null @@ -1,27 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -OutputVector translate_sum_rows(const NodeContext & context) { - num_inputs_check(context, 1, 1); - - auto input = process_view_input_new(context, 0); - auto res = std::make_shared( - input, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1}), true); - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/transpose.cpp b/ggml/src/ggml-openvino/openvino/op/transpose.cpp deleted file mode 100644 index 0651a410a1a0..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/transpose.cpp +++ /dev/null @@ -1,52 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -OutputVector translate_transpose(const NodeContext & context) { - num_inputs_check(context, 1, 1); - - // Compute permute order from input/output shape and stride information - // so it adapts to different input and output layouts. - auto input_stride = context.get_input_stride(0); - auto output_stride = context.get_output_stride(); - - // Compute permute order by matching output and input stride rankings. - // Build pairs. - std::vector> output_stride_dims; - std::vector> input_stride_dims; - - for (int i = 0; i < 4; ++i) { - output_stride_dims.push_back({output_stride[i], i}); - input_stride_dims.push_back({input_stride[i], i}); - } - - // Sort by stride in descending order. - std::sort(output_stride_dims.rbegin(), output_stride_dims.rend()); - std::sort(input_stride_dims.rbegin(), input_stride_dims.rend()); - - // Build permute order. - std::vector permute_order(4); - for (int i = 0; i < 4; ++i) { - int output_dim = output_stride_dims[i].second; - int input_dim = input_stride_dims[i].second; - permute_order[output_dim] = input_dim; - } - - auto input = process_view_input_new(context, 0); - - auto res = std::make_shared( - input, ov::op::v0::Constant::create(ov::element::i64, {4}, permute_order)); - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/tri.cpp b/ggml/src/ggml-openvino/openvino/op/tri.cpp deleted file mode 100644 index 9b7774a383ec..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/tri.cpp +++ /dev/null @@ -1,82 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -// GGML TRI zeroes out elements outside a triangular region of a square matrix. -// The type param (stored in op_params[0]) maps to ggml_tri_type: -// 0 = UPPER_DIAG : keep where col >= row -// 1 = UPPER : keep where col > row -// 2 = LOWER_DIAG : keep where col <= row -// 3 = LOWER : keep where col < row -// -// In OV layout (ggml [ne0, ne1, ne2, ne3] → OV [ne3, ne2, ne1, ne0]): -// ggml dim 0 (ne0, cols) → OV axis 3 -// ggml dim 1 (ne1, rows) → OV axis 2 -// The matrix is square so ne0 == ne1. -OutputVector translate_tri(const NodeContext & context) { - num_inputs_check(context, 1, 1); - - auto x = context.get_input(0); // OV shape: [ne3, ne2, ne1, ne0] - - int32_t tri_type = context.get_output_op_params()[0]; - - auto shape = context.get_input_shape(0).to_shape(); - int64_t n = static_cast(shape[3]); // ne0 == ne1 - - // Build index range [0, 1, ..., n-1] - auto start = ov::op::v0::Constant::create(ov::element::i64, {}, {int64_t(0)}); - auto stop = ov::op::v0::Constant::create(ov::element::i64, {}, {n}); - auto step = ov::op::v0::Constant::create(ov::element::i64, {}, {int64_t(1)}); - auto range = std::make_shared(start, stop, step, ov::element::i64); - - // col_idx shape [1, 1, 1, n] — broadcasts over batch and row dims - auto col_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{1, 1, 1, n}); - auto col_idx = std::make_shared(range, col_shape, false); - - // row_idx shape [1, 1, n, 1] — broadcasts over batch and col dims - auto row_shape = ov::op::v0::Constant::create(ov::element::i64, {4}, std::vector{1, 1, n, 1}); - auto row_idx = std::make_shared(range, row_shape, false); - - // Build boolean mask: true where element should be kept - std::shared_ptr mask; - switch (tri_type) { - case 0: // UPPER_DIAG: col >= row - mask = std::make_shared(col_idx, row_idx); - break; - case 1: // UPPER: col > row - mask = std::make_shared(col_idx, row_idx); - break; - case 2: // LOWER_DIAG: col <= row - mask = std::make_shared(col_idx, row_idx); - break; - case 3: // LOWER: col < row - mask = std::make_shared(col_idx, row_idx); - break; - default: - throw std::runtime_error("translate_tri: invalid tri_type " + std::to_string(tri_type)); - } - - auto zero = ov::op::v0::Constant::create(ov::element::f32, {}, {0.0f}); - auto res = std::make_shared(mask, x, zero); - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/unary.cpp b/ggml/src/ggml-openvino/openvino/op/unary.cpp deleted file mode 100644 index 1bb7aa2c8e56..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/unary.cpp +++ /dev/null @@ -1,138 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" -#include "ggml-openvino/ggml-openvino-extra.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -OutputVector translate_unary_gelu(const NodeContext & context) { - num_inputs_check(context, 1, 1); - auto input = process_view_input_new(context, 0); - auto res = std::make_shared(input, ov::op::GeluApproximationMode::TANH); - return rename_outputs_with_suffix({res}, context.get_name()); -} - -OutputVector translate_unary_gelu_erf(const NodeContext & context) { - num_inputs_check(context, 1, 1); - auto input = process_view_input_new(context, 0); - auto res = std::make_shared(input, ov::op::GeluApproximationMode::ERF); - return rename_outputs_with_suffix({res}, context.get_name()); -} - -OutputVector translate_unary_gelu_quick(const NodeContext & context) { - num_inputs_check(context, 1, 1); - auto input = process_view_input_new(context, 0); - auto scale = ov::op::v0::Constant::create(input.get_element_type(), ov::Shape{}, {1.702f}); - auto mul = std::make_shared(input, scale); - auto sig = std::make_shared(mul); - auto res = std::make_shared(input, sig); - return rename_outputs_with_suffix({res}, context.get_name()); -} - -OutputVector translate_unary_elu(const NodeContext & context) { - num_inputs_check(context, 1, 1); - auto input = process_view_input_new(context, 0); - auto res = std::make_shared(input, 1.0); - return rename_outputs_with_suffix({res}, context.get_name()); -} - -OutputVector translate_unary_hardsigmoid(const NodeContext & context) { - num_inputs_check(context, 1, 1); - // compute in f32 like the ggml reference: 1/6 is not exact in f16/bf16 (NPU cannot take the f32 path) - auto input = process_view_input_new(context, 0); - const auto type = ggml_openvino_is_npu() ? input.get_element_type() : ov::element::f32; - ov::Output x = input; - if (type != input.get_element_type()) { - x = std::make_shared(input, type); - } - auto alpha = ov::op::v0::Constant::create(type, ov::Shape{}, {1.0f / 6.0f}); - auto beta = ov::op::v0::Constant::create(type, ov::Shape{}, {0.5f}); - ov::Output res = std::make_shared(x, alpha, beta); - if (type != input.get_element_type()) { - res = std::make_shared(res, input.get_element_type()); - } - return rename_outputs_with_suffix({res}, context.get_name()); -} - -OutputVector translate_unary_step(const NodeContext & context) { - num_inputs_check(context, 1, 1); - auto input = process_view_input_new(context, 0); - auto zero = ov::op::v0::Constant::create(input.get_element_type(), ov::Shape{}, {0.0f}); - auto cond = std::make_shared(input, zero); - auto res = std::make_shared(cond, input.get_element_type()); - return rename_outputs_with_suffix({res}, context.get_name()); -} - -OutputVector translate_unary_round(const NodeContext & context) { - num_inputs_check(context, 1, 1); - auto input = process_view_input_new(context, 0); - auto res = std::make_shared(input, ov::op::v5::Round::RoundMode::HALF_AWAY_FROM_ZERO); - return rename_outputs_with_suffix({res}, context.get_name()); -} - -OutputVector translate_unary_expm1(const NodeContext & context) { - num_inputs_check(context, 1, 1); - // compute in f32 like the ggml reference: exp(x) - 1 in f16 loses the small-x digits (NPU cannot take the f32 path) - auto input = process_view_input_new(context, 0); - const auto type = ggml_openvino_is_npu() ? input.get_element_type() : ov::element::f32; - ov::Output x = input; - if (type != input.get_element_type()) { - x = std::make_shared(input, type); - } - auto exp = std::make_shared(x); - auto one = ov::op::v0::Constant::create(type, ov::Shape{}, {1.0f}); - ov::Output res = std::make_shared(exp, one); - if (type != input.get_element_type()) { - res = std::make_shared(res, input.get_element_type()); - } - return rename_outputs_with_suffix({res}, context.get_name()); -} - -OutputVector translate_unary_softplus(const NodeContext & context) { - num_inputs_check(context, 1, 1); - - if (ggml_openvino_getenv_int("GGML_OPENVINO_NATIVE_SOFTPLUS") != 0) { - return translate_1to1_match_1_input(context); - } - - auto input = process_view_input_new(context, 0); - const auto element_type = input.get_element_type(); - auto one = ov::op::v0::Constant::create(element_type, ov::Shape{}, {1.0f}); - - auto positive = std::make_shared(input); - auto abs = std::make_shared(input); - auto neg_abs = std::make_shared(abs); - auto exp_neg_abs = std::make_shared(neg_abs); - auto log_term = std::make_shared(std::make_shared(one, exp_neg_abs)); - auto res = std::make_shared(positive, log_term); - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/upscale.cpp b/ggml/src/ggml-openvino/openvino/op/upscale.cpp deleted file mode 100644 index 03f3a91b2e61..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/upscale.cpp +++ /dev/null @@ -1,181 +0,0 @@ -#include "../node_context.h" -#include "../op_table.h" -#include "../utils.h" -#include "ggml.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -OutputVector translate_upscale(const NodeContext & context) { - num_inputs_check(context, 1, 1); - - using Interpolate = ov::op::v4::Interpolate; - auto input = process_view_input_new(context, 0); - - const auto input_shape = context.get_input_shape(0).to_shape(); - const auto output_shape = context.get_output_shape().to_shape(); - - if (input_shape == output_shape) { - return rename_outputs_with_suffix({input}, context.get_name()); - } - - ov::Output res = input; - - // Resample batch / channel dimensions (ne[3] and ne[2], corresponding to OV axes 0 and 1) - // using nearest-neighbor index mapping: i0_d = floor(i_d * in_d / out_d). - for (size_t axis = 0; axis < 2; ++axis) { - const size_t in_dim = input_shape[axis]; - const size_t out_dim = output_shape[axis]; - if (in_dim != out_dim) { - std::vector indices(out_dim); - for (size_t i = 0; i < out_dim; ++i) { - indices[i] = static_cast((i * in_dim) / out_dim); - } - auto indices_node = ov::op::v0::Constant::create(ov::element::i64, {indices.size()}, indices); - auto axis_node = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {axis}); - res = std::make_shared(res, indices_node, axis_node); - } - } - - // Spatial interpolation for ne[1] and ne[0] (OV axes 2 and 3) - if (input_shape[2] != output_shape[2] || input_shape[3] != output_shape[3]) { - const int32_t * op_params = context.get_output_op_params(); - const int32_t mode_flags = op_params != nullptr ? op_params[0] : 0; - - const int op_case = context.get_op_case(); - const bool align_corners = (mode_flags & GGML_SCALE_FLAG_ALIGN_CORNERS) != 0; - const bool antialias = (mode_flags & GGML_SCALE_FLAG_ANTIALIAS) != 0; - - if (op_case == 2 && antialias) { // GGML_SCALE_MODE_BILINEAR with antialias - const size_t H_in = input_shape[2]; - const size_t W_in = input_shape[3]; - const size_t H_out = output_shape[2]; - const size_t W_out = output_shape[3]; - - const float pixel_offset = 0.5f; - - // Height projection: [H_out, H_in] - std::vector Wy_data(H_out * H_in, 0.0f); - const float sf1 = static_cast(H_out) / H_in; - const float support1 = std::max(1.0f, 1.0f / sf1); - const float invscale1 = 1.0f / support1; - - for (size_t i1 = 0; i1 < H_out; ++i1) { - const float y = (static_cast(i1) + pixel_offset) / sf1; - const auto y_start = static_cast(y - support1 + pixel_offset); - const size_t y_min = y_start > 0 ? static_cast(y_start) : 0; - const auto y_end = static_cast(y + support1 + pixel_offset); - const size_t y_max = y_end > 0 ? std::min(static_cast(y_end), H_in) : 0; - - float total_weight = 0.0f; - for (size_t sy = y_min; sy < y_max; ++sy) { - float diff = std::abs((static_cast(sy) - y + pixel_offset) * invscale1); - float weight = std::max(1.0f - diff, 0.0f); - Wy_data[i1 * H_in + sy] = weight; - total_weight += weight; - } - if (total_weight > 0.0f) { - for (size_t sy = y_min; sy < y_max; ++sy) { - Wy_data[i1 * H_in + sy] /= total_weight; - } - } - } - - // Width projection: [W_in, W_out] - std::vector Wx_data(W_in * W_out, 0.0f); - const float sf0 = static_cast(W_out) / W_in; - const float support0 = std::max(1.0f, 1.0f / sf0); - const float invscale0 = 1.0f / support0; - - for (size_t i0 = 0; i0 < W_out; ++i0) { - const float x = (static_cast(i0) + pixel_offset) / sf0; - const auto x_start = static_cast(x - support0 + pixel_offset); - const size_t x_min = x_start > 0 ? static_cast(x_start) : 0; - const auto x_end = static_cast(x + support0 + pixel_offset); - const size_t x_max = x_end > 0 ? std::min(static_cast(x_end), W_in) : 0; - - float total_weight = 0.0f; - for (size_t sx = x_min; sx < x_max; ++sx) { - float diff = std::abs((static_cast(sx) - x + pixel_offset) * invscale0); - float weight = std::max(1.0f - diff, 0.0f); - Wx_data[sx * W_out + i0] = weight; - total_weight += weight; - } - if (total_weight > 0.0f) { - for (size_t sx = x_min; sx < x_max; ++sx) { - Wx_data[sx * W_out + i0] /= total_weight; - } - } - } - - auto Wy_node = ov::op::v0::Constant::create(ov::element::f32, {H_out, H_in}, Wy_data); - auto Wx_node = ov::op::v0::Constant::create(ov::element::f32, {W_in, W_out}, Wx_data); - - auto res_y = std::make_shared(Wy_node, res); - res = std::make_shared(res_y, Wx_node); - } else { - Interpolate::InterpolateAttrs attrs; - attrs.shape_calculation_mode = Interpolate::ShapeCalcMode::SIZES; - attrs.antialias = antialias; - attrs.pads_begin = {0, 0, 0, 0}; - attrs.pads_end = {0, 0, 0, 0}; - - switch (op_case) { - case 1: // GGML_SCALE_MODE_NEAREST - attrs.mode = Interpolate::InterpolateMode::NEAREST; - attrs.nearest_mode = Interpolate::NearestMode::FLOOR; - attrs.coordinate_transformation_mode = Interpolate::CoordinateTransformMode::ASYMMETRIC; - break; - case 2: // GGML_SCALE_MODE_BILINEAR - attrs.mode = Interpolate::InterpolateMode::LINEAR; - attrs.coordinate_transformation_mode = align_corners - ? Interpolate::CoordinateTransformMode::ALIGN_CORNERS - : Interpolate::CoordinateTransformMode::HALF_PIXEL; - break; - case 3: // GGML_SCALE_MODE_BICUBIC - attrs.mode = Interpolate::InterpolateMode::CUBIC; - attrs.cube_coeff = -0.75; - attrs.coordinate_transformation_mode = align_corners - ? Interpolate::CoordinateTransformMode::ALIGN_CORNERS - : Interpolate::CoordinateTransformMode::HALF_PIXEL; - break; - default: - FRONT_END_OP_CONVERSION_CHECK(false, "Unsupported upscale op_case: ", op_case); - } - - std::vector target_shape_vec = { - static_cast(output_shape[2]), - static_cast(output_shape[3]), - }; - std::vector scales_vec = { - static_cast(output_shape[2]) / static_cast(input_shape[2]), - static_cast(output_shape[3]) / static_cast(input_shape[3]), - }; - - auto target_shape_node = ov::op::v0::Constant::create(ov::element::i64, {2}, target_shape_vec); - auto scales_node = ov::op::v0::Constant::create(ov::element::f32, {2}, scales_vec); - auto axes_node = ov::op::v0::Constant::create(ov::element::i64, {2}, {2, 3}); - - res = std::make_shared( - res, target_shape_node, scales_node, axes_node, attrs); - } - } - - return rename_outputs_with_suffix({res}, context.get_name()); -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op/view.cpp b/ggml/src/ggml-openvino/openvino/op/view.cpp deleted file mode 100644 index 486c2ff28ce4..000000000000 --- a/ggml/src/ggml-openvino/openvino/op/view.cpp +++ /dev/null @@ -1,259 +0,0 @@ -#include "../op_table.h" -#include "../utils.h" - -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace op { - -OutputVector translate_view(const NodeContext & context) { - num_inputs_check(context, 1, 1); - - if (context.get_op_case() == 1) { - // Static-mode identity pass-through for VIEWs over a GATED_DELTA_NET combined output or - // the conv_input CONCAT; the consuming op (CPY/RMS_NORM) does its own runtime-correct - // slicing on the full tensor (see ggml-decoder.cpp compute_op_case, GGML_OP_VIEW). - return {context.get_input(0)}; - } - - if (!context.is_static()) { - // On the stateless/non-static path VIEW is normally a no-op (consumers re-slice). - // EXCEPTION: the MoE expert aggregation slices each expert plane out of - // ffn_moe_weighted [n_embd, n_expert_used, n_tokens] with ggml_view_2d and then - // sums the planes with a chain of ADDs (llama-graph.cpp). Those ADDs read this - // VIEW node directly from the tensor map and do NOT re-slice, so a no-op here - // makes every plane the full tensor and the expert sum collapses. Materialize the - // single-expert slice here. Gated by name (ffn_moe_weighted...view) so it can't - // affect any other view. - const std::string & vname = context.get_name(); - if (vname.find("ffn_moe_weighted") != std::string::npos) { - auto src_ps = context.get_input_shape(0); - auto dst_ps = context.get_output_shape(); - if (src_ps.rank().is_static() && dst_ps.rank().is_static() && src_ps.rank() == dst_ps.rank() && - src_ps.is_static() && dst_ps.is_static()) { - auto sst = context.get_input_stride(0); - auto dst = context.get_output_stride(); - size_t voff = context.get_output_op_offset(); - auto ss = src_ps.to_shape(); - auto dd = dst_ps.to_shape(); - const size_t nd = ss.size(); - // Stateful models drop the leading size-1 batch dim, so the real OV tensor can - // be one rank lower than the ggml shape metadata above. Axis indices derived - // from that metadata must be shifted down by the difference before they are - // used as OV axes. Bail out if the dims we would drop are not all 1. - const auto in_rank = context.get_input(0).get_partial_shape().rank(); - const int axis_shift = - in_rank.is_static() ? (int) nd - (int) in_rank.get_length() : 0; - bool shift_ok = axis_shift >= 0 && (size_t) axis_shift < nd; - for (int a = 0; a < axis_shift && shift_ok; ++a) { - shift_ok = (ss[a] == 1 && dd[a] == 1); - } - if (shift_ok && sst.size() == nd && dst.size() == nd) { - // Map each dst axis of size>1 to a src axis with equal (size,stride); - // the unmatched src axis of size>1 is the indexed expert axis. - // dst_to_src[d] records which src axis each dst axis came from, so we can - // later pull the dynamic (token) dim from the right source axis at runtime. - std::vector used(nd, false); - std::vector dst_to_src(nd, -1); - bool ok = true; - for (size_t d = 0; d < nd; ++d) { - if (dd[d] == 1) { - continue; - } - int found = -1; - for (size_t s = 0; s < nd; ++s) { - if (!used[s] && ss[s] == dd[d] && sst[s] == dst[d]) { - found = (int) s; - break; - } - } - if (found < 0) { - ok = false; - break; - } - used[found] = true; - dst_to_src[d] = found; - } - int dropped = -1; - if (ok) { - for (size_t s = 0; s < nd; ++s) { - if (!used[s] && ss[s] > 1) { - if (dropped >= 0) { - ok = false; - break; - } - dropped = (int) s; - } - } - } - if (ok && dropped >= 0) { - const size_t dstr = sst[dropped]; - const int64_t dsz = (int64_t) ss[dropped]; - if (dstr > 0 && voff % dstr == 0) { - const int64_t sel = (int64_t) (voff / dstr); - if (sel >= 0 && sel < dsz) { - ov::Output sl = std::make_shared( - context.get_input(0), - ov::op::v0::Constant::create(ov::element::i64, {1}, {sel}), - ov::op::v0::Constant::create(ov::element::i64, {1}, {sel + 1}), - ov::op::v0::Constant::create(ov::element::i64, {1}, {1}), - ov::op::v0::Constant::create(ov::element::i64, {1}, - {dropped - axis_shift})); - // Build the reshape target from the (concrete) dst shape, but - // keep the dynamic token axis dynamic instead of freezing it - // to the captured n_tokens. Without this the constant dst - // shape bakes in the prefill token count and the static value - // flows downstream, turning every later decoder layer static - // (the GPU in-place-concat KV-cache bug). The token axis is - // PERMUTED between the sliced input and the dst (e.g. input - // [1,tok,expert,emb] -> dst [1,1,tok,emb]), so special_zero - // (which copies the same-position dim) is not enough: pull the - // dynamic dim from the correct SOURCE axis via ShapeOf+Gather - // and place it at the dst token position. - const int32_t dyn = context.get_op_dynamic_dim(); // output ggml axis, -1 if none - // still in ggml metadata axis space; get_shape() reverses ggml order - int dst_ov_axis = (dyn != -1) ? ((int) nd - 1 - (int) dyn) : -1; - int src_ov_axis = (dst_ov_axis >= axis_shift && dst_ov_axis < (int) nd) - ? dst_to_src[dst_ov_axis] - : -1; - if (dst_ov_axis >= 0 && src_ov_axis >= axis_shift) { - // target = concat of per-axis scalars; the token axis is a - // runtime Gather of the slice's shape, the rest are constants. - auto sl_shape = std::make_shared(sl, ov::element::i64); - auto tok_dim = std::make_shared( - sl_shape, - ov::op::v0::Constant::create(ov::element::i64, {1}, - {src_ov_axis - axis_shift}), - ov::op::v0::Constant::create(ov::element::i64, {}, {0})); - ov::OutputVector parts; - for (int a = axis_shift; a < (int) nd; ++a) { - if (a == dst_ov_axis) { - parts.push_back(tok_dim); - } else { - parts.push_back(ov::op::v0::Constant::create( - ov::element::i64, {1}, {(int64_t) dd[a]})); - } - } - auto dc = std::make_shared(parts, 0); - auto rs = std::make_shared(sl, dc, false); - return rename_outputs_with_suffix({rs}, context.get_name()); - } - std::vector dd_ov(dd.begin() + axis_shift, dd.end()); - auto dc = ov::op::v0::Constant::create(ov::element::i64, {dd_ov.size()}, dd_ov); - auto rs = std::make_shared(sl, dc, false); - return rename_outputs_with_suffix({rs}, context.get_name()); - } - } - } - } - } - } - return {context.get_input(0)}; - } - - auto input = context.get_input(0); - auto src_shape = context.get_input_shape(0); - auto dst_shape = context.get_output_shape(); - - if (src_shape.rank().is_dynamic() || dst_shape.rank().is_dynamic()) { - return {input}; - } - - int64_t src_elems = 1; - int64_t dst_elems = 1; - for (int64_t i = 0; i < src_shape.rank().get_length(); ++i) { - if (src_shape[i].is_dynamic()) { - return {input}; - } - src_elems *= src_shape[i].get_length(); - } - for (int64_t i = 0; i < dst_shape.rank().get_length(); ++i) { - if (dst_shape[i].is_dynamic()) { - return {input}; - } - dst_elems *= dst_shape[i].get_length(); - } - - if (dst_elems >= src_elems) { - return {input}; - } - - auto src_stride = context.get_input_stride(0); - auto dst_stride = context.get_output_stride(); - size_t view_offset = context.get_output_op_offset(); - - bool same_stride = (src_stride.size() == dst_stride.size()); - if (same_stride) { - for (size_t i = 0; i < src_stride.size(); ++i) { - if (src_stride[i] != dst_stride[i]) { - same_stride = false; - break; - } - } - } - - if (!same_stride) { - return {input}; - } - - auto src_ov_shape = src_shape.to_shape(); - auto dst_ov_shape = dst_shape.to_shape(); - size_t ndims = src_ov_shape.size(); - if (dst_ov_shape.size() != ndims) { - return {input}; - } - - std::vector diff_dims; - for (size_t i = 0; i < ndims; ++i) { - if (src_ov_shape[i] != dst_ov_shape[i]) { - diff_dims.push_back(static_cast(i)); - } - } - - if (diff_dims.size() != 1) { - return {input}; - } - - int slice_dim = diff_dims[0]; - int64_t dim_size = static_cast(src_ov_shape[slice_dim]); - - size_t ov_stride_for_dim = 1; - for (size_t i = slice_dim + 1; i < ndims; ++i) { - ov_stride_for_dim *= src_ov_shape[i]; - } - size_t elem_size = src_stride.back(); - if (elem_size == 0) { - elem_size = 1; - } - - int64_t begin_val = 0; - if (ov_stride_for_dim > 0 && elem_size > 0) { - begin_val = static_cast((view_offset / elem_size) / ov_stride_for_dim); - } - int64_t end_val = begin_val + static_cast(dst_ov_shape[slice_dim]); - - if (begin_val < 0 || end_val > dim_size) { - return {input}; - } - - auto sliced = - std::make_shared(input, ov::op::v0::Constant::create(ov::element::i64, {1}, {begin_val}), - ov::op::v0::Constant::create(ov::element::i64, {1}, {end_val}), - ov::op::v0::Constant::create(ov::element::i64, {1}, {1}), - ov::op::v0::Constant::create(ov::element::i64, {1}, {slice_dim})); - - sliced->set_friendly_name(context.get_output_name()); - return {sliced->output(0)}; -} - -} // namespace op -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op_table.cpp b/ggml/src/ggml-openvino/openvino/op_table.cpp deleted file mode 100644 index 12a0953e3cf9..000000000000 --- a/ggml/src/ggml-openvino/openvino/op_table.cpp +++ /dev/null @@ -1,121 +0,0 @@ -#include "op_table.h" - -#include "utils.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { - -std::unordered_map get_supported_ops() { - using namespace ov::op; - return { - {"GGML_OP_ADD", op::translate_add }, - {"GGML_OP_ADD1", op::translate_1to1_match_2_inputs }, - {"GGML_OP_ADD_ID", op::translate_add_id }, - {"GGML_OP_CONCAT", op::translate_concat }, - {"GGML_OP_CONT", op::translate_cont }, - {"GGML_OP_DIV", op::translate_div }, - {"GGML_OP_FILL", op::translate_fill }, - {"GGML_OP_GET_ROWS", op::translate_get_rows }, - {"GGML_OP_IM2COL", op::translate_im2col }, - {"GGML_OP_IM2COL_3D", op::translate_im2col_3d }, - {"GGML_OP_MUL", op::translate_1to1_match_2_inputs}, - {"GGML_OP_MUL_MAT", op::translate_mulmat }, - {"GGML_OP_MUL_MAT_ID", op::translate_mul_mat_id }, - {"GGML_OP_PERMUTE", op::translate_permute }, - {"GGML_OP_RESHAPE", op::translate_reshape }, - {"GGML_OP_RMS_NORM", op::translate_rms_norm }, - {"GGML_OP_NORM", op::translate_norm }, - {"GGML_OP_L2_NORM", op::translate_l2_norm }, - {"GGML_OP_SUM_ROWS", op::translate_sum_rows }, - {"GGML_OP_ROPE", op::translate_rope }, - {"GGML_OP_SCALE", op::translate_scale }, - {"GGML_OP_SQR", op::translate_sqr }, - {"GGML_OP_SQRT", op::translate_sqrt }, - {"GGML_OP_SOFT_MAX", op::translate_soft_max }, - {"GGML_OP_ARGSORT", op::translate_argsort }, - {"GGML_OP_SUB", op::translate_1to1_match_2_inputs}, - {"GGML_OP_TRANSPOSE", op::translate_transpose }, - {"GGML_OP_SIN", op::translate_1to1_match_1_input }, - {"GGML_OP_COS", op::translate_1to1_match_1_input }, - {"GGML_OP_LOG", op::translate_1to1_match_1_input }, - {"GGML_OP_MEAN", op::translate_mean }, - {"GGML_OP_SUM", op::translate_sum }, - {"GGML_UNARY_OP_GELU", op::translate_unary_gelu }, - {"GGML_UNARY_OP_GELU_ERF", op::translate_unary_gelu_erf }, - {"GGML_UNARY_OP_GELU_QUICK", op::translate_unary_gelu_quick }, - {"GGML_UNARY_OP_ELU", op::translate_unary_elu }, - {"GGML_UNARY_OP_HARDSWISH", op::translate_1to1_match_1_input }, - {"GGML_UNARY_OP_HARDSIGMOID", op::translate_unary_hardsigmoid }, - {"GGML_UNARY_OP_STEP", op::translate_unary_step }, - {"GGML_UNARY_OP_ABS", op::translate_1to1_match_1_input }, - {"GGML_UNARY_OP_SGN", op::translate_1to1_match_1_input }, - {"GGML_UNARY_OP_FLOOR", op::translate_1to1_match_1_input }, - {"GGML_UNARY_OP_CEIL", op::translate_1to1_match_1_input }, - {"GGML_UNARY_OP_ROUND", op::translate_unary_round }, - {"GGML_UNARY_OP_EXPM1", op::translate_unary_expm1 }, - {"GGML_UNARY_OP_SIGMOID", op::translate_1to1_match_1_input }, - {"GGML_UNARY_OP_SILU", op::translate_1to1_match_1_input }, - {"GGML_UNARY_OP_SOFTPLUS", op::translate_unary_softplus }, - {"GGML_UNARY_OP_TANH", op::translate_1to1_match_1_input }, - {"GGML_UNARY_OP_EXP", op::translate_1to1_match_1_input }, - {"GGML_UNARY_OP_NEG", op::translate_1to1_match_1_input }, - {"GGML_UNARY_OP_RELU", op::translate_1to1_match_1_input }, - {"GGML_OP_VIEW", op::translate_view }, - {"GGML_GLU_OP_SWIGLU", op::translate_glu_swiglu }, - {"GGML_GLU_OP_SWIGLU_OAI", op::translate_glu_swiglu_oai }, - {"GGML_GLU_OP_SWIGLU_CLAMP", op::translate_glu_swiglu_clamp }, - {"GGML_GLU_OP_GEGLU", op::translate_glu_geglu }, - {"GGML_GLU_OP_GEGLU_QUICK", op::translate_glu_geglu_quick }, - {"GGML_OP_SET_ROWS", op::translate_set_rows }, - {"GGML_OP_CPY", op::translate_cpy }, - {"GGML_OP_FLASH_ATTN_EXT", op::translate_flash_attn_ext }, - {"GGML_OP_CLAMP", op::translate_clamp }, - {"GGML_OP_PAD", op::translate_pad }, - {"GGML_OP_SSM_CONV", op::translate_ssm_conv }, - {"GGML_OP_GATED_DELTA_NET", op::translate_gated_delta_net }, - {"GGML_OP_REPEAT", op::translate_repeat }, - {"GGML_OP_CUMSUM", op::translate_cumsum }, - {"GGML_OP_FILL", op::translate_fill }, - {"GGML_OP_DIAG", op::translate_diag }, - {"GGML_OP_TRI", op::translate_tri }, - {"GGML_OP_SET", op::translate_set }, - {"GGML_OP_POOL_2D", op::translate_pool_2d }, - {"GGML_OP_ROLL", op::translate_roll }, - {"GGML_OP_UPSCALE", op::translate_upscale }, - {"GGML_OP_CONV_2D", op::translate_conv_2d }, - {"GGML_OP_CONV_2D_DW", op::translate_conv_2d_dw }, - {"GGML_OP_CONV_TRANSPOSE_1D", op::translate_conv_transpose_1d }, - {"GGML_OP_CONV_TRANSPOSE_2D", op::translate_conv_transpose_2d }, - {"GGML_OP_CONV_3D", op::translate_conv_3d }, - // solve_tri has accuracy issues on GPU - // {"GGML_OP_SOLVE_TRI", op::translate_solve_tri }, - }; -} - -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/op_table.h b/ggml/src/ggml-openvino/openvino/op_table.h deleted file mode 100644 index 6ed7549a2652..000000000000 --- a/ggml/src/ggml-openvino/openvino/op_table.h +++ /dev/null @@ -1,83 +0,0 @@ -#pragma once - -#include "node_context.h" - -namespace ov { -namespace frontend { -namespace ggml { - -namespace op { - -#define GGML_OP_CONVERTER(op) OutputVector op(const NodeContext & context) - -GGML_OP_CONVERTER(translate_add); -GGML_OP_CONVERTER(translate_cont); -GGML_OP_CONVERTER(translate_concat); -GGML_OP_CONVERTER(translate_add_id); -GGML_OP_CONVERTER(translate_div); -GGML_OP_CONVERTER(translate_fill); -GGML_OP_CONVERTER(translate_get_rows); -GGML_OP_CONVERTER(translate_im2col); -GGML_OP_CONVERTER(translate_im2col_3d); -GGML_OP_CONVERTER(translate_mulmat); -GGML_OP_CONVERTER(translate_mul_mat_id); -GGML_OP_CONVERTER(translate_permute); -GGML_OP_CONVERTER(translate_reshape); -GGML_OP_CONVERTER(translate_rms_norm); -GGML_OP_CONVERTER(translate_norm); -GGML_OP_CONVERTER(translate_l2_norm); -GGML_OP_CONVERTER(translate_sum_rows); -GGML_OP_CONVERTER(translate_sqr); -GGML_OP_CONVERTER(translate_rope); -GGML_OP_CONVERTER(translate_scale); -GGML_OP_CONVERTER(translate_sqrt); -GGML_OP_CONVERTER(translate_unary_softplus); -GGML_OP_CONVERTER(translate_soft_max); -GGML_OP_CONVERTER(translate_transpose); -GGML_OP_CONVERTER(translate_view); -GGML_OP_CONVERTER(translate_glu_swiglu); -GGML_OP_CONVERTER(translate_glu_swiglu_oai); -GGML_OP_CONVERTER(translate_glu_swiglu_clamp); -GGML_OP_CONVERTER(translate_glu_geglu); -GGML_OP_CONVERTER(translate_glu_geglu_quick); -GGML_OP_CONVERTER(translate_set_rows); -GGML_OP_CONVERTER(translate_cpy); -GGML_OP_CONVERTER(translate_argsort); -GGML_OP_CONVERTER(translate_flash_attn_ext); -GGML_OP_CONVERTER(translate_clamp); -GGML_OP_CONVERTER(translate_pad); -GGML_OP_CONVERTER(translate_ssm_conv); -GGML_OP_CONVERTER(translate_gated_delta_net); -GGML_OP_CONVERTER(translate_repeat); -GGML_OP_CONVERTER(translate_cumsum); -GGML_OP_CONVERTER(translate_fill); -GGML_OP_CONVERTER(translate_set); -GGML_OP_CONVERTER(translate_diag); -GGML_OP_CONVERTER(translate_tri); -GGML_OP_CONVERTER(translate_solve_tri); -GGML_OP_CONVERTER(translate_pool_2d); -GGML_OP_CONVERTER(translate_roll); -GGML_OP_CONVERTER(translate_upscale); -GGML_OP_CONVERTER(translate_mean); -GGML_OP_CONVERTER(translate_sum); -GGML_OP_CONVERTER(translate_unary_gelu); -GGML_OP_CONVERTER(translate_unary_gelu_erf); -GGML_OP_CONVERTER(translate_unary_gelu_quick); -GGML_OP_CONVERTER(translate_unary_elu); -GGML_OP_CONVERTER(translate_unary_hardsigmoid); -GGML_OP_CONVERTER(translate_unary_step); -GGML_OP_CONVERTER(translate_unary_round); -GGML_OP_CONVERTER(translate_unary_expm1); -GGML_OP_CONVERTER(translate_conv_2d); -GGML_OP_CONVERTER(translate_conv_2d_dw); -GGML_OP_CONVERTER(translate_conv_transpose_1d); -GGML_OP_CONVERTER(translate_conv_transpose_2d); -GGML_OP_CONVERTER(translate_conv_3d); - -} // namespace op - -std::unordered_map get_supported_ops(); - -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/pass/fuse_argsort_topk.cpp b/ggml/src/ggml-openvino/openvino/pass/fuse_argsort_topk.cpp deleted file mode 100644 index 9cf54165604c..000000000000 --- a/ggml/src/ggml-openvino/openvino/pass/fuse_argsort_topk.cpp +++ /dev/null @@ -1,70 +0,0 @@ -#include "fuse_argsort_topk.h" - -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace pass { - -FuseArgsortTopK::FuseArgsortTopK() { - // GGML argsort materializes the full ordering before a VIEW keeps the prefix. - // Let TopK produce only that prefix when all index consumers are such views. - auto pattern = ov::pass::pattern::wrap_type(); - - const auto callback = [](ov::pass::pattern::Matcher & m) { - auto topk = ov::as_type_ptr(m.get_match_root()); - if (!topk->output(0).get_target_inputs().empty() || topk->output(1).get_target_inputs().empty() || - topk->get_sort_type() != ov::op::v11::TopK::SortType::SORT_VALUES) { - return false; - } - - const auto shape = topk->get_output_partial_shape(1); - if (shape.rank().is_dynamic()) { - return false; - } - const int64_t rank = shape.rank().get_length(); - const int64_t axis = topk->get_axis(); - if (shape[axis].is_dynamic()) { - return false; - } - - int64_t prefix = -1; - for (const auto & input : topk->output(1).get_target_inputs()) { - const auto * slice = ov::as_type(input.get_node()); - if (!slice || input.get_index() != 0 || slice->get_input_size() != 5) { - return false; - } - int64_t values[4]; - for (size_t i = 0; i < 4; ++i) { - auto value = ov::as_type_ptr(slice->get_input_node_shared_ptr(i + 1)); - if (!value || ov::shape_size(value->get_shape()) != 1) { - return false; - } - values[i] = value->cast_vector()[0]; - } - const int64_t slice_axis = values[3] < 0 ? values[3] + rank : values[3]; - if (values[0] != 0 || values[2] != 1 || slice_axis != axis || values[1] <= 0 || - values[1] >= shape[axis].get_length() || (prefix != -1 && prefix != values[1])) { - return false; - } - prefix = values[1]; - } - - // ggml_argsort_top_k sorts the full row, then exposes only its first k indices. - // Keep other TopK forms unchanged because their ordering can be observable. - topk->set_argument(1, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{}, {prefix})); - topk->validate_and_infer_types(); - return true; - }; - - register_matcher(std::make_shared(pattern, "ov::frontend::ggml::pass::FuseArgsortTopK"), callback); -} - -} // namespace pass -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/pass/fuse_argsort_topk.h b/ggml/src/ggml-openvino/openvino/pass/fuse_argsort_topk.h deleted file mode 100644 index c248ebed35a6..000000000000 --- a/ggml/src/ggml-openvino/openvino/pass/fuse_argsort_topk.h +++ /dev/null @@ -1,19 +0,0 @@ -#pragma once - -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace pass { - -class FuseArgsortTopK : public ov::pass::MatcherPass { -public: - OPENVINO_MATCHER_PASS_RTTI("ov::frontend::ggml::pass::FuseArgsortTopK") - FuseArgsortTopK(); -}; - -} // namespace pass -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/pass/fuse_moe_compressed.cpp b/ggml/src/ggml-openvino/openvino/pass/fuse_moe_compressed.cpp deleted file mode 100644 index db8fc6133488..000000000000 --- a/ggml/src/ggml-openvino/openvino/pass/fuse_moe_compressed.cpp +++ /dev/null @@ -1,516 +0,0 @@ -#include "fuse_moe_compressed.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "../op/gather_matmul.hpp" -#include "../op/moe_compressed.hpp" - -namespace ov { -namespace frontend { -namespace ggml { -namespace pass { - -namespace { - -struct dequant_inputs { - ov::Output weight; - ov::Output scale; - ov::Output zp; - bool has_zp = false; - bool ok = false; -}; - -// Peel the chain built by make_int4_weights/make_int8_weights back to its Constant inputs. -// Grouped weights keep the pre-Reshape rank-4 form [n_expert, n, k/group, group] with scale -// and zp at [n_expert, n, k/group, 1], which is the layout MOECompressed expects. Channel-wise -// weights stay rank-3 with a rank-3 scale and carry no zp. -dequant_inputs unwrap_dequant(const ov::Output & b) { - dequant_inputs res; - - auto node = b.get_node_shared_ptr(); - while (ov::is_type(node) || ov::is_type(node)) { - node = node->get_input_node_shared_ptr(0); - } - - auto mul = ov::as_type_ptr(node); - if (!mul) { - return res; - } - res.scale = mul->input_value(1); - - auto lhs = mul->get_input_node_shared_ptr(0); - if (auto sub = ov::as_type_ptr(lhs)) { - // Take the zero point down to its Constant: an integer zp is wrapped in a Convert to f16, - // and the op wants the integer form. A natively quantized expert instead carries an exact - // f16 zp (-min/scale) with no integer behind it, which the MoE kernel does not accept. - auto zp_node = sub->get_input_node_shared_ptr(1); - while (ov::is_type(zp_node)) { - zp_node = zp_node->get_input_node_shared_ptr(0); - } - res.zp = zp_node->output(0); - res.has_zp = true; - lhs = sub->get_input_node_shared_ptr(0); - } - while (ov::is_type(lhs)) { - lhs = lhs->get_input_node_shared_ptr(0); - } - if (!ov::is_type(lhs)) { - return res; - } - - res.weight = lhs->output(0); - res.ok = res.scale.get_partial_shape().is_static() && res.weight.get_partial_shape().is_static(); - return res; -} - -size_t logical_k(const ov::Shape & shape) { - return shape.size() == 4 ? shape[2] * shape[3] : shape.back(); -} - -// Slice a [n_expert, m, ...] compressed weight/scale/zp Constant along axis 1 (m), [begin, end). -// Built by copying raw bytes directly instead of a graph Slice op: CommonOptimizations rewrites -// v8::Slice into v1::StridedSlice for constant folding regardless of disable_constant_folding -// (the rewrite doesn't carry the marking over), and that reference evaluator crashes on -// sub-byte (u4/i4) element types. Each axis-1 "row" (the trailing dims) is confirmed -// byte-aligned here -- weight/scale/zp trailing sizes are always whole groups -- so a -// byte-range memcpy per row is exact for both regular and sub-byte element types. -ov::Output slice_experts_dim1(const ov::Output & tensor, int64_t begin, int64_t end) { - auto constant = ov::as_type_ptr(tensor.get_node_shared_ptr()); - OPENVINO_ASSERT(constant, "slice_experts_dim1 expects a Constant input"); - - const auto & shape = constant->get_shape(); - OPENVINO_ASSERT(shape.size() >= 2, "slice_experts_dim1 expects rank >= 2"); - const auto & type = constant->get_element_type(); - - size_t row_elems = 1; - for (size_t i = 2; i < shape.size(); ++i) { - row_elems *= shape[i]; - } - const size_t row_bits = row_elems * type.bitwidth(); - OPENVINO_ASSERT(row_bits % 8 == 0, "slice_experts_dim1: row is not byte-aligned"); - const size_t row_bytes = row_bits / 8; - - ov::Shape out_shape = shape; - out_shape[1] = static_cast(end - begin); - - std::vector out_data(shape[0] * static_cast(end - begin) * row_bytes); - const auto * src = static_cast(constant->get_data_ptr()); - const size_t src_row_bytes = shape[1] * row_bytes; - for (size_t e = 0; e < shape[0]; ++e) { - std::memcpy(out_data.data() + e * static_cast(end - begin) * row_bytes, - src + e * src_row_bytes + static_cast(begin) * row_bytes, - static_cast(end - begin) * row_bytes); - } - - return std::make_shared(type, out_shape, out_data.data()); -} - -} // namespace - -FuseMoeCompressed::FuseMoeCompressed() { - using namespace ov::pass::pattern; - - // The gate and up projections each get their own Reshape/Transpose of the hidden state and - // their own Reshape of the routing ids, so every branch needs its own sub-pattern. On GPU - // mul_mat_id also converts the activations to f16 before the op and back to f32 after it, - // so those Converts are matched as optional. - auto hidden_gate_m = any_input(); - auto a_gate_reshape_m = wrap_type({ hidden_gate_m, any_input() }); - auto a_gate_m = - wrap_type({ optional({ a_gate_reshape_m }), any_input() }); - auto hidden_up_m = any_input(); - auto a_up_m = wrap_type( - { optional({ wrap_type({ hidden_up_m, any_input() }) }), - any_input() }); - - auto gate_w_m = any_input(); - auto up_w_m = any_input(); - auto down_w_m = any_input(); - auto ids_gate_m = any_input(); - auto ids_up_m = any_input(); - auto ids_down_m = any_input(); - - auto bgm_gate_m = wrap_type({ a_gate_m, gate_w_m, ids_gate_m, any_input() }); - auto gate_u_m = optional({ wrap_type( - { wrap_type({ bgm_gate_m, any_input() }), any_input() }) }); - - auto silu_m = wrap_type({ gate_u_m }); - - auto bgm_up_m = wrap_type({ a_up_m, up_w_m, ids_up_m, any_input() }); - auto up_u_m = optional({ wrap_type( - { wrap_type({ bgm_up_m, any_input() }), any_input() }) }); - auto swiglu_m = wrap_type({ silu_m, up_u_m }); - - auto d_t_m = wrap_type( - { optional({ wrap_type({ swiglu_m, any_input() }) }), - any_input() }); - auto bgm_down_m = wrap_type({ d_t_m, down_w_m, ids_down_m, any_input() }); - auto down_u_m = optional({ wrap_type( - { wrap_type({ bgm_down_m, any_input() }), any_input() }) }); - - auto routing_m = any_input(); - auto weighted_m = wrap_type({ down_u_m, routing_m }); - auto root_m = wrap_type({ weighted_m, any_input() }); - - const auto callback = [=](Matcher & m) { - auto & pm = m.get_pattern_value_map(); - - const auto gate = unwrap_dequant(pm.at(gate_w_m)); - const auto up = unwrap_dequant(pm.at(up_w_m)); - const auto down = unwrap_dequant(pm.at(down_w_m)); - if (!gate.ok || !up.ok || !down.ok) { - return false; - } - - const auto gate_shape = gate.weight.get_shape(); - const auto up_shape = up.weight.get_shape(); - const auto down_shape = down.weight.get_shape(); - if (gate_shape != up_shape || gate_shape.size() < 3 || down_shape.size() < 3) { - return false; - } - - // MOECompressed carries one group_size and one has_zp for all three projections, so a - // model whose down-proj is quantized differently from gate/up cannot be described. This - // happens when ggml requantizes Q5_K/Q6_K experts to channel-wise int8. - if (gate.has_zp != down.has_zp || gate_shape.size() != down_shape.size()) { - return false; - } - - // The kernel only takes an integer zero point (moe_3gemm_swiglu_opt validate_impl). - if (gate.has_zp) { - static const std::set int_zp_types = { ov::element::u4, ov::element::i4, - ov::element::u8, ov::element::i8 }; - if (int_zp_types.count(gate.zp.get_element_type()) == 0 || - int_zp_types.count(down.zp.get_element_type()) == 0) { - return false; - } - } - - // Config holds a single group_size for all three projections. - const auto group_of = [](const dequant_inputs & w) { - const auto s = w.weight.get_shape(); - return s.size() == 4 ? s[3] : logical_k(s); - }; - if (group_of(gate) != group_of(up) || group_of(gate) != group_of(down)) { - return false; - } - - // all three branches must route the same hidden state through the same experts - if (pm.at(hidden_gate_m) != pm.at(hidden_up_m)) { - return false; - } - - auto ids = pm.at(ids_down_m); - const auto ids_pshape = ids.get_partial_shape(); - if (ids_pshape.rank().is_dynamic() || ids_pshape[ids_pshape.rank().get_length() - 1].is_dynamic()) { - return false; - } - const size_t top_k = ids_pshape[ids_pshape.rank().get_length() - 1].get_length(); - - // routing weights arrive as [1, n_tokens, top_k, 1]; the op wants [..., top_k] - auto routing = pm.at(routing_m); - const auto routing_pshape = routing.get_partial_shape(); - if (routing_pshape.rank().is_dynamic() || routing_pshape.rank().get_length() != 4 || - routing_pshape[3] != 1) { - return false; - } - // MOE requires routing weights and ids to have the same shape. Drop the trailing 1 of the - // routing weights and give the ids the leading batch dim, so both become [1, n_tokens, top_k]. - routing = std::make_shared( - routing, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{ 1 }, { 3 })); - if (ids_pshape.rank().get_length() == 2) { - ids = std::make_shared( - ids, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{ 1 }, { 0 })); - } - if (routing.get_partial_shape() != ids.get_partial_shape()) { - return false; - } - - const size_t down_k = logical_k(down_shape); - const auto down_scale_shape = down.scale.get_shape(); - const size_t down_groups = down_scale_shape.size() >= 3 ? down_scale_shape[2] : 1; - - ov::op::internal::MOECompressed::Config config; - config.expert_type = ov::op::internal::MOE::Expert_type::GEMM3_SWIGLU; - config.activation_type = ov::op::internal::MOE::Activation_type::SWIGLU; - config.expert_alpha = 0.0f; - config.expert_beta = 1.0f; - config.gate_idx = 0; - config.hidden_size = logical_k(gate_shape); - config.inter_size = gate_shape[1]; - config.num_expert = gate_shape[0]; - config.num_shared_expert = 0; - config.top_k = top_k; - config.group_size = down_groups <= 1 ? std::numeric_limits::max() : down_k / down_groups; - config.has_batch_dim = true; - config.has_zp = gate.has_zp; - // dynamic makes the output follow the hidden state, so the plugin can lower this region - // to f16 together with the rest of the graph - config.out_type = ov::element::dynamic; - - auto absent_zp = [] { - auto zp = std::make_shared(ov::element::dynamic, ov::Shape{ 0 }); - ov::pass::disable_constant_folding(zp); - return zp->output(0); - }; - - // MOE takes its output type from the hidden state. Transpose the activations before the - // f16 Convert that mul_mat_id adds on GPU, so the op stays f32 like the block it replaces - // and the plugin can lower the whole region uniformly. - const auto a_transpose = pm.at(a_gate_m).get_node_shared_ptr(); - ov::Output hidden = - std::make_shared(pm.at(a_gate_reshape_m), a_transpose->input_value(1)); - - const ov::OutputVector args = { - hidden, routing, ids, - gate.weight, gate.scale, gate.has_zp ? gate.zp : absent_zp(), - up.weight, up.scale, up.has_zp ? up.zp : absent_zp(), - down.weight, down.scale, down.has_zp ? down.zp : absent_zp(), - }; - - auto moe = std::make_shared(args, config); - - // MOE takes its output type from the hidden state, which is f16 on GPU, while the rest of - // the ggml graph works in f32. - ov::Output result = moe->output(0); - const auto root_type = m.get_match_root()->get_output_element_type(0); - if (result.get_element_type() != root_type) { - result = std::make_shared(result, root_type); - } - - result.get_node_shared_ptr()->set_friendly_name(m.get_match_root()->get_friendly_name()); - ov::copy_runtime_info(m.get_matched_nodes(), result.get_node_shared_ptr()); - ov::replace_node(m.get_match_root(), result.get_node_shared_ptr()); - register_new_node(moe); - return true; - }; - - register_matcher(std::make_shared(root_m, "ov::frontend::ggml::pass::FuseMoeCompressed"), callback); -} - -FuseMoeCompressedFusedGateUp::FuseMoeCompressedFusedGateUp() { - using namespace ov::pass::pattern; - - // A single GatherMatmul computes the fused gate+up projection; the gate/up split happens - // AFTER the GEMM via two Slice ops on the last (feature) axis (see get_glu_inputs in - // op/glu_geglu.cpp), unlike FuseMoeCompressed's two-separate-GatherMatmul models. - auto hidden_m = any_input(); - auto a_reshape_m = wrap_type({ hidden_m, any_input() }); - auto a_m = wrap_type({ optional({ a_reshape_m }), any_input() }); - - auto gate_up_w_m = any_input(); - auto ids_gate_up_m = any_input(); - auto bgm_fused_m = wrap_type({ a_m, gate_up_w_m, ids_gate_up_m, any_input() }); - auto gu_u_m = optional({ wrap_type( - { wrap_type({ bgm_fused_m, any_input() }), any_input() }) }); - - auto gate_slice_m = wrap_type({ gu_u_m, any_input(), any_input(), any_input(), any_input() }); - auto up_slice_m = wrap_type({ gu_u_m, any_input(), any_input(), any_input(), any_input() }); - auto gelu_m = wrap_type({ gate_slice_m }); - auto geglu_m = wrap_type({ gelu_m, up_slice_m }); - - auto d_t_m = wrap_type( - { optional({ wrap_type({ geglu_m, any_input() }) }), - any_input() }); - auto down_w_m = any_input(); - auto ids_down_m = any_input(); - auto bgm_down_m = wrap_type({ d_t_m, down_w_m, ids_down_m, any_input() }); - auto down_u_m = optional({ wrap_type( - { wrap_type({ bgm_down_m, any_input() }), any_input() }) }); - - // gemma-4 applies an extra per-expert output scale to the down projection before the - // router-weight multiply (llama-graph.cpp's ffn_down_exps.scale); FuseMoeCompressed's - // Qwen-shaped models have no such scale, so this node is specific to this pattern. - auto down_scale_m = any_input(); - auto down_scaled_m = wrap_type({ down_u_m, down_scale_m }); - - auto routing_m = any_input(); - auto weighted_m = wrap_type({ down_scaled_m, routing_m }); - auto root_m = wrap_type({ weighted_m, any_input() }); - - const auto callback = [=](Matcher & m) { - auto & pm = m.get_pattern_value_map(); - - const auto gate_up = unwrap_dequant(pm.at(gate_up_w_m)); - const auto down = unwrap_dequant(pm.at(down_w_m)); - if (!gate_up.ok || !down.ok) { - return false; - } - - // oneDNN's weight-decompression GEMM needs a real zero point, so a symmetric (no zp) - // projection cannot be fused here -- e.g. the GPU-only Q4_0 fallback in - // ggml_openvino_get_extracted_layout for grouped 8-bit expert weights. - if (!gate_up.has_zp || !down.has_zp) { - return false; - } - - const auto gate_up_shape = gate_up.weight.get_shape(); - const auto down_shape = down.weight.get_shape(); - if (gate_up_shape.size() < 3 || down_shape.size() < 3 || gate_up_shape.size() != down_shape.size()) { - return false; - } - - // The op reads the zero point straight off a weight port, so it must already be an - // integer Constant. Requantized experts (GGML_OPENVINO_REQUANT_KQUANT=q4_asym64_all) - // are; a native Q4_K expert keeps an exact f16 zp and is left to the unfused path - // rather than rounded here -- rounding would put a live node on that port and the - // expert GEMM would read the wrong zero point. - static const std::set int_types = { ov::element::u4, ov::element::i4, - ov::element::u8, ov::element::i8 }; - if (int_types.count(gate_up.weight.get_element_type()) == 0 || - int_types.count(down.weight.get_element_type()) == 0 || - int_types.count(gate_up.zp.get_element_type()) == 0 || - int_types.count(down.zp.get_element_type()) == 0) { - return false; - } - - const auto group_of = [](const dequant_inputs & w) { - const auto s = w.weight.get_shape(); - return s.size() == 4 ? s[3] : logical_k(s); - }; - if (group_of(gate_up) != group_of(down)) { - return false; - } - - auto gelu_node = ov::as_type_ptr(pm.at(gelu_m).get_node_shared_ptr()); - if (!gelu_node || gelu_node->get_approximation_mode() != ov::op::GeluApproximationMode::ERF) { - return false; - } - - // Read the actual gate/up split point from the matched Slice nodes instead of assuming - // inter_size/2, so this stays correct if the model's ffn_dim convention ever changes. - auto gate_slice = ov::as_type_ptr(pm.at(gate_slice_m).get_node_shared_ptr()); - auto up_slice = ov::as_type_ptr(pm.at(up_slice_m).get_node_shared_ptr()); - auto gate_end_c = ov::as_type_ptr(gate_slice->input_value(2).get_node_shared_ptr()); - auto up_begin_c = ov::as_type_ptr(up_slice->input_value(1).get_node_shared_ptr()); - if (!gate_end_c || !up_begin_c) { - return false; - } - const auto gate_end = gate_end_c->cast_vector().at(0); - const auto up_begin = up_begin_c->cast_vector().at(0); - if (gate_end != up_begin || gate_end <= 0 || gate_end >= static_cast(gate_up_shape[1])) { - return false; - } - const int64_t mid = gate_end; - const int64_t full = static_cast(gate_up_shape[1]); - - auto ids = pm.at(ids_down_m); - const auto ids_pshape = ids.get_partial_shape(); - if (ids_pshape.rank().is_dynamic() || ids_pshape[ids_pshape.rank().get_length() - 1].is_dynamic()) { - return false; - } - const size_t top_k = ids_pshape[ids_pshape.rank().get_length() - 1].get_length(); - - // routing weights arrive as [1, n_tokens, top_k, 1]; the op wants [..., top_k] - auto routing = pm.at(routing_m); - const auto routing_pshape = routing.get_partial_shape(); - if (routing_pshape.rank().is_dynamic() || routing_pshape.rank().get_length() != 4 || - routing_pshape[3] != 1) { - return false; - } - // Fold gemma-4's per-expert output scale into the routing weights: the reduction is - // sum_e(routing[e] * scale[e] * down_out[e]), and MOECompressed only takes one - // per-expert weight, so pre-multiply it into routing here (same [.., top_k, 1] shape). - auto down_scale = pm.at(down_scale_m); - if (down_scale.get_partial_shape() != routing_pshape) { - return false; - } - routing = std::make_shared(routing, down_scale); - routing = std::make_shared( - routing, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{ 1 }, { 3 })); - if (ids_pshape.rank().get_length() == 2) { - ids = std::make_shared( - ids, ov::op::v0::Constant::create(ov::element::i64, ov::Shape{ 1 }, { 0 })); - } - if (routing.get_partial_shape() != ids.get_partial_shape()) { - return false; - } - - const size_t down_k = logical_k(down_shape); - const auto down_scale_shape = down.scale.get_shape(); - const size_t down_groups = down_scale_shape.size() >= 3 ? down_scale_shape[2] : 1; - - // Split the fused gate_up weight/scale/zp on the output axis. slice_experts_dim1 copies - // raw bytes out of the Constant, so each half stays a Constant -- which is what the op - // needs on its weight ports. - const ov::Output gate_weight = slice_experts_dim1(gate_up.weight, 0, mid); - const ov::Output up_weight = slice_experts_dim1(gate_up.weight, mid, full); - const ov::Output gate_scale = slice_experts_dim1(gate_up.scale, 0, mid); - const ov::Output up_scale = slice_experts_dim1(gate_up.scale, mid, full); - const ov::Output gate_zp = slice_experts_dim1(gate_up.zp, 0, mid); - const ov::Output up_zp = slice_experts_dim1(gate_up.zp, mid, full); - - ov::op::internal::MOECompressed::Config config; - config.expert_type = ov::op::internal::MOE::Expert_type::GEMM3_SWIGLU; - config.activation_type = ov::op::internal::MOE::Activation_type::GEGLU_ERF; - config.expert_alpha = 0.0f; - config.expert_beta = 1.0f; - config.gate_idx = 0; - config.hidden_size = logical_k(gate_up_shape); - config.inter_size = static_cast(mid); - config.num_expert = gate_up_shape[0]; - config.num_shared_expert = 0; - config.top_k = top_k; - config.group_size = down_groups <= 1 ? std::numeric_limits::max() : down_k / down_groups; - config.has_batch_dim = true; - config.has_zp = true; - config.out_type = ov::element::dynamic; - - // Rebuild the activations Transpose before mul_mat_id's f16 GPU Convert, so the op - // stays f32 like the block it replaces (mirrors FuseMoeCompressed). - const auto a_transpose = pm.at(a_m).get_node_shared_ptr(); - ov::Output hidden = - std::make_shared(pm.at(a_reshape_m), a_transpose->input_value(1)); - - // w0 is the activated (gate) lane and w1 the multiplied (up) lane -- see - // moe_3gemm_swiglu_opt.cpp: "scratch.up = up(x) * silu(gate(x))". - const ov::OutputVector args = { - hidden, routing, ids, - gate_weight, gate_scale, gate_zp, - up_weight, up_scale, up_zp, - down.weight, down.scale, down.zp, - }; - - auto moe = std::make_shared(args, config); - - ov::Output result = moe->output(0); - const auto root_type = m.get_match_root()->get_output_element_type(0); - if (result.get_element_type() != root_type) { - result = std::make_shared(result, root_type); - } - - result.get_node_shared_ptr()->set_friendly_name(m.get_match_root()->get_friendly_name()); - ov::copy_runtime_info(m.get_matched_nodes(), result.get_node_shared_ptr()); - ov::replace_node(m.get_match_root(), result.get_node_shared_ptr()); - register_new_node(moe); - return true; - }; - - register_matcher( - std::make_shared(root_m, "ov::frontend::ggml::pass::FuseMoeCompressedFusedGateUp"), callback); -} - -} // namespace pass -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/pass/fuse_moe_compressed.h b/ggml/src/ggml-openvino/openvino/pass/fuse_moe_compressed.h deleted file mode 100644 index 5c574ebad1ae..000000000000 --- a/ggml/src/ggml-openvino/openvino/pass/fuse_moe_compressed.h +++ /dev/null @@ -1,30 +0,0 @@ -#include "openvino/pass/matcher_pass.hpp" - -namespace ov { -namespace frontend { -namespace ggml { -namespace pass { - -// Folds the MoE expert block emitted for MUL_MAT_ID (3 GatherMatmul + SwiGLU + routing -// weighting + expert reduction) into a single ov::op::internal::MOECompressed. -class FuseMoeCompressed : public ov::pass::MatcherPass { -public: - OPENVINO_MATCHER_PASS_RTTI("ov::frontend::ggml::pass::FuseMoeCompressed") - FuseMoeCompressed(); -}; - -// Folds the MoE expert block emitted for a model whose gate and up projections share one -// fused MUL_MAT_ID weight (gemma-4: one GatherMatmul + Slice/Slice split, GEGLU activation) -// into a single ov::op::internal::MOECompressed, GEMM3_SWIGLU/GEGLU_ERF. Splits the fused -// weight/scale/zp into gate/up halves so it lands on the same 3-GEMM expert-grouped kernel -// FuseMoeCompressed uses for models with separate gate/up weights. -class FuseMoeCompressedFusedGateUp : public ov::pass::MatcherPass { -public: - OPENVINO_MATCHER_PASS_RTTI("ov::frontend::ggml::pass::FuseMoeCompressedFusedGateUp") - FuseMoeCompressedFusedGateUp(); -}; - -} // namespace pass -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/pass/fuse_moe_router.cpp b/ggml/src/ggml-openvino/openvino/pass/fuse_moe_router.cpp deleted file mode 100644 index d6ed7f551f33..000000000000 --- a/ggml/src/ggml-openvino/openvino/pass/fuse_moe_router.cpp +++ /dev/null @@ -1,153 +0,0 @@ -#include "fuse_moe_router.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace pass { - -namespace { - -bool constant_is(const ov::Output & output, const std::vector & values) { - auto node = ov::as_type_ptr(output.get_node_shared_ptr()); - return node && node->get_element_type().is_integral_number() && node->cast_vector() == values; -} - -} // namespace - -FuseMoeRouter::FuseMoeRouter() { - using namespace ov::pass::pattern; - using namespace ov::op; - - // Match the GGML rank-4 routing chain. The GPU plugin recognizes the resulting - // Softmax -> TopK -> ReduceSum -> Divide subgraph as MoERouterFused. - auto logits = wrap_type(); - auto softmax = wrap_type({logits}); - auto topk = wrap_type({softmax, any_input()}); - topk->set_output_size(2); - auto ids = wrap_type({topk->output(1), any_input(), any_input(), any_input(), any_input()}); - auto ids_2d = wrap_type({ids, any_input()}); - auto probs = wrap_type({softmax, any_input()}); - auto data = wrap_type({probs, any_input()}); - auto data_batch = wrap_type({wrap_type({data}), any_input(), any_input()}); - auto ids_count = wrap_type({wrap_type({ids_2d}), any_input(), any_input()}); - auto target = wrap_type({data_batch, ids_count}); - auto broadcast = wrap_type({ids_2d, target}); - auto gather = wrap_type({data, broadcast, any_input()}); - auto weights_4d = wrap_type({gather, any_input()}); - auto weights = wrap_type({weights_4d, any_input()}); - auto sum = wrap_type({weights, any_input()}); - auto clamp = wrap_type({sum}); - auto tile = wrap_type({clamp, any_input()}); - auto norm = wrap_type({weights, tile}); - - const auto callback = [=](Matcher & m) { - const auto & pm = m.get_pattern_value_map(); - const auto node = [&](const std::shared_ptr & p) { return pm.at(p).get_node_shared_ptr(); }; - const auto input_is = [&](const std::shared_ptr & p, size_t i, const std::vector & v) { - return constant_is(node(p)->input_value(i), v); - }; - auto mm = ov::as_type_ptr(node(logits)); - auto sm = ov::as_type_ptr(node(softmax)); - auto tk = ov::as_type_ptr(node(topk)); - auto reduce = ov::as_type_ptr(node(sum)); - auto limit = ov::as_type_ptr(node(clamp)); - const auto shape = mm->get_output_partial_shape(0); - if (shape.rank() != 4 || shape[0] != 1 || shape[1] != 1 || shape[3].is_dynamic() || - mm->get_transpose_a() || mm->get_input_partial_shape(0).rank() != 4 || - mm->get_input_partial_shape(1).rank() != 2 || (sm->get_axis() != -1 && sm->get_axis() != 3) || - tk->get_axis() != 3 || tk->get_mode() != v11::TopK::Mode::MAX || - tk->get_sort_type() != v11::TopK::SortType::SORT_VALUES || tk->get_stable() || - tk->get_index_element_type() != ov::element::i32 || - !tk->output(0).get_target_inputs().empty()) { - return false; - } - const int64_t experts = shape[3].get_length(); - auto end = ov::as_type_ptr(node(ids)->get_input_node_shared_ptr(2)); - if (!end || !end->get_element_type().is_integral_number() || ov::shape_size(end->get_shape()) != 1) { - return false; - } - const int64_t k = end->cast_vector()[0]; - const auto sorted_shape = tk->get_output_partial_shape(1); - if (k <= 0 || k > experts || sorted_shape[3].is_dynamic() || sorted_shape[3].get_length() < k || - !input_is(probs, 1, {1, -1, experts, 1}) || - !input_is(data, 1, {0}) || !input_is(ids_2d, 1, {0, 1}) || - !input_is(weights_4d, 1, {0}) || !input_is(weights, 1, {1, 1, -1, k}) || - ov::as_type_ptr(node(probs))->get_special_zero() || - ov::as_type_ptr(node(weights))->get_special_zero() || - !input_is(gather, 2, {1}) || ov::as_type_ptr(node(gather))->get_batch_dims() != 1 || - !input_is(data_batch, 1, {0}) || !input_is(data_batch, 2, {0}) || - !input_is(ids_count, 1, {1}) || !input_is(ids_count, 2, {0}) || - ov::as_type_ptr(node(data_batch))->get_batch_dims() != 0 || - ov::as_type_ptr(node(ids_count))->get_batch_dims() != 0 || - ov::as_type_ptr(node(target))->get_axis() != 0 || - ov::as_type_ptr(node(broadcast))->get_broadcast_spec().m_type != ov::op::BroadcastType::BIDIRECTIONAL || - !reduce->get_keep_dims() || (!input_is(sum, 1, {-1}) && !input_is(sum, 1, {3})) || - !input_is(tile, 1, {1, 1, 1, k}) || - ov::as_type_ptr(node(norm))->get_autob().m_type != ov::op::AutoBroadcastType::NUMPY) { - return false; - } - // The top-k softmax sum is at least k / experts. Leave margin for rounding. - // This guard is only valid for unbiased softmax routing. - if (!(limit->get_min() <= 0.5 * double(k) / experts && limit->get_max() >= 2.0)) { - return false; - } - std::vector> slices; - for (const auto & input : tk->output(1).get_target_inputs()) { - auto slice = ov::as_type_ptr(input.get_node()->shared_from_this()); - if (!slice || input.get_index() != 0 || slice->get_input_size() != 5 || - !constant_is(slice->input_value(1), {0}) || !constant_is(slice->input_value(2), {k}) || - !constant_is(slice->input_value(3), {1}) || - (!constant_is(slice->input_value(4), {3}) && !constant_is(slice->input_value(4), {-1}))) { - return false; - } - slices.push_back(slice); - } - - // Remove GGML's leading singleton dimension while building the plugin pattern, - // then restore it on both outputs for the following GGML nodes. - auto axis0 = v0::Constant::create(ov::element::i64, ov::Shape{1}, {0}); - auto hidden = std::make_shared(mm->input_value(0), axis0); - auto routing = std::make_shared(hidden, mm->input_value(1), false, mm->get_transpose_b()); - auto probabilities = std::make_shared(routing, -1); - auto selected = std::make_shared(probabilities, - v0::Constant::create(ov::element::i64, ov::Shape{}, {k}), 2, - v11::TopK::Mode::MAX, v11::TopK::SortType::SORT_VALUES, tk->get_index_element_type(), false); - auto total = std::make_shared(selected->output(0), - v0::Constant::create(ov::element::i64, ov::Shape{1}, {-1}), true); - auto normalized = std::make_shared(selected->output(0), total); - auto weights_out = std::make_shared(normalized, axis0); - auto ids_out = std::make_shared(selected->output(1), axis0); - ov::copy_runtime_info({mm, sm, tk, node(norm)}, {hidden, routing, probabilities, selected, total, normalized, weights_out, ids_out}); - ov::replace_node(node(norm), weights_out); - for (const auto & slice : slices) { - ov::replace_node(slice, ids_out); - } - return true; - }; - register_matcher(std::make_shared(norm, "ov::frontend::ggml::pass::FuseMoeRouter"), callback); -} - -} // namespace pass -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/pass/fuse_moe_router.h b/ggml/src/ggml-openvino/openvino/pass/fuse_moe_router.h deleted file mode 100644 index 49e01f16de08..000000000000 --- a/ggml/src/ggml-openvino/openvino/pass/fuse_moe_router.h +++ /dev/null @@ -1,19 +0,0 @@ -#pragma once - -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace pass { - -class FuseMoeRouter : public ov::pass::MatcherPass { -public: - OPENVINO_MATCHER_PASS_RTTI("ov::frontend::ggml::pass::FuseMoeRouter") - FuseMoeRouter(); -}; - -} // namespace pass -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.cpp b/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.cpp deleted file mode 100644 index 6fffd6718a91..000000000000 --- a/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.cpp +++ /dev/null @@ -1,283 +0,0 @@ -#include "fuse_to_conv.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace opp = ov::pass::pattern; - -namespace ov { -namespace frontend { -namespace ggml { -namespace pass { - -FuseToConv::FuseToConv() { - const auto m_in0 = opp::any_input(); - const auto m_in1 = opp::any_input(); - const auto m_matmul = opp::wrap_type({m_in0, m_in1}); - - const auto callback = [=](opp::Matcher & m) { - auto matmul_node = ov::as_type_ptr(m.get_match_root()); - if (!matmul_node) { - return false; - } - - auto unwrap = [](ov::Output n) { - while (ov::is_type(n.get_node_shared_ptr()) || - ov::is_type(n.get_node_shared_ptr())) { - n = n.get_node_shared_ptr()->input_value(0); - } - return n; - }; - - auto get_im2col = [&](ov::Output trace) - -> std::pair, std::shared_ptr> { - auto t2 = ov::as_type_ptr(unwrap(std::move(trace)).get_node_shared_ptr()); - auto r1 = t2 ? ov::as_type_ptr(t2->get_input_node_shared_ptr(0)) : nullptr; - auto t1 = r1 ? ov::as_type_ptr(r1->get_input_node_shared_ptr(0)) : nullptr; - auto eip = - t1 ? ov::as_type_ptr(t1->get_input_node_shared_ptr(0)) : nullptr; - auto pad = eip ? ov::as_type_ptr(eip->get_input_node_shared_ptr(0)) : nullptr; - return {eip, pad}; - }; - - bool weight_is_in0 = true; - auto [eip, pad] = get_im2col(matmul_node->input_value(1)); - ov::Output w_trace = matmul_node->input_value(0); - if (!pad) { - std::tie(eip, pad) = get_im2col(matmul_node->input_value(0)); - w_trace = matmul_node->input_value(1); - weight_is_in0 = false; - if (!pad) { - return false; - } - } - - auto pb_const = ov::as_type_ptr(pad->get_input_node_shared_ptr(1)); - auto pe_const = ov::as_type_ptr(pad->get_input_node_shared_ptr(2)); - if (!pb_const || !pe_const) { - return false; - } - - const auto pb = pb_const->cast_vector(); - const auto pe = pe_const->cast_vector(); - if (pb.size() < 4 || pe.size() < 4) { - return false; - } - - auto image_input = pad->input_value(0); - auto image_shape = image_input.get_partial_shape(); - if (image_shape.rank() != 4 || image_shape[1].is_dynamic()) { - return false; - } - const size_t IC = static_cast(image_shape[1].get_length()); - - w_trace = unwrap(w_trace); - auto weight_pshape = w_trace.get_partial_shape(); - if (!weight_pshape.is_static()) { - return false; - } - - const auto & ws = weight_pshape.to_shape(); - const size_t KH = eip->get_sizes()[0]; - const size_t KW = eip->get_sizes()[1]; - const size_t kernel_spatial_ic = IC * KH * KW; - - size_t groups = 0; - if (IC == 1 && image_shape[0].is_static() && image_shape[2].is_static() && image_shape[3].is_static()) { - if ((ws.size() == 4 && ws[1] == 1 && ws[2] == KH && ws[3] == KW && ws[0] > 1) || - (ws.size() == 2 && ws[1] == KH * KW && ws[0] > 1) || - (ws.size() == 3 && ws[1] == 1 && ws[2] == KH * KW && ws[0] > 1)) { - groups = ws[0]; - } else if (ws.size() == 4 && ws[0] == 1 && ws[2] == 1 && ws[3] == KH * KW && ws[1] > 1) { - groups = ws[1]; - } - } - - const bool is_depthwise = groups > 1 && (image_shape[0].get_length() % groups == 0); - - ov::Output conv_out; - size_t OC = 0; - - if (is_depthwise) { - const size_t N = static_cast(image_shape[0].get_length()) / groups; - const size_t IH = static_cast(image_shape[2].get_length()); - const size_t IW = static_cast(image_shape[3].get_length()); - OC = groups; - - auto img_shape_const = register_new_node( - ov::element::i64, ov::Shape{4}, - std::vector{static_cast(N), static_cast(groups), static_cast(IH), - static_cast(IW)}); - ov::Output image_reshaped = - register_new_node(image_input, img_shape_const, false); - - const ov::Shape conv_w_shape = {groups, 1, 1, KH, KW}; - ov::Output weight_input; - if (auto weight_const = ov::as_type_ptr(w_trace.get_node_shared_ptr())) { - weight_input = register_new_node(weight_const->get_element_type(), conv_w_shape, - weight_const->get_data_ptr()); - } else { - auto shape_const = register_new_node( - ov::element::i64, ov::Shape{5}, - std::vector{static_cast(groups), 1, 1, static_cast(KH), - static_cast(KW)}); - weight_input = register_new_node(w_trace, shape_const, false); - } - - if (weight_input.get_element_type() != image_reshaped.get_element_type()) { - weight_input = register_new_node(weight_input, image_reshaped.get_element_type()); - } - - conv_out = register_new_node( - image_reshaped, weight_input, eip->get_strides(), ov::CoordinateDiff{pb[2], pb[3]}, - ov::CoordinateDiff{pe[2], pe[3]}, ov::Strides{eip->get_rates()[0], eip->get_rates()[1]}, - ov::op::PadType::EXPLICIT); - } else { - if ((ws.size() == 4 && ws[1] == IC && ws[2] == KH && ws[3] == KW) || - (ws.size() == 3 && ws[1] == IC && ws[2] == KW) || (ws.size() == 2 && ws[1] == kernel_spatial_ic)) { - OC = ws[0]; - } else if ((ws.size() == 4 && ws[0] == 1 && ws[2] == IC && ws[3] == KW) || - (ws.size() == 2 && ws[0] == kernel_spatial_ic)) { - OC = ws[1]; - } else if (ws.size() == 3 && ws[0] == 1 && ws[1] == IC && ws[2] == KW) { - OC = 1; - } else if (ws.size() == 4 && ws[3] == kernel_spatial_ic) { - OC = ws[2]; - } else if (ws.size() == 4 && ws[2] == kernel_spatial_ic) { - OC = ws[3]; - } else if (kernel_spatial_ic && ov::shape_size(ws) % kernel_spatial_ic == 0) { - OC = ov::shape_size(ws) / kernel_spatial_ic; - } else { - return false; - } - - const ov::Shape conv_w_shape = {OC, IC, KH, KW}; - - ov::Output weight_input; - if (auto weight_const = ov::as_type_ptr(w_trace.get_node_shared_ptr())) { - weight_input = register_new_node(weight_const->get_element_type(), conv_w_shape, - weight_const->get_data_ptr()); - } else { - auto shape_const = register_new_node( - ov::element::i64, ov::Shape{4}, - std::vector{static_cast(OC), static_cast(IC), static_cast(KH), - static_cast(KW)}); - weight_input = register_new_node(w_trace, shape_const, false); - } - - if (weight_input.get_element_type() != image_input.get_element_type()) { - weight_input = register_new_node(weight_input, image_input.get_element_type()); - } - - conv_out = register_new_node( - image_input, weight_input, eip->get_strides(), ov::CoordinateDiff{pb[2], pb[3]}, - ov::CoordinateDiff{pe[2], pe[3]}, ov::Strides{eip->get_rates()[0], eip->get_rates()[1]}, - ov::op::PadType::EXPLICIT); - } - - constexpr auto target_type = ov::element::f32; - if (conv_out.get_element_type() != target_type) { - conv_out = register_new_node(conv_out, target_type); - } - - std::shared_ptr add_node; - ov::Output bias_input; - - auto try_fuse_bias = [&](const std::shared_ptr & n) { - auto add = ov::as_type_ptr(n); - if (!add) { - return false; - } - for (size_t i = 0; i < 2; ++i) { - if (ov::is_type(add->get_input_node_shared_ptr(i))) { - bias_input = add->input_value(i); - add_node = add; - return true; - } - } - return false; - }; - - for (const auto & consumer : matmul_node->output(0).get_target_inputs()) { - auto n = consumer.get_node()->shared_from_this(); - if (try_fuse_bias(n)) { - break; - } - if (ov::is_type(n) || ov::is_type(n)) { - for (const auto & next : n->output(0).get_target_inputs()) { - if (try_fuse_bias(next.get_node()->shared_from_this())) { - break; - } - } - } - if (add_node) { - break; - } - } - - ov::Output final_out = conv_out; - std::shared_ptr target_node = matmul_node; - - if (add_node) { - ov::Output bias = bias_input; - if (bias.get_element_type() != target_type) { - bias = register_new_node(bias, target_type); - } - auto bias_shape = register_new_node( - ov::element::i64, ov::Shape{4}, std::vector{1, static_cast(OC), 1, 1}); - bias = register_new_node(bias, bias_shape, false); - final_out = register_new_node(conv_out, bias); - target_node = add_node; - } - - if (!is_depthwise) { - auto perm = register_new_node( - ov::element::i64, ov::Shape{4}, - weight_is_in0 ? std::vector{1, 0, 2, 3} : std::vector{0, 2, 3, 1}); - final_out = register_new_node(final_out, perm); - } - - auto orig_shape = target_node->get_output_partial_shape(0); - if (orig_shape.is_static() && final_out.get_partial_shape().is_static()) { - if (ov::shape_size(orig_shape.to_shape()) != ov::shape_size(final_out.get_shape())) { - return false; - } - } - if (orig_shape.is_static() && final_out.get_partial_shape() != orig_shape) { - auto shape_const = register_new_node(ov::element::i64, ov::Shape{orig_shape.size()}, - orig_shape.to_shape()); - final_out = register_new_node(final_out, shape_const, false); - } - - auto orig_type = target_node->get_output_element_type(0); - if (final_out.get_element_type() != orig_type) { - final_out = register_new_node(final_out, orig_type); - } - - final_out.get_node_shared_ptr()->set_friendly_name(target_node->get_friendly_name()); - ov::copy_runtime_info(m.get_matched_nodes(), final_out.get_node_shared_ptr()); - ov::replace_node(target_node, final_out.get_node_shared_ptr()); - - return true; - }; - - register_matcher(std::make_shared(m_matmul, "ov::frontend::ggml::pass::FuseToConv"), callback); -} - -} // namespace pass -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.h b/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.h deleted file mode 100644 index feac14b13ff2..000000000000 --- a/ggml/src/ggml-openvino/openvino/pass/fuse_to_conv.h +++ /dev/null @@ -1,17 +0,0 @@ -#include "openvino/pass/matcher_pass.hpp" - -namespace ov { -namespace frontend { -namespace ggml { -namespace pass { - -class FuseToConv : public ov::pass::MatcherPass { -public: - OPENVINO_MATCHER_PASS_RTTI("ov::frontend::ggml::pass::FuseToConv") - FuseToConv(); -}; - -} // namespace pass -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.cpp b/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.cpp deleted file mode 100644 index 0671542ee383..000000000000 --- a/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.cpp +++ /dev/null @@ -1,60 +0,0 @@ -#include "fuse_to_sdpa.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace pass { - -FuseToSDPA::FuseToSDPA() { - // Not maintained since FLASH_ATTN_EXT has replaced this pattern - const auto m_k = ov::pass::pattern::any_input(); - const auto m_q = ov::pass::pattern::any_input(); - const auto m_qk = ov::pass::pattern::wrap_type({m_q, m_k}); - const auto m_qk_f32 = ov::pass::pattern::wrap_type({m_qk}); - const auto m_scale = ov::pass::pattern::any_input(); - const auto m_scaled_qk = ov::pass::pattern::wrap_type({m_qk_f32, m_scale}); - const auto m_mask = ov::pass::pattern::any_input(); - const auto m_masked_qk = ov::pass::pattern::wrap_type({m_scaled_qk, m_mask}); - const auto m_softmax_qk = ov::pass::pattern::wrap_type({m_masked_qk}); - const auto m_softmax_qk_f16 = ov::pass::pattern::wrap_type({m_softmax_qk}); - const auto m_v = ov::pass::pattern::any_input(); - const auto m_qkv = ov::pass::pattern::wrap_type({m_softmax_qk_f16, m_v}); - - const auto callback = [=](ov::pass::pattern::Matcher & m) { - auto & pattern_to_output = m.get_pattern_value_map(); - auto k = pattern_to_output[m_k]; - auto q = pattern_to_output[m_q]; - auto v = pattern_to_output[m_v]; - auto mask = pattern_to_output[m_mask]; - auto scale = pattern_to_output[m_scale]; - - auto mask_f16 = register_new_node(mask, ov::element::f16); - auto scale_f16 = register_new_node(scale, ov::element::f16); - auto sdpa = std::make_shared(q, k, v, mask_f16, scale_f16, false); - - ov::replace_node(m.get_match_root(), sdpa); - ov::copy_runtime_info(m.get_matched_nodes(), sdpa); - - return true; - }; - register_matcher(std::make_shared(m_qkv, "ov::frontend::ggml::pass::FuseToSDPA"), - callback); -} - -} // namespace pass -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.h b/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.h deleted file mode 100644 index 8b5164d23293..000000000000 --- a/ggml/src/ggml-openvino/openvino/pass/fuse_to_sdpa.h +++ /dev/null @@ -1,17 +0,0 @@ -#include "openvino/pass/matcher_pass.hpp" - -namespace ov { -namespace frontend { -namespace ggml { -namespace pass { - -class FuseToSDPA : public ov::pass::MatcherPass { -public: - OPENVINO_MATCHER_PASS_RTTI("ov::frontend::ggml::pass::FuseToSDPA") - FuseToSDPA(); -}; - -} // namespace pass -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/pass/kv_state_seq_axis.cpp b/ggml/src/ggml-openvino/openvino/pass/kv_state_seq_axis.cpp deleted file mode 100644 index 04de2d008c72..000000000000 --- a/ggml/src/ggml-openvino/openvino/pass/kv_state_seq_axis.cpp +++ /dev/null @@ -1,114 +0,0 @@ -#include "kv_state_seq_axis.h" - -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { -namespace pass { - -namespace { - -const std::vector & seq_axis_perm() { - // [1, seq, n_heads_kv, head_size] <-> [1, n_heads_kv, seq, head_size] - static const std::vector perm{0, 2, 1, 3}; - return perm; -} - -// True when the state still has the frontend's stateful KV layout, so the sequence axis -// can be moved: rank 4, batch and both head dims static, and seq the only dynamic dim, -// at dim 1. Any KV head count is fine. With a single head the rewrite is pure metadata -// ([1, seq, 1, head] and [1, 1, seq, head] are the same memory); with several heads it -// also drops the reader-side transpose of the whole accumulated state, which is where -// most of the gain comes from at depth. -bool can_move_seq_axis(const ov::PartialShape & shape) { - return shape.rank().is_static() && shape.rank().get_length() == 4 && shape[0].is_static() && - shape[1].is_dynamic() && shape[2].is_static() && shape[3].is_static(); -} - -std::shared_ptr match_kv_append(const std::shared_ptr & assign) { - auto concat = ov::as_type_ptr(assign->get_input_node_shared_ptr(0)); - if (!concat || concat->get_input_size() != 2 || concat->get_axis() != 1) { - return nullptr; - } - auto read_value = ov::as_type_ptr(concat->get_input_node_shared_ptr(0)); - if (!read_value || read_value->get_variable() != assign->get_variable()) { - return nullptr; - } - if (!can_move_seq_axis(read_value->get_output_partial_shape(0))) { - return nullptr; - } - return concat; -} - -} // namespace - -bool KVStateSeqAxis::run_on_model(const std::shared_ptr & model) { - std::vector> assigns; - for (const auto & op : model->get_ops()) { - if (auto assign = ov::as_type_ptr(op)) { - assigns.push_back(assign); - } - } - - bool changed = false; - for (const auto & assign : assigns) { - auto concat = match_kv_append(assign); - if (!concat) { - continue; - } - auto read_value = ov::as_type_ptr(concat->get_input_node_shared_ptr(0)); - - auto variable = read_value->get_variable(); - auto info = variable->get_info(); - const auto & shape = info.data_shape; - info.data_shape = ov::PartialShape{shape[0], shape[2], shape[1], shape[3]}; - variable->update(info); - read_value->validate_and_infer_types(); - - auto readers = concat->output(0).get_target_inputs(); - - auto new_rows = concat->input_value(1); - auto perm_in = ov::op::v0::Constant::create(ov::element::i64, {4}, seq_axis_perm()); - concat->set_argument(1, std::make_shared(new_rows, perm_in)); - concat->set_axis(2); - concat->validate_and_infer_types(); - - // Readers still expect seq at dim 1. A reader that is itself the inverse - // Transpose wanted seq at dim 2 all along, so drop it; give anything else the - // inverse Transpose so its input is unchanged. - for (const auto & reader : readers) { - auto * node = reader.get_node(); - if (ov::is_type(node)) { - continue; - } - bool dropped = false; - if (auto * transpose = ov::as_type(node)) { - auto order = ov::as_type_ptr(transpose->get_input_node_shared_ptr(1)); - if (order && order->cast_vector() == seq_axis_perm()) { - ov::replace_output_update_name(transpose->output(0), concat->output(0)); - dropped = true; - } - } - if (!dropped) { - auto perm_out = ov::op::v0::Constant::create(ov::element::i64, {4}, seq_axis_perm()); - reader.replace_source_output(std::make_shared(concat->output(0), perm_out)); - } - } - changed = true; - } - - return changed; -} - -} // namespace pass -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/pass/mark_decompression_convert_constant_folding.h b/ggml/src/ggml-openvino/openvino/pass/mark_decompression_convert_constant_folding.h deleted file mode 100644 index c229e25fb203..000000000000 --- a/ggml/src/ggml-openvino/openvino/pass/mark_decompression_convert_constant_folding.h +++ /dev/null @@ -1,29 +0,0 @@ -#pragma once - -#include "mark_decompression_convert_constant_folding.h" -#include "openvino/core/visibility.hpp" -#include "openvino/pass/matcher_pass.hpp" - -#ifdef OPENVINO_STATIC_LIBRARY -# define TRANSFORMATIONS_API -#else -# ifdef IMPLEMENT_OPENVINO_API -# define TRANSFORMATIONS_API OPENVINO_CORE_EXPORTS -# else -# define TRANSFORMATIONS_API OPENVINO_CORE_IMPORTS -# endif // IMPLEMENT_OPENVINO_API -#endif // OPENVINO_STATIC_LIBRARY - -namespace ov { -namespace pass { - -class TRANSFORMATIONS_API MarkCompressedFloatConstants; - -} // namespace pass -} // namespace ov - -class ov::pass::MarkCompressedFloatConstants : public MatcherPass { -public: - OPENVINO_MATCHER_PASS_RTTI("MarkCompressedFloatConstants") - MarkCompressedFloatConstants(); -}; diff --git a/ggml/src/ggml-openvino/openvino/pass/mark_dequantization_subgraph.h b/ggml/src/ggml-openvino/openvino/pass/mark_dequantization_subgraph.h deleted file mode 100644 index d51303d5b4d4..000000000000 --- a/ggml/src/ggml-openvino/openvino/pass/mark_dequantization_subgraph.h +++ /dev/null @@ -1,44 +0,0 @@ -// Copyright (C) 2018-2026 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// -// Local mirror of OpenVINO's ov::pass::MarkDequantization pass declaration. -// -// The pass body is provided by the linked libopenvino.so; only the declaration is needed here so -// we can register it directly in our own TranslateSession::apply_transformations (same approach as -// MarkCompressedFloatConstants's local mirror in mark_decompression_convert_constant_folding.h). This -// lets us mark our GatherMatmul dequantization chain with disable_constant_folding regardless of the -// CPU/GPU plugin's own is_decompression_multiply() consumer allowlist. -// The class layout must stay in sync with -// openvino/src/common/transformations/include/transformations/low_precision/mark_dequantization_subgraph.hpp - -#pragma once - -#include "openvino/core/type/element_type.hpp" -#include "openvino/core/visibility.hpp" -#include "openvino/pass/matcher_pass.hpp" - -#ifdef OPENVINO_STATIC_LIBRARY -# define TRANSFORMATIONS_API -#else -# ifdef IMPLEMENT_OPENVINO_API -# define TRANSFORMATIONS_API OPENVINO_CORE_EXPORTS -# else -# define TRANSFORMATIONS_API OPENVINO_CORE_IMPORTS -# endif // IMPLEMENT_OPENVINO_API -#endif // OPENVINO_STATIC_LIBRARY - -namespace ov { -namespace pass { - -class TRANSFORMATIONS_API MarkDequantization; - -} // namespace pass -} // namespace ov - -class ov::pass::MarkDequantization : public MatcherPass { -public: - OPENVINO_MATCHER_PASS_RTTI("MarkDequantization") - explicit MarkDequantization(const element::TypeVector & precisions, - bool fold_subtract_const = false, - bool fold_multiply_const = true); -}; diff --git a/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.cpp b/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.cpp deleted file mode 100644 index 09c213f3e17c..000000000000 --- a/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.cpp +++ /dev/null @@ -1,59 +0,0 @@ -#include "squeeze_matmul.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace opp = ov::pass::pattern; - -namespace ov { -namespace frontend { -namespace ggml { -namespace pass { - -// For quantized models, NPUW expects the activation to be 3d in DQ(DynamicQuantization) opt, e.g. DQMatMulGQ2i -SqueezeMatmul::SqueezeMatmul() { - auto m_act = opp::any_input(); - auto m_wei = opp::any_input(); - auto m_matmul = opp::wrap_type({m_act, m_wei}); - - const auto callback = [=](ov::pass::pattern::Matcher & m) { - const auto & pattern_map = m.get_pattern_value_map(); - auto matmul_node = - ov::as_type_ptr(pattern_map.at(m_matmul).get_node_shared_ptr()); - auto act = pattern_map.at(m_act); - auto wei = pattern_map.at(m_wei); - auto act_shape = act.get_partial_shape(); - auto wei_shape = wei.get_partial_shape(); - if (act_shape.rank().is_dynamic() || wei_shape.rank().is_dynamic()) { - return false; - } - if (act_shape.rank().get_length() == 4 && wei_shape.rank().get_length() == 2) { - auto axis = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{1}, {0}); - auto squeezed_act = std::make_shared(act, axis); - auto new_matmul = std::make_shared(squeezed_act, wei, matmul_node->get_transpose_a(), - matmul_node->get_transpose_b()); - auto unsqueezed_output = std::make_shared(new_matmul, axis); - unsqueezed_output->set_friendly_name(matmul_node->get_friendly_name()); - ov::copy_runtime_info(matmul_node, {squeezed_act, new_matmul, unsqueezed_output}); - ov::replace_node(matmul_node, unsqueezed_output); - return true; - } - return false; - }; - - register_matcher(std::make_shared(m_matmul, "ov::frontend::ggml::pass::SqueezeMatmul"), - callback); -} - -} // namespace pass -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.h b/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.h deleted file mode 100644 index f8fbc69d546e..000000000000 --- a/ggml/src/ggml-openvino/openvino/pass/squeeze_matmul.h +++ /dev/null @@ -1,17 +0,0 @@ -#include "openvino/pass/matcher_pass.hpp" - -namespace ov { -namespace frontend { -namespace ggml { -namespace pass { - -class SqueezeMatmul : public ov::pass::MatcherPass { -public: - OPENVINO_MATCHER_PASS_RTTI("ov::frontend::ggml::pass::SqueezeMatmul") - SqueezeMatmul(); -}; - -} // namespace pass -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/rt_info/weightless_caching_attributes.hpp b/ggml/src/ggml-openvino/openvino/rt_info/weightless_caching_attributes.hpp deleted file mode 100644 index f051891c481e..000000000000 --- a/ggml/src/ggml-openvino/openvino/rt_info/weightless_caching_attributes.hpp +++ /dev/null @@ -1,41 +0,0 @@ -// Copyright (C) 2018-2026 Intel Corporation -// SPDX-License-Identifier: Apache-2.0 -// - -#pragma once - -#include -#include -#include - -namespace ov { - -/** - * @brief Holds weightless caching attributes of a single constant. - * - * WeightlessCacheAttribute class represents runtime info attribute that holds - * the values of original size of the constant in bytes and the binary offset of the - * constant's data in the weights file used by the weightless caching mechanism. It's - * not copyable in case the data was changed (the original node was replaced by a new - * one produced during the tranformation pipeline) - in that case weightless caching - * can't be used for that constant. - */ -class OPENVINO_API WeightlessCacheAttribute : public RuntimeAttribute { -public: - OPENVINO_RTTI("WeightlessCacheAttribute", "0", RuntimeAttribute) - - WeightlessCacheAttribute() = delete; - - WeightlessCacheAttribute(size_t original_size, size_t bin_offset, ov::element::Type original_dtype) - : original_size(original_size), - bin_offset(bin_offset), - original_dtype(original_dtype) {} - - bool is_copyable() const override; - - size_t original_size; - size_t bin_offset; - ov::element::Type original_dtype; -}; - -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/translate_session.cpp b/ggml/src/ggml-openvino/openvino/translate_session.cpp deleted file mode 100644 index 5a3d11f2dcdd..000000000000 --- a/ggml/src/ggml-openvino/openvino/translate_session.cpp +++ /dev/null @@ -1,534 +0,0 @@ -#include "translate_session.h" - -#include "ggml-impl.h" -#include "ggml-openvino/ggml-openvino-extra.h" -#include "ggml-openvino/openvino/node_context.h" -#include "ggml-openvino/openvino/utils.h" -#include "input_model.h" -#include "pass/fuse_argsort_topk.h" -#include "pass/fuse_moe_router.h" -#include "pass/fuse_moe_compressed.h" -#include "pass/fuse_to_conv.h" -#include "pass/kv_state_seq_axis.h" -#include "pass/mark_decompression_convert_constant_folding.h" -#include "pass/mark_dequantization_subgraph.h" -#include "pass/squeeze_matmul.h" -#include "rt_info/weightless_caching_attributes.hpp" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { - -using namespace ov::op; - -namespace { - -std::shared_ptr create_parameter(const std::string & name, - const ModelInputInfo & input_info) { - auto param_node = std::make_shared(input_info.type, input_info.shape); - param_node->set_friendly_name(name); - param_node->output(0).get_tensor().set_names({name}); - return param_node; -} - -std::shared_ptr create_extra_input(const std::string & name, const ModelExtraInputInfo & input_info) { - if (input_info.is_parameter) { - auto param_node = std::make_shared(input_info.type, input_info.shape); - param_node->set_friendly_name(name); - param_node->output(0).get_tensor().set_names({name}); - return param_node; - } - - auto constant = std::make_shared(input_info.type, input_info.shape, - std::vector{input_info.value}); - constant->set_friendly_name(name); - return constant; -} - -ov::pass::MakeStateful::ParamResPairs get_kv_param_res_pairs( - const std::shared_ptr & model, - const std::map & kv_param_res_names) { - ov::pass::MakeStateful::ParamResPairs pairs; - const auto & params = model->get_parameters(); - const auto & results = model->get_results(); - - for (const auto & param_res : kv_param_res_names) { - const auto & param_name = param_res.first; - const auto & res_name = param_res.second; - - auto param_it = std::find_if(params.begin(), params.end(), [&](const std::shared_ptr & node) { - return node->get_friendly_name() == param_name; - }); - - OPENVINO_ASSERT(param_it != params.end(), "The tensor name ", param_name, - " is not associated with any of " - "Parameters in the network."); - - auto res_it = std::find_if(results.begin(), results.end(), [&](const std::shared_ptr & node) { - return node->get_friendly_name() == res_name; - }); - - OPENVINO_ASSERT(res_it != results.end(), "The tensor name ", res_name, - " is not associated with any of " - "Results in the network."); - - std::shared_ptr param = *param_it; - std::shared_ptr res = *res_it; - pairs.emplace_back(param, res); - } - return pairs; -} - -void add_sliced_mask_stateful(TensorMap & tensor_map, bool imrope) { - auto create_sliced_mask = [&](const std::string & mask_name, const std::string & sliced_name) { - if ((tensor_map.find(mask_name) != tensor_map.end()) && - (tensor_map.find("token_len_per_seq") != tensor_map.end()) && - (tensor_map.find("inp_pos") != tensor_map.end())) { - auto token_len_per_seq = tensor_map.at("token_len_per_seq").get_node_shared_ptr(); - auto mask = tensor_map.at(mask_name).get_node_shared_ptr(); - std::shared_ptr mask_sliced = mask; - auto one = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - auto zero = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); - auto three = ov::op::v0::Constant::create(ov::element::i64, {1}, {3}); - auto neg_one = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); - - auto step = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - auto axes = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); - - auto inp_pos = tensor_map.at("inp_pos").get_node_shared_ptr(); - // IMROPE's fourth position plane is zero for text; use the token's first plane. - ov::Output last_index = neg_one; - if (imrope) { - last_index = std::make_shared(token_len_per_seq, neg_one); - } - auto last_inp_pos = std::make_shared(inp_pos, last_index, three); - auto last_inp_pos_1d = std::make_shared( - last_inp_pos, ov::op::v0::Constant::create(ov::element::i64, {1}, {1}), false); - auto last_inp_pos_cvt = std::make_shared(last_inp_pos_1d, ov::element::i64); - auto last_inp_pos_inc = std::make_shared(last_inp_pos_cvt, one); - - mask_sliced = std::make_shared(mask, zero, last_inp_pos_inc, step, axes); - mask_sliced = std::make_shared(mask_sliced, ov::element::f16); - mask_sliced->set_friendly_name(sliced_name); - - tensor_map.insert({sliced_name, mask_sliced->output(0)}); - } - }; - - create_sliced_mask("self_kq_mask", "KQ_mask_sliced"); - create_sliced_mask("KQ_mask", "KQ_mask_sliced"); - create_sliced_mask("self_kq_mask_swa", "KQ_mask_swa_sliced"); -} - -// Rebuild the sliding-window mask from absolute positions. -// ggml caps self_kq_mask_swa at the size of its own SWA cache, but the stateful KV state is -// Concat-appended and grows without bound, so past that cap the two disagree on length and the -// mask add fails. A pure-Concat state is ordered by position, so positions can rebuild the mask. -// swa_window holds the real n_swa, read back from the ggml mask in ggml-decoder.cpp. -// No-op when the graph has no SWA mask, or when the window could not be read back. -void add_position_mask_stateful_swa(TensorMap & tensor_map) { - if (tensor_map.find("self_kq_mask_swa") == tensor_map.end() || tensor_map.find("inp_pos") == tensor_map.end() || - tensor_map.find("swa_window") == tensor_map.end()) { - return; - } - - auto inp_pos = tensor_map.at("inp_pos").get_node_shared_ptr(); - - auto zero_i64 = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); - auto one_i64 = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - auto three = ov::op::v0::Constant::create(ov::element::i64, {1}, {3}); - auto neg_one = ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}); - - auto query_pos = std::make_shared(inp_pos, ov::element::i64); - auto query_pos_1d = std::make_shared( - query_pos, ov::op::v0::Constant::create(ov::element::i64, {1}, {-1}), false); - - auto last_pos = std::make_shared(inp_pos, neg_one, three); - auto last_pos_1d = std::make_shared(last_pos, one_i64, false); - auto last_pos_cvt = std::make_shared(last_pos_1d, ov::element::i64); - auto total_len = std::make_shared(last_pos_cvt, one_i64); - auto total_len_scalar = std::make_shared(total_len); - - auto cached_pos = std::make_shared( - ov::op::v0::Constant::create(ov::element::i64, {}, {0}), total_len_scalar, - ov::op::v0::Constant::create(ov::element::i64, {}, {1}), ov::element::i64); - - auto query_col = std::make_shared( - query_pos_1d, ov::op::v0::Constant::create(ov::element::i64, {2}, {-1, 1}), false); - auto cached_row = std::make_shared( - cached_pos, ov::op::v0::Constant::create(ov::element::i64, {2}, {1, -1}), false); - auto diff = std::make_shared(query_col, cached_row); - - auto swa_window = tensor_map.at("swa_window").get_node_shared_ptr(); - auto window = std::make_shared(swa_window, ov::element::i64); - auto causal_ok = std::make_shared(diff, zero_i64); - auto window_ok = std::make_shared(diff, window); - auto keep = std::make_shared(causal_ok, window_ok); - - auto zero_f = ov::op::v0::Constant::create(ov::element::f32, {}, {0.0f}); - auto neg_inf_f = ov::op::v0::Constant::create(ov::element::f32, {}, {-std::numeric_limits::infinity()}); - std::shared_ptr mask = std::make_shared(keep, zero_f, neg_inf_f); - - auto batch_axis = ov::op::v0::Constant::create(ov::element::i64, {1}, {0}); - mask = std::make_shared(mask, batch_axis); - mask = std::make_shared(mask, batch_axis); - mask = std::make_shared(mask, ov::element::f16); - mask->set_friendly_name("KQ_mask_swa_sliced"); - - tensor_map["KQ_mask_swa_sliced"] = mask->output(0); -} - -void add_rope_sin_cos(TensorMap & tensor_map, GgmlDecoder & ggml_model_decoder) { - // When ROPE ops in the graph have divergent op_params (e.g. gemma4's mixed - // SWA/non-SWA layers with different n_dims or freq_base), a shared sin/cos - // precompute cannot broadcast across every ROPE use. Skip it here and let - // translate_rope() build sin/cos per-op from its own op_params. - if (ggml_model_decoder.has_mixed_rope_params()) { - return; - } - int32_t * rope_params = ggml_model_decoder.get_rope_params(); - if (tensor_map.find("inp_pos") == tensor_map.end() || rope_params == nullptr) { - return; - } - auto inp_pos = tensor_map.at("inp_pos").get_node_shared_ptr(); - std::shared_ptr rope_freqs_weight; - if (tensor_map.find("rope_freqs.weight") != tensor_map.end()) { - rope_freqs_weight = tensor_map.at("rope_freqs.weight").get_node_shared_ptr(); - } - - auto sin_cos = make_sin_cos(rope_params, inp_pos, rope_freqs_weight); - auto sin_theta = sin_cos.first; - auto cos_theta = sin_cos.second; - - cos_theta.get_node_shared_ptr()->set_friendly_name("rope_cos"); - sin_theta.get_node_shared_ptr()->set_friendly_name("rope_sin"); - tensor_map.insert({"rope_cos", cos_theta}); - tensor_map.insert({"rope_sin", sin_theta}); -} - -// Create common patterns -void preprocess(TensorMap & tensor_map, GgmlDecoder & ggml_model_decoder) { - if (ggml_model_decoder.is_stateful()) { - add_sliced_mask_stateful(tensor_map, ggml_model_decoder.get_rope_params()[2] == GGML_ROPE_TYPE_IMROPE); - add_position_mask_stateful_swa(tensor_map); - } - // This optimization is error-prone - // add_rope_sin_cos(tensor_map, ggml_model_decoder); -} - -} // namespace - -TranslateSession::TranslateSession(const frontend::InputModel::Ptr & input_model, - const std::unordered_map & translator_map, - bool naive) : - m_input_model(input_model), - m_translator_map(translator_map), - m_ov_model(nullptr), - m_naive(naive) {} - -std::shared_ptr TranslateSession::get_converted_model() { - if (m_ov_model) { - return m_ov_model; - } - m_ov_model = translate_graph(m_input_model); - return m_ov_model; -} - -std::shared_ptr TranslateSession::translate_graph(const frontend::InputModel::Ptr & input_model) { - ov::ParameterVector params; - ov::ResultVector results; - auto tensor_map = std::make_shared(); - std::shared_ptr resulting_model; - - const auto & ggml_model = ov::as_type_ptr(input_model); - std::shared_ptr ggml_model_decoder = ggml_model->get_model_decoder(); - - for (const auto & it : ggml_model_decoder->get_model_inputs()) { - auto param_node = create_parameter(it.first, it.second); - params.push_back(param_node); - (*tensor_map)[it.first] = param_node; - } - - for (const auto & it : ggml_model_decoder->get_model_extra_inputs()) { - auto input_node = create_extra_input(it.first, it.second); - if (it.second.is_parameter) { - params.push_back(ov::as_type_ptr(input_node)); - } - (*tensor_map)[it.first] = input_node; - } - - for (const auto & it : ggml_model_decoder->get_model_weights()) { - (*tensor_map)[it.first] = it.second; - } - - auto translate_node = [&](const std::shared_ptr & decoder, int node_idx) { - auto operation_type = decoder->get_op_type(node_idx); - if (operation_type == "GGML_OP_NONE") { - return ov::OutputVector{}; - } - - const auto & node_output_names = decoder->get_output_names(node_idx); - if (operation_type == "GGML_OP_VIEW" && decoder->get_op_case(node_idx) == 2 && node_output_names.size() == 1) { - auto direct_output = tensor_map->find(node_output_names[0]); - if (direct_output != tensor_map->end()) { - // GDN publishes its native attention/state outputs under the two GGML VIEW names. - // Keep those mappings instead of rebuilding slices of a packed temporary. - return ov::OutputVector{direct_output->second}; - } - } - - auto it = m_translator_map.find(operation_type); - FRONT_END_OP_CONVERSION_CHECK(it != m_translator_map.end(), "Translation for operation type ", operation_type, - " is not implemented."); - NodeContext node_context(decoder, tensor_map, node_idx, this); - ov::OutputVector converted_outputs = it->second(node_context); - - FRONT_END_OP_CONVERSION_CHECK(node_output_names.size() == converted_outputs.size(), "Number of ", - operation_type, " outputs greater than number of converted outputs, which are ", - node_output_names.size(), " and ", converted_outputs.size(), " respectively."); - - for (size_t i = 0; i < node_output_names.size(); ++i) { - auto output_name = node_output_names[i]; - if (i < converted_outputs.size() && converted_outputs[i].get_node_shared_ptr() != nullptr) { - (*tensor_map)[output_name] = converted_outputs[i]; - } - } - return converted_outputs; - }; - - // To handle cases like this - // 3: [ 18432, 1, 1, 1] RESHAPE cache_r_l0 (reshaped)#3 - // [ 18432, 1, 1, 1] 0: NONE cache_r_l0 - // 4: [ 0, 1, 1, 1] VIEW cache_r_l0 (reshaped) (view)#4 - // [ 18432, 1, 1, 1] 0: RESHAPE cache_r_l0 (reshaped)#3 - // 5: [ 0, 1, 1, 1] SCALE cache_r_l0 (reshaped) (view) (view)#5 - // [ 0, 1, 1, 1] 0: VIEW cache_r_l0 (reshaped) (view)#4 - // 6: [ 1, 1, 1, 1] VIEW (view)#6 - // [ 1, 1, 1, 1] 0: NONE leaf_5 - // 7: [ 18432, 1, 1, 1] GET_ROWS conv_states-0#7 - // [ 18432, 1, 1, 1] 0: RESHAPE cache_r_l0 (reshaped)#3 - // [ 1, 1, 1, 1] 1: VIEW (view)#6 - // The scale is in-place which modifies cache_r_l0 (reshaped)#3 - // The translation of scale overwrites cache_r in the tensor_map, - // but we also need to overwrite the old cache_r_l0 (reshaped)#3 - auto refresh_inplace_aliases = [&](const std::shared_ptr & decoder, int inplace_node_idx, - const std::string & view_src_name) { - for (int node_idx = 0; node_idx < inplace_node_idx; node_idx++) { - if (decoder->is_view_like_alias_of(node_idx, view_src_name)) { - translate_node(decoder, node_idx); - } - } - }; - - auto node_visitor = [&](const std::shared_ptr & decoder, int node_idx) { - auto converted_outputs = translate_node(decoder, node_idx); - if (converted_outputs.empty()) { - return; - } - const auto inplace_src = decoder->get_inplace_op_src(node_idx); - if (inplace_src.empty()) { - return; - } - if (converted_outputs[0].get_node_shared_ptr() != nullptr) { - (*tensor_map)[inplace_src] = converted_outputs[0]; - } - refresh_inplace_aliases(decoder, node_idx, inplace_src); - }; - - if (!m_naive) { - preprocess(*tensor_map, *ggml_model_decoder); - } - ggml_model_decoder->visit_subgraph(node_visitor); - - for (const auto & name : ggml_model_decoder->get_model_output_names()) { - FRONT_END_GENERAL_CHECK(tensor_map->find(name) != tensor_map->end(), - "Output name not found in tensor map: ", name); - auto result = std::make_shared(tensor_map->at(name)); - result->set_friendly_name(name); - results.push_back(result); - } - - // Debug-only hook: GGML_OPENVINO_DEBUG_NODE=,,... adds extra - // Result nodes for arbitrary intermediate tensors (looked up by name in - // tensor_map), on top of the real model outputs above. These debug - // Results are deliberately NOT added to ggml_decoder's model outputs, so - // the caller (ov_graph_compute_dynamic in utils.cpp) will not bind them - // to any ggml tensor buffer -- OpenVINO allocates its own tensor for - // them. This avoids the risk of reading a ggml buffer that has since - // been overwritten by a later in-place op (ggml aggressively reuses - // buffers), which can happen if trying to inspect an intermediate value - // via GGML_OPENVINO_DEBUG_OUTPUT by hacking it into a real output. - // - // tensor_map keys are usually the plain ggml tensor name (e.g. "embd"), - // but tensors that are recomputed multiple times in the same cgraph - // (GGML_TENSOR_FLAG_COMPUTE) are disambiguated with a "#" suffix - // (e.g. "cache_k_l0#4853", see get_tensor_ov_name()) which is not - // predictable ahead of time. To keep the env var usable, a requested - // name is matched either exactly, or as the "name" part before "#" of a - // suffixed key (first match wins; ambiguous requests should include the - // full "name#hash" form seen in a previous run's log/dump). - if (const char * debug_nodes = ggml_openvino_getenv_str("GGML_OPENVINO_DEBUG_NODE")) { - std::stringstream ss(debug_nodes); - std::string name; - while (std::getline(ss, name, ',')) { - auto it = tensor_map->find(name); - if (it == tensor_map->end()) { - it = std::find_if(tensor_map->begin(), tensor_map->end(), [&](const auto & entry) { - return entry.first.compare(0, name.size(), name) == 0 && entry.first.size() > name.size() && - entry.first[name.size()] == '#'; - }); - } - if (it == tensor_map->end()) { - GGML_LOG_WARN("GGML_OPENVINO_DEBUG_NODE: node '%s' not found in tensor map, skipping\n", name.c_str()); - continue; - } - auto result = std::make_shared(it->second); - result->set_friendly_name("__debug_" + it->first); - results.push_back(result); - } - } - - ov::ParameterVector used_params; - for (const auto & param : params) { - if (!param->output(0).get_target_inputs().empty()) { - used_params.push_back(param); - } - } - // if (auto diff = params.size() - used_params.size()) { - // GGML_LOG_INFO("%zu parameters are not used in the model.", diff); - // } - resulting_model = std::make_shared(results, used_params); - - apply_transformations(resulting_model); - - // Set WeightlessCacheAttribute on large constants to avoid unnecessary memory copies - // in the NPUW plugin. Without this attribute, NPUW's LazyTensor constructor - // (lazy_tensor.cpp, op::Const::Const) will memcpy every constant "in case export - // occurs", doubling memory usage per compile_model call. - // - // The bin_offset field serves as a unique key (not a real file offset) — this is - // the same convention the GPU plugin uses for non-IR models (see - // Plugin::set_weightless_cache_attributes in intel_gpu/src/plugin/plugin.cpp). - // Each constant must have a distinct bin_offset, otherwise GPU's weightless cache - // import will map multiple constants to the same data. - // - // Small constants (< 16 elements) are excluded since they may be introduced by - // optimization patterns and the overhead is negligible. - // - // Note: use shape_size() rather than byte_size()/element_type().size() - GatherMatmul's default - // bias is a Constant(element::dynamic, Shape{0}), whose element_type().size() is 0 and would - // divide by zero. - size_t offset = 0; - for (auto & node : resulting_model->get_ordered_ops()) { - if (auto cnst = ov::as_type_ptr(node); cnst && ov::shape_size(cnst->get_shape()) >= 16) { - auto & rt_info = cnst->get_rt_info(); - if (rt_info.find(ov::WeightlessCacheAttribute::get_type_info_static()) == rt_info.end()) { - rt_info[ov::WeightlessCacheAttribute::get_type_info_static()] = - ov::WeightlessCacheAttribute(cnst->get_byte_size(), offset++, cnst->get_element_type()); - } - } - } - return resulting_model; -} - -std::shared_ptr TranslateSession::apply_transformations(std::shared_ptr model) { - auto ggml_model_decoder = ov::as_type_ptr(m_input_model)->get_model_decoder(); - { - ov::pass::Manager manager; - manager.set_per_pass_validation(true); - manager.register_pass(); - // Marks the Convert/Subtract/Multiply nodes of our GatherMatmul dequantization chain - // (make_int4_weights/make_int8_weights, for_gather_matmul=true) with disable_constant_folding, - // so it survives ConstantFolding regardless of whether the target plugin's own - // is_decompression_multiply() recognizes GatherMatmul as a valid consumer. - manager.register_pass( - std::vector{ov::element::u8, ov::element::i8, ov::element::u4, ov::element::i4}); - manager.register_pass(); - manager.register_pass(); - - // MOECompressed has no CPU plugin implementation, so enable it by default only on GPU. - // GGML_OPENVINO_MOE_OP=0 keeps the unfused GatherMatmul path for fallback/debugging. - if (ggml_openvino_is_gpu() && - ggml_openvino_getenv_int("GGML_OPENVINO_MOE_OP", 1) != 0) { - manager.register_pass(); - manager.register_pass(); - // Same fusion for models whose gate and up projections share one fused expert - // weight (gemma-4). It only matches that shape and only when the experts carry an - // integer zero point, so it is a no-op on the separate-gate/up models above. - manager.register_pass(); - } - - if (ggml_model_decoder->is_stateful()) { - const auto kv_param_res_names = ggml_model_decoder->get_kv_param_res_names(); - const auto kv_param_res_pairs = get_kv_param_res_pairs(model, kv_param_res_names); - manager.register_pass(kv_param_res_pairs); - // Must run after MakeStateful, which is what creates the ReadValue/Assign pairs. - if (!ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT")) { - manager.register_pass(); - } - } - - if (ggml_model_decoder->is_static()) { - manager.register_pass(); - } - manager.run_passes(model); - if (ggml_model_decoder->is_stateful()) { - ov::preprocess::PrePostProcessor ppp(model); - for (size_t i = 0; i < model->get_output_size(); i++) { - auto model_output_shape = model->output(i).get_partial_shape(); - if (model_output_shape.rank().is_static() && model_output_shape.rank().get_length() == 3) { - ppp.output(i).postprocess().custom([](const ov::Output& node) { - auto axes = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{1}, {0}); - return std::make_shared(node, axes); - }); - } - } - model = ppp.build(); - } - } - return model; -} - -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/translate_session.h b/ggml/src/ggml-openvino/openvino/translate_session.h deleted file mode 100644 index 675e63223a97..000000000000 --- a/ggml/src/ggml-openvino/openvino/translate_session.h +++ /dev/null @@ -1,29 +0,0 @@ -#pragma once - -#include "input_model.h" -#include "node_context.h" - -namespace ov { -namespace frontend { -namespace ggml { - -class TranslateSession { -public: - TranslateSession(const frontend::InputModel::Ptr & input_model, - const std::unordered_map & translator_map, - bool naive = false); - - std::shared_ptr get_converted_model(); - std::shared_ptr translate_graph(const frontend::InputModel::Ptr & input_model); - -private: - std::shared_ptr apply_transformations(std::shared_ptr model); - const frontend::InputModel::Ptr m_input_model; - const std::unordered_map & m_translator_map; - std::shared_ptr m_ov_model; - bool m_naive; -}; - -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/utils.cpp b/ggml/src/ggml-openvino/openvino/utils.cpp deleted file mode 100644 index 40ba938a62ce..000000000000 --- a/ggml/src/ggml-openvino/openvino/utils.cpp +++ /dev/null @@ -1,987 +0,0 @@ -#include "utils.h" - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { - -void num_inputs_check(const NodeContext & context, size_t min_inputs, size_t max_inputs) { - auto input_size = context.get_input_size(); - FRONT_END_OP_CONVERSION_CHECK(input_size >= min_inputs, "Got less inputs than expected"); - FRONT_END_OP_CONVERSION_CHECK(input_size <= max_inputs, "Got more inputs than expected"); -} - -int non_cont_dim(std::vector ne, std::vector nb) { - int dim = nb.size() - 1; - size_t bytes = nb[dim]; - for (int i = dim; i > 0; i--) { - bytes *= ne[i]; - if (bytes != nb[i - 1]) { - return i; - } - } - return 0; -} - -std::shared_ptr get_dimensions(const std::shared_ptr & shape, - const std::vector & dims) { - using namespace ov::op; - const auto zero = v0::Constant::create(ov::element::i32, ov::Shape{}, {0}); - const auto dims_const = v0::Constant::create(ov::element::i32, ov::Shape{dims.size()}, dims); - return std::make_shared(shape, dims_const, zero); -} - -std::shared_ptr get_dimensions(const std::shared_ptr & node, const std::vector & dims) { - return get_dimensions(std::make_shared(node), dims); -} - -OutputVector rename_outputs_with_suffix(const OutputVector & outputs, const std::string & suffix) { - for (const auto & output : outputs) { - auto node = output.get_node_shared_ptr(); - std::string name = node->get_friendly_name(); - name += "_"; - name += suffix; - node->set_friendly_name(name); - // Uncomment to dump every node's inferred shape (used to hunt down dynamic dims on NPU). - // std::cout << name << " " << output.get_partial_shape() << std::endl; - } - return outputs; -} - -namespace { -ov::Output rope_yarn_ramp_mix(int num_elements, const float corr_dims[2], float ext_factor) { - std::vector dim_ids_vec(num_elements); - std::iota(dim_ids_vec.begin(), dim_ids_vec.end(), 0.0f); - auto dim_ids = ov::op::v0::Constant::create(ov::element::f32, Shape{1, 1, 1, (size_t) num_elements}, dim_ids_vec); - auto corr_low = ov::op::v0::Constant::create(ov::element::f32, Shape{1, 1, 1, 1}, {corr_dims[0]}); - auto corr_high = ov::op::v0::Constant::create(ov::element::f32, Shape{1, 1, 1, 1}, {corr_dims[1]}); - auto denom = std::make_shared( - std::make_shared(corr_high, corr_low), - ov::op::v0::Constant::create(ov::element::f32, Shape{1, 1, 1, 1}, {0.001f})); - auto ramp_y = - std::make_shared(std::make_shared(dim_ids, corr_low), denom); - auto ramp_clamped = std::make_shared(ramp_y, 0.0f, 1.0f); - // rope_yarn_ramp returns (1 - clamp(y)), so invert before scaling - auto one = ov::op::v0::Constant::create(ov::element::f32, Shape{1, 1, 1, 1}, {1.0f}); - auto ramp_inverted = std::make_shared(one, ramp_clamped); - auto ext_factor_node = ov::op::v0::Constant::create(ov::element::f32, Shape{}, {ext_factor}); - auto ramp_mix = std::make_shared(ramp_inverted, ext_factor_node); - return ramp_mix; -} - -float ggml_rope_yarn_corr_dim(int n_dims, int n_ctx_orig, float n_rot, float base) { -#ifndef M_PI -# define M_PI 3.14159265358979323846 -#endif - return n_dims * logf(n_ctx_orig / (n_rot * 2 * (float) M_PI)) / (2 * logf(base)); -} - -void ggml_rope_yarn_corr_dims(int n_dims, - int n_ctx_orig, - float freq_base, - float beta_fast, - float beta_slow, - float dims[2]) { - float start = floorf(ggml_rope_yarn_corr_dim(n_dims, n_ctx_orig, beta_fast, freq_base)); - float end = ceilf(ggml_rope_yarn_corr_dim(n_dims, n_ctx_orig, beta_slow, freq_base)); - dims[0] = std::max(0.0f, start); - dims[1] = std::min(static_cast(n_dims - 1), end); -} -} // namespace - -std::pair, ov::Output> make_sin_cos(int32_t * rope_params, - std::shared_ptr inp_pos, - std::shared_ptr rope_freqs_weight, - int mode, - bool stateful, - int64_t head_dim) { - constexpr int TYPE_IMROPE = 2; - constexpr int TYPE_VISION = 3; - constexpr int TYPE_MROPE = 4; - - const bool is_imrope = (mode == TYPE_IMROPE); - const bool is_vision = (mode == TYPE_VISION); - const bool is_mrope = (mode == TYPE_MROPE); - const bool is_multi = is_imrope || is_vision || is_mrope; - - if (stateful) { - inp_pos = - std::make_shared(inp_pos, ov::op::v0::Constant::create(ov::element::i64, {1}, {0})); - inp_pos = std::make_shared(inp_pos, ov::element::f32); - auto pos_perm = - std::make_shared(ov::element::i64, ov::Shape{3}, std::vector{2, 1, 0}); - inp_pos = std::make_shared(inp_pos, pos_perm); - } else if (is_multi) { - inp_pos = std::make_shared(inp_pos, ov::element::f32); - auto pos_shape = ov::op::v0::Constant::create(ov::element::i64, ov::Shape{5}, {0, 0, 0, 4, -1}); - inp_pos = std::make_shared(inp_pos, pos_shape, true); - auto pos_transpose_shape = - std::make_shared(ov::element::i64, ov::Shape{5}, std::vector{0, 1, 2, 4, 3}); - inp_pos = std::make_shared(inp_pos, pos_transpose_shape); - } else { - inp_pos = std::make_shared(inp_pos, ov::element::f32); - auto pos_perm = - std::make_shared(ov::element::i64, ov::Shape{4}, std::vector{0, 3, 1, 2}); - inp_pos = std::make_shared(inp_pos, pos_perm); - } - - float freq_base; - float freq_scale; - float ext_factor; - float attn_factor; - float beta_fast; - float beta_slow; - const int n_dims = rope_params[1]; - const size_t n_dims_half = n_dims >> 1; - const int n_ctx_orig = rope_params[4]; - memcpy(&freq_base, rope_params + 5, sizeof(float)); - memcpy(&freq_scale, rope_params + 6, sizeof(float)); - memcpy(&ext_factor, rope_params + 7, sizeof(float)); - memcpy(&attn_factor, rope_params + 8, sizeof(float)); - memcpy(&beta_fast, rope_params + 9, sizeof(float)); - memcpy(&beta_slow, rope_params + 10, sizeof(float)); - - const float theta_scale = powf(freq_base, -2.0f / n_dims); - - Output theta; - float mscale = attn_factor; - if (is_multi) { - const size_t num_pairs = is_vision ? (n_dims > 0 ? (size_t) n_dims : (size_t) (head_dim / 2)) : n_dims_half; - int sections[4]; - memcpy(sections, rope_params + 11, sizeof(int) * 4); - int sect_dims = sections[0] + sections[1] + sections[2] + sections[3]; - if (sect_dims <= 0) { - sect_dims = num_pairs; - } - int sec_w = sections[0] + sections[1]; - int sec_e = sec_w + sections[2]; - std::vector gather_indices(num_pairs); - std::vector factor(num_pairs); - for (size_t j = 0; j < num_pairs; j++) { - int sector = j % sect_dims; - int plane = 0; - if (is_imrope) { - if (sector % 3 == 1 && sector < 3 * sections[1]) { - plane = 1; - } else if (sector % 3 == 2 && sector < 3 * sections[2]) { - plane = 2; - } else if (sector % 3 == 0 && sector < 3 * sections[0]) { - plane = 0; - } else { - plane = 3; - } - factor[j] = std::pow(theta_scale, j); - } else if (is_vision) { - int p_idx = 0; - if (sector < sections[0]) { - plane = 0; - p_idx = sector; - } else if (sector < sections[0] + sections[1]) { - plane = 1; - p_idx = sector - sections[0]; - } else if (sector < sec_w + sections[2]) { - plane = 2; - p_idx = sector - sec_w; - } else { - plane = 3; - p_idx = sector - sec_e; - } - factor[j] = std::pow(theta_scale, p_idx); - } else { - if (sector >= sections[0] && sector < sec_w) { - plane = 1; - } else if (sector >= sec_w && sector < sec_e) { - plane = 2; - } else if (sector >= sec_e) { - plane = 3; - } else { - plane = 0; - } - factor[j] = std::pow(theta_scale, j); - } - gather_indices[j] = plane; - } - auto gather_indices_const = - std::make_shared(ov::element::i64, ov::Shape{num_pairs}, gather_indices); - auto gather_axis = ov::op::v0::Constant::create(ov::element::i32, ov::Shape{}, {4}); - inp_pos = std::make_shared(inp_pos, gather_indices_const, gather_axis); - Output factor_node = - std::make_shared(ov::element::f32, ov::Shape{num_pairs}, factor); - if (rope_freqs_weight) { - Output rope_factors = std::make_shared( - rope_freqs_weight, - ov::op::v0::Constant::create(ov::element::i64, {1}, {0}), - ov::op::v0::Constant::create(ov::element::i64, {1}, {(int64_t) num_pairs}), - ov::op::v0::Constant::create(ov::element::i64, {1}, {1}), - ov::op::v0::Constant::create(ov::element::i64, {1}, {rope_freqs_weight->get_output_partial_shape(0).rank().get_length() - 1})); - rope_factors = std::make_shared( - rope_factors, - ov::op::v0::Constant::create(ov::element::i64, {1}, {(int64_t) num_pairs}), - false); - factor_node = std::make_shared(factor_node, rope_factors); - } - auto theta_extrap = std::make_shared(inp_pos, factor_node); - auto theta_interp = std::make_shared( - theta_extrap, ov::op::v0::Constant::create(ov::element::f32, {1}, {freq_scale})); - if (ext_factor == 0.0f) { - theta = theta_interp; - } else { - float corr_dims[2]; - ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims); - auto ramp_mix = rope_yarn_ramp_mix(num_pairs, corr_dims, ext_factor); - Output one = ov::op::v0::Constant::create(ov::element::f32, Shape{1, 1, 1, 1}, {1.0f}); - auto one_minus_ramp = std::make_shared(one, ramp_mix); - theta = std::make_shared( - std::make_shared(theta_interp, one_minus_ramp), - std::make_shared(theta_extrap, ramp_mix)); - mscale *= (1.0f + 0.1f * std::log(1.0f / freq_scale)); - } - } else { - std::vector factor(n_dims_half); - Output freq_factors; - float corr_dims[2]; - ggml_rope_yarn_corr_dims(n_dims, n_ctx_orig, freq_base, beta_fast, beta_slow, corr_dims); - factor[0] = 1.0f; - for (size_t i = 1; i < factor.size(); i++) { - factor[i] = theta_scale * factor[i - 1]; - } - if (stateful) { - freq_factors = - std::make_shared(ov::element::f32, ov::Shape{1, 1, factor.size()}, factor); - } else { - freq_factors = - std::make_shared(ov::element::f32, ov::Shape{1, 1, 1, factor.size()}, factor); - } - if (rope_freqs_weight) { - Output rope_factors = std::make_shared( - rope_freqs_weight, - ov::op::v0::Constant::create(ov::element::i64, {1}, {0}), - ov::op::v0::Constant::create(ov::element::i64, {1}, {(int64_t) n_dims_half}), - ov::op::v0::Constant::create(ov::element::i64, {1}, {1}), - ov::op::v0::Constant::create(ov::element::i64, {1}, {rope_freqs_weight->get_output_partial_shape(0).rank().get_length() - 1})); - if (stateful) { - rope_factors = std::make_shared( - rope_factors, - ov::op::v0::Constant::create(ov::element::i64, {3}, {(int64_t) 1, (int64_t) 1, (int64_t) n_dims_half}), - false); - } else { - rope_factors = std::make_shared( - rope_factors, - ov::op::v0::Constant::create(ov::element::i64, {4}, {(int64_t) 1, (int64_t) 1, (int64_t) 1, (int64_t) n_dims_half}), - false); - } - freq_factors = std::make_shared(freq_factors, rope_factors); - } - - auto theta_extrap = std::make_shared(freq_factors, inp_pos); - auto theta_interp = std::make_shared( - theta_extrap, ov::op::v0::Constant::create(ov::element::f32, {1}, {freq_scale})); - - if (ext_factor == 0.0f) { - theta = theta_interp; - } else { - auto ramp_mix = rope_yarn_ramp_mix(n_dims_half, corr_dims, ext_factor); - Output one; - if (stateful) { - one = ov::op::v0::Constant::create(ov::element::f32, Shape{1, 1, 1}, {1.0f}); - } else { - one = ov::op::v0::Constant::create(ov::element::f32, Shape{1, 1, 1, 1}, {1.0f}); - } - auto one_minus_ramp = std::make_shared(one, ramp_mix); - - theta = - std::make_shared(std::make_shared(theta_interp, one_minus_ramp), - std::make_shared(theta_extrap, ramp_mix)); - mscale *= (1.0f + 0.1f * std::log(1.0f / freq_scale)); - } - } - - Output cos_theta = std::make_shared(theta); - Output sin_theta = std::make_shared(theta); - - if (mscale != 1.0f) { - auto mscale_node = ov::op::v0::Constant::create(ov::element::f32, Shape{}, {mscale}); - - cos_theta = std::make_shared(cos_theta, mscale_node); - sin_theta = std::make_shared(sin_theta, mscale_node); - } - - return std::make_pair(sin_theta, cos_theta); -} - -ov::Output process_view_input(const NodeContext & context, int input_index, int slice_len, int axis) { - // Only works for VIEW operations that does a non-strided slice with optinal reshape on the slice result. - // The function only does the slice part, the reshape (if any) should be handled by the caller. - // Default axis is -1, which means slicing the last dimension. - // If the VIEW reshapes the result, `slice_len` should be provided - auto input = context.get_input(input_index); - auto * op_params = (size_t *) context.get_input_op_params(input_index); - auto src_stride = context.get_input_stride(input_index); - - int64_t slice_start = op_params[0] / src_stride[3]; - if (slice_len == 0) { - slice_len = context.get_input_shape(input_index)[3].get_length(); - } - int64_t slice_end = slice_start + slice_len; - - auto begin = ov::op::v0::Constant::create(ov::element::i64, {1}, {slice_start}); - auto end = ov::op::v0::Constant::create(ov::element::i64, {1}, {slice_end}); - auto stride = ov::op::v0::Constant::create(ov::element::i64, {1}, {1}); - ov::Output axes; - if (axis == -1) { - axes = ov::op::v0::Constant::create(ov::element::i64, {1}, {context.is_stateful() ? 2 : 3}); - } else { - axes = ov::op::v0::Constant::create(ov::element::i64, {1}, {axis}); - } - auto sliced = std::make_shared(input, begin, end, stride, axes); - return sliced; -} - -ov::Output process_view_input_new(const NodeContext & context, int input_index) { - auto input = context.get_input(input_index); - - // Check if this input has view inputs - size_t view_input_size = context.get_view_input_size(input_index); - if (view_input_size == 0) { - // No view inputs, return the input as is - return input; - } - - // If translate_view already resolved this VIEW (produced a Slice), the input - // will already have the expected shape — skip re-slicing. - // - // Two notions of "matches" are accepted per axis: - // - both dims static and equal, OR - // - both dims dynamic. - // The dynamic case matters for the MoE expert-plane views: translate_view now emits a - // DYNAMIC-token slice (so the token dim is not frozen). An all-static-only check would - // see the dynamic token dim, decide the shapes "don't match", and fall through to - // re-slice/flatten the already-resolved view (a Reshape to the full flattened - // n_expert_used*n_embd tail, which then conflicts with the single-plane input). Treat a - // dynamic-vs-dynamic axis as matching so the already-resolved view is reused as-is. - // - // A third case matters for split-model MoE fragments: translate_view resolves the - // expert-plane view against the fragment's INPUT parameter. When the graph is split - // the token axis of that parameter may already be concrete (static n_tokens) even - // though get_view_input_ov_shape() still reports it as dynamic (-1). The resolved - // view is then static [1,1,n_tokens,n_embd] while `expected` is [1,1,?,n_embd]. - // An "expected dynamic, actual static" axis is a valid concretization of the SAME - // resolved view, so treat it as matching too. Falling through to process_single_view - // here would re-slice/re-flatten the already-resolved single-plane view against the - // recorded (multi-plane) source strides and emit a constant-target Reshape whose baked - // dims no longer divide the concretized input -> "dimensions do not evenly divide". - // A fourth case matters for stateful execution: `expected` comes from ggml metadata and is - // always GGML_MAX_DIMS ranks, but stateful drops the leading size-1 batch dim, so the - // resolved view is one rank lower. Compare the common trailing dims and require the - // leading expected dims we skip to be 1. Without this the rank test below never matches on - // the stateful path and every already-resolved MoE expert-plane view is re-sliced. - auto expected_ov_shape = context.get_view_input_ov_shape(input_index, 0); - auto actual_shape = input.get_partial_shape(); - if (expected_ov_shape.rank().is_static() && actual_shape.rank().is_static() && - expected_ov_shape.rank().get_length() >= actual_shape.rank().get_length()) { - const int64_t n_actual = actual_shape.rank().get_length(); - const int64_t shift = expected_ov_shape.rank().get_length() - n_actual; - bool shapes_match = true; - for (int64_t i = 0; i < shift; ++i) { - if (!expected_ov_shape[i].is_static() || expected_ov_shape[i].get_length() != 1) { - shapes_match = false; - break; - } - } - for (int64_t i = 0; i < n_actual && shapes_match; ++i) { - const auto & exp = expected_ov_shape[i + shift]; - const bool both_dynamic = exp.is_dynamic() && actual_shape[i].is_dynamic(); - const bool both_static_equal = - exp.is_static() && actual_shape[i].is_static() && exp == actual_shape[i]; - // expected dynamic, actual static: the resolved view already carries the - // concrete size for this fragment; reuse it rather than re-materializing. - const bool expected_dyn_actual_static = exp.is_dynamic() && actual_shape[i].is_static(); - if (!both_dynamic && !both_static_equal && !expected_dyn_actual_static) { - shapes_match = false; - break; - } - } - if (shapes_match) { - return input; - } - } - - // In static mode, use Split instead of Slice for single-dimension reductions. - // This ensures NPUW's FOLD doesn't parametrize per-layer slice indices (which - // would introduce dynamic shapes). A shared Split node sits outside the repeated - // subgraph boundary; each layer receives one of its output ports. - if (context.is_static() && view_input_size == 1) { - auto view_stride_v = context.get_view_input_stride(input_index, 0); - auto view_src_stride_v = context.get_view_input_src_stride(input_index, 0); - auto view_ggml_shape = context.get_view_input_ggml_shape(input_index, 0); - auto view_src_ggml_shape = context.get_view_input_src_ggml_shape(input_index, 0); - auto view_offset = context.get_view_input_offset(input_index, 0); - auto view_src_offset = context.get_view_input_src_offset(input_index, 0); - - size_t ndims = view_ggml_shape.size(); - std::vector diff_dims; - if (view_src_ggml_shape.size() == ndims) { - for (size_t i = 0; i < ndims; ++i) { - if (view_ggml_shape[i] != view_src_ggml_shape[i]) { - diff_dims.push_back(static_cast(i)); - } - } - } - - if (diff_dims.size() == 1) { - int split_dim = diff_dims[0]; - int64_t num_splits = static_cast(view_src_ggml_shape[split_dim]); - int64_t chunk_size = static_cast(view_ggml_shape[split_dim]); - - // Only apply when slicing exactly 1 element from a multi-element dimension - if (chunk_size == 1 && num_splits > 1) { - // Check suffix strides match (dimensions after split_dim) - bool suffix_ok = view_stride_v.size() == view_src_stride_v.size(); - if (suffix_ok) { - for (size_t i = static_cast(split_dim) + 1; i < ndims; ++i) { - if (view_stride_v[i] != view_src_stride_v[i]) { - suffix_ok = false; - break; - } - } - } - - if (suffix_ok && view_src_stride_v[split_dim] > 0) { - size_t relative_offset = view_offset >= view_src_offset ? view_offset - view_src_offset : 0; - int64_t split_index = static_cast(relative_offset / view_src_stride_v[split_dim]); - - if (split_index >= 0 && split_index < num_splits) { - auto src_node = input.get_node_shared_ptr(); - std::string rt_key = "split_dim_" + std::to_string(split_dim); - auto & rt_info = src_node->get_rt_info(); - - if (rt_info.find(rt_key) == rt_info.end()) { - auto axis_const = - ov::op::v0::Constant::create(ov::element::i64, {}, {static_cast(split_dim)}); - auto split_node = - std::make_shared(input, axis_const, static_cast(num_splits)); - split_node->set_friendly_name(src_node->get_friendly_name() + "_split"); - rt_info[rt_key] = split_node; - } - - auto split_node = rt_info[rt_key].as>(); - return split_node->output(static_cast(split_index)); - } - } - } - } - } - - // Lambda function to process a single view operation - auto process_single_view = - [](ov::Output current, size_t view_offset, const std::vector & view_stride, - const ov::Shape & view_ggml_shape, const ov::PartialShape & view_ov_shape, const std::string & view_name, - size_t view_src_offset, const std::vector & view_src_stride, const ov::Shape & view_src_ggml_shape, - const ov::PartialShape & view_src_ov_shape, const std::string & view_src_name) -> ov::Output { - // Stateful execution drops a leading size-1 axis, so `current` can be one rank lower - // than the ggml shape metadata (view_stride/view_ggml_shape, always GGML_MAX_DIMS - // entries) assumes. Shift any axis index built from that metadata down by the - // difference before using it as an OV Slice axis. - const auto current_rank = current.get_partial_shape().rank(); - const int axis_shift = current_rank.is_static() ? - static_cast(view_stride.size()) - static_cast(current_rank.get_length()) : - 0; - - auto build_reshape_pattern = [](const ov::PartialShape & target_ov_shape, - const ov::Shape & target_ggml_shape) -> std::vector { - const size_t ndims = target_ggml_shape.size(); - std::vector reshape_pattern(ndims); - size_t dynamic_dims = 0; - - if (target_ov_shape.rank().is_static() && - target_ov_shape.rank().get_length() == static_cast(ndims)) { - for (size_t i = 0; i < ndims; ++i) { - if (target_ov_shape[i].is_static()) { - reshape_pattern[i] = target_ov_shape[i].get_length(); - } else { - reshape_pattern[i] = -1; - ++dynamic_dims; - } - } - } else { - dynamic_dims = 2; - } - - if (dynamic_dims > 1) { - for (size_t i = 0; i < ndims; ++i) { - reshape_pattern[i] = static_cast(target_ggml_shape[i]); - } - } - - return reshape_pattern; - }; - - auto build_prefix_tail_reshape_pattern = [](const ov::PartialShape & target_ov_shape, - const ov::Shape & target_ggml_shape, size_t prefix_dims, - int64_t tail_dim) -> std::vector { - std::vector reshape_pattern(prefix_dims + 1); - size_t dynamic_dims = 0; - - if (target_ov_shape.rank().is_static() && - target_ov_shape.rank().get_length() == static_cast(target_ggml_shape.size())) { - for (size_t i = 0; i < prefix_dims; ++i) { - if (target_ov_shape[i].is_static()) { - reshape_pattern[i] = target_ov_shape[i].get_length(); - } else { - reshape_pattern[i] = -1; - ++dynamic_dims; - } - } - } else { - dynamic_dims = 2; - } - - if (dynamic_dims > 1) { - for (size_t i = 0; i < prefix_dims; ++i) { - reshape_pattern[i] = static_cast(target_ggml_shape[i]); - } - } - - reshape_pattern[prefix_dims] = tail_dim; - return reshape_pattern; - }; - - bool same_stride = view_stride.size() == view_src_stride.size(); - if (same_stride) { - for (size_t i = 0; i < view_stride.size(); ++i) { - if (view_stride[i] != view_src_stride[i]) { - same_stride = false; - break; - } - } - } - - bool same_ggml_shape = view_ggml_shape.size() == view_src_ggml_shape.size(); - if (same_ggml_shape) { - for (size_t i = 0; i < view_ggml_shape.size(); ++i) { - if (view_ggml_shape[i] != view_src_ggml_shape[i]) { - same_ggml_shape = false; - break; - } - } - } - - if (same_stride && same_ggml_shape) { - return current; - } - - if (same_stride) { - const size_t relative_offset = view_offset >= view_src_offset ? view_offset - view_src_offset : 0; - const size_t ndims = view_stride.size(); - - std::vector diff_dims; - if (view_ggml_shape.size() == ndims && view_src_ggml_shape.size() == ndims) { - for (size_t i = 0; i < ndims; ++i) { - if (view_ggml_shape[i] != view_src_ggml_shape[i]) { - diff_dims.push_back(static_cast(i)); - } - } - } - - if (diff_dims.size() == 1) { - const int slice_dim = diff_dims[0]; - const int64_t dim_size = static_cast(view_src_ggml_shape[slice_dim]); - - if (view_stride[slice_dim] > 0 && relative_offset % view_stride[slice_dim] == 0) { - const int64_t begin_val = static_cast((relative_offset / view_stride[slice_dim]) % - static_cast(dim_size)); - const int64_t end_val = begin_val + static_cast(view_ggml_shape[slice_dim]); - - if (begin_val >= 0 && end_val <= dim_size) { - auto sliced = std::make_shared( - current, ov::op::v0::Constant::create(ov::element::i64, {1}, {begin_val}), - ov::op::v0::Constant::create(ov::element::i64, {1}, {end_val}), - ov::op::v0::Constant::create(ov::element::i64, {1}, {1}), - ov::op::v0::Constant::create(ov::element::i64, {1}, {slice_dim - axis_shift})); - - if (view_ov_shape.is_static()) { - auto reshaped = std::make_shared( - sliced, - ov::op::v0::Constant::create(ov::element::i64, {ndims}, view_ov_shape.to_shape()), - false); - reshaped->set_friendly_name(view_name); - return reshaped; - } - - sliced->set_friendly_name(view_name); - return sliced; - } - } - - int64_t tail_src_elems = 1; - int64_t tail_dst_elems = 1; - for (size_t i = slice_dim; i < ndims; ++i) { - tail_src_elems *= static_cast(view_src_ggml_shape[i]); - tail_dst_elems *= static_cast(view_ggml_shape[i]); - } - - const size_t elem_stride = view_stride[ndims - 1]; - int64_t tail_begin = 0; - if (elem_stride > 0) { - tail_begin = - static_cast((relative_offset / elem_stride) % static_cast(tail_src_elems)); - } - const int64_t tail_end = tail_begin + tail_dst_elems; - - if (tail_begin >= 0 && tail_end <= tail_src_elems) { - std::vector flat_shape; - flat_shape.reserve(slice_dim); - for (int i = 0; i < slice_dim; ++i) { - flat_shape.push_back(static_cast(view_src_ggml_shape[i])); - } - flat_shape.push_back(tail_src_elems); - const size_t flat_ndims = flat_shape.size(); - - auto flat = std::make_shared( - current, ov::op::v0::Constant::create(ov::element::i64, {flat_ndims}, flat_shape), false); - - auto sliced = std::make_shared( - flat, ov::op::v0::Constant::create(ov::element::i64, {1}, {tail_begin}), - ov::op::v0::Constant::create(ov::element::i64, {1}, {tail_end}), - ov::op::v0::Constant::create(ov::element::i64, {1}, {1}), - ov::op::v0::Constant::create(ov::element::i64, {1}, {slice_dim})); - - if (view_ov_shape.is_static()) { - auto reshaped = std::make_shared( - sliced, ov::op::v0::Constant::create(ov::element::i64, {ndims}, view_ov_shape.to_shape()), - false); - reshaped->set_friendly_name(view_name); - return reshaped; - } - - sliced->set_friendly_name(view_name); - return sliced; - } - } - - std::vector begin(ndims, 0); - std::vector end(ndims, 0); - std::vector step(ndims, 1); - std::vector axes(ndims, 0); - - size_t remaining_offset = relative_offset; - for (size_t i = 0; i < ndims; ++i) { - axes[i] = static_cast(i); - if (view_stride[i] > 0) { - begin[i] = static_cast(remaining_offset / view_stride[i]); - remaining_offset %= view_stride[i]; - } - end[i] = begin[i] + static_cast(view_ggml_shape[i]); - } - - bool in_bounds = view_src_ggml_shape.size() == ndims && view_ggml_shape.size() == ndims; - if (in_bounds) { - for (size_t i = 0; i < ndims; ++i) { - if (end[i] > static_cast(view_src_ggml_shape[i])) { - in_bounds = false; - break; - } - } - } - - if (in_bounds && remaining_offset == 0) { - auto sliced = std::make_shared( - current, ov::op::v0::Constant::create(ov::element::i64, {ndims}, begin), - ov::op::v0::Constant::create(ov::element::i64, {ndims}, end), - ov::op::v0::Constant::create(ov::element::i64, {ndims}, step), - ov::op::v0::Constant::create(ov::element::i64, {ndims}, axes)); - - sliced->set_friendly_name(view_name); - return sliced; - } - } else { - bool same_rank = view_stride.size() == view_src_stride.size() && - view_ggml_shape.size() == view_src_ggml_shape.size() && - view_stride.size() == view_ggml_shape.size(); - const size_t relative_offset = view_offset >= view_src_offset ? view_offset - view_src_offset : 0; - - if (same_rank) { - const size_t ndims = view_ggml_shape.size(); - std::vector diff_dims; - for (size_t i = 0; i < ndims; ++i) { - if (view_ggml_shape[i] != view_src_ggml_shape[i]) { - diff_dims.push_back(static_cast(i)); - } - } - - if (diff_dims.size() == 1) { - const size_t slice_dim = static_cast(diff_dims[0]); - bool suffix_stride_match = true; - for (size_t i = slice_dim + 1; i < ndims; ++i) { - if (view_stride[i] != view_src_stride[i]) { - suffix_stride_match = false; - break; - } - } - - if (suffix_stride_match && view_src_stride[slice_dim] > 0 && - relative_offset % view_src_stride[slice_dim] == 0) { - const int64_t begin_val = static_cast(relative_offset / view_src_stride[slice_dim]); - const int64_t end_val = begin_val + static_cast(view_ggml_shape[slice_dim]); - const int64_t dim_size = static_cast(view_src_ggml_shape[slice_dim]); - - if (begin_val >= 0 && end_val <= dim_size) { - auto sliced = std::make_shared( - current, ov::op::v0::Constant::create(ov::element::i64, {1}, {begin_val}), - ov::op::v0::Constant::create(ov::element::i64, {1}, {end_val}), - ov::op::v0::Constant::create(ov::element::i64, {1}, {1}), - ov::op::v0::Constant::create(ov::element::i64, {1}, {static_cast(slice_dim)})); - sliced->set_friendly_name(view_name); - return sliced; - } - } - } - } - - size_t view_elems = 1; - size_t src_elems = 1; - if (same_rank) { - for (size_t i = 0; i < view_ggml_shape.size(); ++i) { - view_elems *= view_ggml_shape[i]; - src_elems *= view_src_ggml_shape[i]; - } - } - - bool same_num_elements = same_rank && view_elems == src_elems; - - if (same_rank && relative_offset == 0 && same_num_elements) { - auto reshape_pattern = build_reshape_pattern(view_ov_shape, view_ggml_shape); - - auto reshaped = std::make_shared( - current, ov::op::v0::Constant::create(ov::element::i64, {reshape_pattern.size()}, reshape_pattern), - false); - reshaped->set_friendly_name(view_name); - return reshaped; - } - - if (same_rank) { - const size_t ndims = view_ggml_shape.size(); - - // Match views that can be expressed as a regular strided slice over the - // already reconstructed source tensor, e.g. offset on one axis plus step > 1 - // on another axis. - bool is_regular_slice = view_src_ggml_shape.size() == ndims; - std::vector begin(ndims, 0); - std::vector end(ndims, 0); - std::vector step(ndims, 1); - std::vector axes(ndims, 0); - size_t remaining_offset = relative_offset; - - if (is_regular_slice) { - for (size_t i = 0; i < ndims; ++i) { - axes[i] = static_cast(i); - - if (view_src_stride[i] == 0 || view_stride[i] == 0 || - view_stride[i] % view_src_stride[i] != 0) { - is_regular_slice = false; - break; - } - - step[i] = static_cast(view_stride[i] / view_src_stride[i]); - if (step[i] <= 0) { - is_regular_slice = false; - break; - } - - begin[i] = static_cast(remaining_offset / view_src_stride[i]); - remaining_offset %= view_src_stride[i]; - - if (view_ggml_shape[i] == 0) { - end[i] = begin[i]; - continue; - } - - end[i] = begin[i] + step[i] * static_cast(view_ggml_shape[i] - 1) + 1; - - if (begin[i] < 0 || end[i] > static_cast(view_src_ggml_shape[i])) { - is_regular_slice = false; - break; - } - } - } - - if (is_regular_slice && remaining_offset == 0) { - auto sliced = std::make_shared( - current, ov::op::v0::Constant::create(ov::element::i64, {ndims}, begin), - ov::op::v0::Constant::create(ov::element::i64, {ndims}, end), - ov::op::v0::Constant::create(ov::element::i64, {ndims}, step), - ov::op::v0::Constant::create(ov::element::i64, {ndims}, axes)); - - sliced->set_friendly_name(view_name); - return sliced; - } - - const size_t elem_stride = view_src_stride.back(); - const bool aligned_offset = elem_stride > 0 && relative_offset % elem_stride == 0; - - if (aligned_offset) { - size_t suffix_start = 0; - size_t expected_stride = elem_stride; - for (int i = static_cast(ndims) - 1; i >= 0; --i) { - if (view_stride[i] != expected_stride) { - suffix_start = static_cast(i + 1); - break; - } - expected_stride *= view_ggml_shape[i]; - } - - size_t prefix_elems = 1; - size_t suffix_elems = 1; - for (size_t i = 0; i < suffix_start; ++i) { - prefix_elems *= view_ggml_shape[i]; - } - for (size_t i = suffix_start; i < ndims; ++i) { - suffix_elems *= view_ggml_shape[i]; - } - - if (prefix_elems > 0 && src_elems % prefix_elems == 0) { - const size_t src_tail_elems = src_elems / prefix_elems; - const int64_t tail_begin = static_cast(relative_offset / elem_stride); - const int64_t tail_end = tail_begin + static_cast(suffix_elems); - - if (tail_begin >= 0 && tail_end <= static_cast(src_tail_elems)) { - auto prefix_tail_pattern = build_prefix_tail_reshape_pattern( - view_ov_shape, view_ggml_shape, suffix_start, static_cast(src_tail_elems)); - - auto prefix_tail = std::make_shared( - current, - ov::op::v0::Constant::create(ov::element::i64, {prefix_tail_pattern.size()}, - prefix_tail_pattern), - false); - - ov::Output selected = prefix_tail; - if (tail_begin != 0 || tail_end != static_cast(src_tail_elems)) { - selected = std::make_shared( - prefix_tail, ov::op::v0::Constant::create(ov::element::i64, {1}, {tail_begin}), - ov::op::v0::Constant::create(ov::element::i64, {1}, {tail_end}), - ov::op::v0::Constant::create(ov::element::i64, {1}, {1}), - ov::op::v0::Constant::create(ov::element::i64, {1}, - {static_cast(suffix_start)})); - } - - auto reshape_pattern = build_reshape_pattern(view_ov_shape, view_ggml_shape); - auto reshaped = std::make_shared( - selected, - ov::op::v0::Constant::create(ov::element::i64, {reshape_pattern.size()}, - reshape_pattern), - false); - reshaped->set_friendly_name(view_name); - return reshaped; - } - } - } - } - - return current; - } - - (void) view_name; - (void) view_src_ov_shape; - (void) view_src_name; - - return current; - }; - - // Special case: ggml collapses VIEW-of-VIEW chains so that `view_offs` is always an - // ABSOLUTE offset from the true root allocation, regardless of how many VIEW levels - // are in between (see ggml_new_tensor_impl). `src[0]` is still the immediate op-graph - // parent though, which can be a DIFFERENT (already narrowed) VIEW with the SAME ggml - // shape as this one but a different absolute offset -- e.g. a per-layer deepstack - // slice `view_2d(embd, n_embd, n_tokens, embd->nb[1], layer*n_embd*sizeof(float))` - // whose src[0] ("embd") is itself already a zero-offset VIEW of the true root (the - // padded embedding). Chaining through "embd" here would try to re-slice an already - // 2-narrowed tensor using a root-relative offset, going out of bounds and silently - // falling back to a no-op (returning the wrong, already-resolved sibling slice). - // Detect this (same shape as the immediate src, but different absolute offset) and - // re-slice directly from the untouched root using the innermost view's absolute - // offset against the ROOT's own shape/stride instead of chaining through src[0]. - { - auto innermost_offset = context.get_view_input_offset(input_index, 0); - auto innermost_src_offset = context.get_view_input_src_offset(input_index, 0); - auto innermost_shape = context.get_view_input_ggml_shape(input_index, 0); - auto innermost_src_shape = context.get_view_input_src_ggml_shape(input_index, 0); - if (innermost_offset != innermost_src_offset && innermost_shape == innermost_src_shape) { - size_t root_view_idx = view_input_size - 1; - auto root_ggml_shape = context.get_view_input_src_ggml_shape(input_index, root_view_idx); - auto root_stride = context.get_view_input_src_stride(input_index, root_view_idx); - auto root_offset = context.get_view_input_src_offset(input_index, root_view_idx); - auto root_ov_shape = context.get_view_input_src_ov_shape(input_index, root_view_idx); - auto root_name = context.get_view_input_src_name(input_index, root_view_idx); - auto innermost_stride = context.get_view_input_stride(input_index, 0); - auto innermost_ov_shape = context.get_view_input_ov_shape(input_index, 0); - auto innermost_name = context.get_view_input_name(input_index, 0); - - return process_single_view(input, innermost_offset, innermost_stride, innermost_shape, innermost_ov_shape, - innermost_name, root_offset, root_stride, root_ggml_shape, root_ov_shape, - root_name); - } - } - - // Process views from the base tensor (last) to the current view (first) - // Start with the base tensor - ov::Output current = input; - - // Process each view in reverse order (from base to current) - for (int view_idx = view_input_size - 1; view_idx >= 0; view_idx--) { - auto view_offset = context.get_view_input_offset(input_index, view_idx); - auto view_stride = context.get_view_input_stride(input_index, view_idx); - auto view_ggml_shape = context.get_view_input_ggml_shape(input_index, view_idx); - auto view_ov_shape = context.get_view_input_ov_shape(input_index, view_idx); - auto view_name = context.get_view_input_name(input_index, view_idx); - - // print view info - // std::cout << "View " << view_idx << ": name = " << view_name << ", offset = " << view_offset << ", stride = [" - // << view_stride[0] << "," << view_stride[1] << "," << view_stride[2] << "," << view_stride[3] - // << "], ggml shape = [" << view_ggml_shape[0] << "," << view_ggml_shape[1] << "," - // << view_ggml_shape[2] << "," << view_ggml_shape[3] << "], ov shape = " << view_ov_shape << std::endl; - - auto view_src_offset = context.get_view_input_src_offset(input_index, view_idx); - auto view_src_stride = context.get_view_input_src_stride(input_index, view_idx); - auto view_src_ggml_shape = context.get_view_input_src_ggml_shape(input_index, view_idx); - auto view_src_ov_shape = context.get_view_input_src_ov_shape(input_index, view_idx); - auto view_src_name = context.get_view_input_src_name(input_index, view_idx); - // print source view info - // std::cout << "View " << view_idx << ": source name = " << view_src_name - // << ", source offset = " << view_src_offset << ", source stride = [" << view_src_stride[0] << "," - // << view_src_stride[1] << "," << view_src_stride[2] << "," << view_src_stride[3] - // << "], source ggml shape = [" << view_src_ggml_shape[0] << "," << view_src_ggml_shape[1] << "," - // << view_src_ggml_shape[2] << "," << view_src_ggml_shape[3] - // << "], source ov shape = " << view_src_ov_shape << std::endl; - - current = process_single_view(current, view_offset, view_stride, view_ggml_shape, view_ov_shape, view_name, - view_src_offset, view_src_stride, view_src_ggml_shape, view_src_ov_shape, - view_src_name); - } - - return current; -} - -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/openvino/utils.h b/ggml/src/ggml-openvino/openvino/utils.h deleted file mode 100644 index effa1f843778..000000000000 --- a/ggml/src/ggml-openvino/openvino/utils.h +++ /dev/null @@ -1,102 +0,0 @@ -#pragma once - -#include "node_context.h" - -#include -#include -#include -#include -#include -#include - -namespace ov { -namespace frontend { -namespace ggml { - -std::string getCurrentTime(); - -void num_inputs_check(const NodeContext & context, size_t min_inputs, size_t max_inputs); - -int non_cont_dim(std::vector ne, std::vector nb); - -template std::vector argsort_descend(const std::vector & v) { - std::vector idx(v.size()); - std::iota(idx.begin(), idx.end(), 0); - std::sort(idx.begin(), idx.end(), [&v](int i1, int i2) { return v[i1] > v[i2]; }); - return idx; -} - -template std::vector sorted_descend(std::vector v) { - std::sort(v.begin(), v.end(), [](T a, T b) { return a > b; }); - return v; -} - -template bool is_permuted(const std::vector & strides) { - for (size_t i = 0; i < strides.size() - 1; ++i) { - if (strides[i] < strides[i + 1]) { - return true; - } - } - return false; -} - -template std::vector permute(const std::vector & x, const std::vector & perm) { - std::vector result; - result.reserve(perm.size()); - for (int i : perm) { - result.push_back(x[i]); - } - return result; -} - -std::shared_ptr get_dimensions(const std::shared_ptr & shape, - const std::vector & dims); -std::shared_ptr get_dimensions(const std::shared_ptr & node, const std::vector & dims); - -OutputVector rename_outputs_with_suffix(const OutputVector & outputs, const std::string & suffix); - -std::pair, ov::Output> make_sin_cos(int32_t * rope_params, - std::shared_ptr inp_pos, - std::shared_ptr rope_freqs_weight = nullptr, - int mode = 0, - bool stateful = false, - int64_t head_dim = 0); - -ov::Output process_view_input(const NodeContext & context, int input_index, int slice_len = 0, int axis = -1); - -ov::Output process_view_input_new(const NodeContext & context, int input_index); - -namespace op { -template OutputVector translate_1to1_match_2_inputs(const NodeContext & context) { - num_inputs_check(context, 2, 2); - auto input_0 = process_view_input_new(context, 0); - auto input_1 = process_view_input_new(context, 1); - - auto output_type = context.get_output_type(); - if (input_0.get_element_type() != input_1.get_element_type()) { - if (input_0.get_element_type() != ov::element::f32) { - input_0 = std::make_shared(input_0, ov::element::f32); - } - if (input_1.get_element_type() != ov::element::f32) { - input_1 = std::make_shared(input_1, ov::element::f32); - } - } - - ov::Output res = std::make_shared(input_0, input_1); - if (res.get_element_type() != output_type) { - res = std::make_shared(res, output_type); - } - return rename_outputs_with_suffix({res}, context.get_name()); -} - -template OutputVector translate_1to1_match_1_input(const NodeContext & context) { - num_inputs_check(context, 1, 1); - auto input = process_view_input_new(context, 0); - auto res = std::make_shared(input); - return rename_outputs_with_suffix({res}, context.get_name()); -} -} // namespace op - -} // namespace ggml -} // namespace frontend -} // namespace ov diff --git a/ggml/src/ggml-openvino/utils.cpp b/ggml/src/ggml-openvino/utils.cpp index 67a4123cb195..0502fd41e8d5 100644 --- a/ggml/src/ggml-openvino/utils.cpp +++ b/ggml/src/ggml-openvino/utils.cpp @@ -5,9 +5,13 @@ #include "ggml-openvino.h" #include "ggml-openvino/ggml-decoder.h" #include "ggml.h" +#include "align-eltwise-ranks.h" +#include "openvino/pass/manager.hpp" +#include "kv-state-seq-axis.h" #include "model-cache.h" -#include "openvino/frontend.h" -#include "openvino/input_model.h" +#include +#include +#include #include #include @@ -20,12 +24,14 @@ #include #include #include +#include #include #include #include #include #include #include +#include #include #include #include @@ -46,8 +52,9 @@ namespace { // * ggml KV layout is a contiguous [1, 1, ctx_per_seq, n_heads_kv*head_size] // so the first n_kv rows are the live prefix and shrinking the ctx axis // gives a valid tensor over the same host storage -// * not an SWA layer (ring cache): once the window has wrapped the first -// n_kv rows no longer contain the live prefix +// * SWA layers slice to attention_size_swa. llama.cpp defines n_kv as the highest used cell + 1, so +// the first n_kv rows always cover every live cell, including after the ring has wrapped (n_kv then +// equals the cache size and the full tensor is bound) // On any unmet pre-condition returns std::nullopt; the caller falls back to // the full-size tensor. std::optional try_make_kv_sliced_tensor(const std::shared_ptr & ggml_decoder, @@ -64,7 +71,7 @@ std::optional try_make_kv_sliced_tensor(const std::shared_ptrget_tensor_used_op(ggml_tensor); - if (!GgmlOvDecoder::is_kvcache(ggml_tensor, op)) { + if (!GgmlOvDecoder::is_kvcache(ggml_tensor, op) || ggml_decoder->is_transposed_kv_cache(ggml_tensor)) { return std::nullopt; } @@ -80,12 +87,11 @@ std::optional try_make_kv_sliced_tensor(const std::shared_ptris_swa_layer(layer); - if (is_swa) { - return std::nullopt; - } - const int ctx_per_seq = ggml_decoder->get_ctx_per_seq(); - const int n_kv = compute_params.attention_size; + const int ctx_per_seq = is_swa ? ggml_decoder->get_ctx_per_seq_swa() : ggml_decoder->get_ctx_per_seq(); + const int n_kv = is_swa ? compute_params.attention_size_swa : compute_params.attention_size; if (ctx_per_seq <= 0 || n_kv <= 0 || n_kv >= ctx_per_seq) { return std::nullopt; } @@ -108,18 +114,146 @@ std::optional try_make_kv_sliced_tensor(const std::shared_ptrdata); } +// The stateful KV state is Concat-appended and holds exactly the positions 0..last_pos, while ggml +// binds masks sized to its own (padded) cache. Re-slice the full mask to the state length, and rebuild +// the sliding-window mask from positions, since ggml caps self_kq_mask_swa at the SWA cache size +// while the state keeps growing. The window is the runtime input "swa_window": it is recovered from +// each call's ggml mask, and the value seen when the model is compiled (e.g. 1 for a one-token +// checkpoint chunk) does not hold for later calls. Runs after GGUFMakeStateful. +static bool rewrite_stateful_masks(const std::shared_ptr & model) { + std::shared_ptr inp_pos; + std::vector> masks; + std::shared_ptr mask_swa; + for (const auto & param : model->get_parameters()) { + const auto & name = param->get_friendly_name(); + if (name == "inp_pos") { + inp_pos = param; + } else if (name == "self_kq_mask" || name == "KQ_mask") { + masks.push_back(param); + } else if (name == "self_kq_mask_swa") { + mask_swa = param; + } + } + if (!inp_pos) { + return false; + } + using namespace ov::op; + auto i64 = [](int64_t v) { return v0::Constant::create(ov::element::i64, {1}, {v}); }; + auto i64_scalar = [](int64_t v) { return v0::Constant::create(ov::element::i64, {}, {v}); }; + + // inp_pos is [1, 1, 1, n_planes * n_tokens]; the first n_tokens entries are the token positions. + auto pos_flat = std::make_shared(std::make_shared(inp_pos, i64(-1), false), + ov::element::i64); + auto last_pos_plus_one = [&](const ov::Output & mask) -> ov::Output { + auto n_tokens = std::make_shared(std::make_shared(mask, ov::element::i64), i64(2), + i64_scalar(0)); + auto query_pos = std::make_shared(pos_flat, i64(0), n_tokens, i64(1), i64(0)); + auto last = std::make_shared(query_pos, i64(-1), i64_scalar(0)); + return std::make_shared(last, i64(1)); + }; + // Snapshot the consumers before building the replacement, which reads the Parameter itself. + auto replace_uses = [](const std::set> & uses, const ov::Output & replacement) { + for (auto input : uses) { + input.replace_source_output(replacement); + } + }; + auto slice_to_state = [&](const std::shared_ptr & mask) { + const auto uses = mask->output(0).get_target_inputs(); + auto sliced = std::make_shared(mask, i64(0), last_pos_plus_one(mask), i64(1), i64(3)); + replace_uses(uses, sliced); + }; + for (const auto & mask : masks) { + slice_to_state(mask); + } + if (mask_swa) { + // keep[q, k] = 0 <= pos[q] - k < swa_window, over cached positions k = 0..last_pos. + auto swa_window = std::make_shared(ov::element::i64, ov::PartialShape{1}); + swa_window->set_friendly_name("swa_window"); + swa_window->output(0).get_tensor().set_names({"swa_window"}); + model->add_parameters({swa_window}); + const auto uses = mask_swa->output(0).get_target_inputs(); + auto n_tokens = std::make_shared(std::make_shared(mask_swa, ov::element::i64), + i64(2), i64_scalar(0)); + auto query_pos = std::make_shared(pos_flat, i64(0), n_tokens, i64(1), i64(0)); + auto total_len = std::make_shared(last_pos_plus_one(mask_swa)); + auto cached_pos = std::make_shared(i64_scalar(0), total_len, i64_scalar(1), ov::element::i64); + auto query_col = std::make_shared( + query_pos, v0::Constant::create(ov::element::i64, {2}, {-1, 1}), false); + auto cached_row = std::make_shared( + cached_pos, v0::Constant::create(ov::element::i64, {2}, {1, -1}), false); + auto diff = std::make_shared(query_col, cached_row); + auto keep = std::make_shared(std::make_shared(diff, i64_scalar(0)), + std::make_shared(diff, swa_window)); + std::shared_ptr mask = std::make_shared( + keep, v0::Constant::create(ov::element::f32, {}, {0.0f}), + v0::Constant::create(ov::element::f32, {}, {-std::numeric_limits::infinity()})); + mask = std::make_shared(mask, v0::Constant::create(ov::element::i64, {2}, {0, 1})); + mask = std::make_shared(mask, mask_swa->get_element_type()); + replace_uses(uses, mask); + } + model->validate_nodes_and_infer_types(); + return true; +} + +// The GGUF frontend converts to a stateless model; a stateful execution is opted into by the caller, +// which registers the KV-cache rewrite (and the mask rewrite that goes with it) as extensions. +static std::shared_ptr convert_to_ov_model(const std::shared_ptr & decoder) { + ov::frontend::gguf::FrontEnd frontend; + // Work around a GPU-plugin defect with rank-mismatched eltwise ops fused into `rms` (see the pass). Opt out with + // GGML_OPENVINO_DISABLE_ELTWISE_RANK_ALIGN=1. + if (ggml_openvino_is_gpu() && !ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_ELTWISE_RANK_ALIGN")) { + frontend.add_extension(std::make_shared( + [](std::shared_ptr model) { + ov::pass::Manager manager; + manager.register_pass(); + return manager.run_passes(model); + })); + } + if (decoder->is_stateful()) { + frontend.add_extension(std::make_shared( + ov::frontend::gguf::pass::GGUFMakeStateful())); + // Must run after GGUFMakeStateful, which creates the ReadValue/Assign pairs it rewrites. + if (!ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_KV_STATE_RELAYOUT")) { + frontend.add_extension( + std::make_shared(ggml_openvino::KVStateSeqAxis())); + } + frontend.add_extension(std::make_shared( + [](std::shared_ptr model) { return rewrite_stateful_masks(model); })); + } + return frontend.convert(frontend.load(std::static_pointer_cast(decoder))); +} + + +// For a KV cache in the 4-D layout (GgmlOvDecoder::is_kvcache_4d), a [1, ctx, n_heads_kv, head_size] tensor over the +// whole cache. Its shape stays the same from one inference to the next, so the GPU plugin keeps writing the cache in +// place and reading it without a copy. +std::optional try_make_kv_4d_tensor(const std::shared_ptr & ggml_decoder, + const ggml_tensor * ggml_tensor) { + if (!ggml_decoder->is_kvcache_4d_input(ggml_tensor)) { + return std::nullopt; + } + const size_t n_heads_kv = static_cast(ggml_decoder->get_n_heads_kv_for_tensor(ggml_tensor)); + const ov::Shape shape{1, static_cast(ggml_tensor->ne[1]), n_heads_kv, + static_cast(ggml_tensor->ne[0]) / n_heads_kv}; + if (ggml_openvino_buffer_is_remote(ggml_tensor)) { + auto remote_context = ggml_openvino_get_remote_context(); + auto gpu_context = remote_context->as(); + return gpu_context.create_tensor(ggml_decoder->get_ov_type(ggml_tensor), shape, ggml_tensor->data); + } + return ov::Tensor(GgmlOvDecoder::get_ov_type(ggml_tensor), shape, ggml_tensor->data); +} + static uint64_t ggml_openvino_model_cache_extra_cfg(bool stateful, bool recurrent) { - const char * manual_gqa_env = ggml_openvino_getenv_str("GGML_OPENVINO_MANUAL_GQA_ATTN"); - const bool manual_gqa_enabled = manual_gqa_env != nullptr ? - ggml_openvino_getenv_int("GGML_OPENVINO_MANUAL_GQA_ATTN") > 0 : - ggml_openvino_is_gpu(); + const bool manual_gqa_enabled = ggml_openvino_manual_gqa_attn_enabled(); uint64_t extra_cfg = 1; // Graph-ordinal port names (invalidate older disk-cache blobs). extra_cfg = extra_cfg * 131 + (stateful ? (recurrent ? 2u : 1u) : 0u); extra_cfg = extra_cfg * 131 + (ggml_openvino_reduce_compile_mem_enabled() ? 1u : 0u); extra_cfg = extra_cfg * 131 + (ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_KV_SLICE") ? 1u : 0u); + extra_cfg = extra_cfg * 131 + (ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_KV_4D") ? 1u : 0u); extra_cfg = extra_cfg * 131 + (manual_gqa_enabled ? 1u : 0u); extra_cfg = extra_cfg * 131 + (ggml_openvino_getenv_int("GGML_OPENVINO_PROFILING") >= 2 ? 1u : 0u); + extra_cfg = extra_cfg * 131 + (ggml_openvino_shape_from_mask_enabled() ? 1u : 0u); return extra_cfg; } @@ -132,7 +266,7 @@ std::map> get_weight_names(ggml_cgraph * } // A conservative, exact in-process key, evaluated only on a context-local cache -// miss. Include topology, layouts, op parameters, constant extra inputs and weight +// miss. Include topology, layouts, op parameters, model inputs and weight // allocation identities. Never use a sampled weight hash or a graph name alone: // different models can have identical topology. OV buffer IDs survive address reuse. static std::string compiled_graph_key(const ggml_cgraph * graph, @@ -194,17 +328,14 @@ static std::string compiled_graph_key(const ggml_cgraph * graph, for (int i = 0; i < graph->n_leafs; ++i) { visit(graph->leafs[i]); } - for (const auto & input : decoder.get_model_extra_inputs()) { - append_string(input.first); - append_string(input.second.type.get_type_name()); - append(input.second.shape.size()); - for (auto dim : input.second.shape) { - append(dim); - } - append(input.second.is_parameter); - if (!input.second.is_parameter) { - append(input.second.value); + for (const auto & input : decoder.get_model_inputs()) { + auto param = std::dynamic_pointer_cast(input.second); + if (!param) { + continue; } + append_string(input.first); + append_string(param->get_element_type().get_type_name()); + append_string(param->get_partial_shape().to_string()); } // Without an allocation generation, pointer reuse could select stale weights. // Such graphs still get private requests; they simply do not share compilation. @@ -241,7 +372,7 @@ static std::string dynamic_graph_signature(const ggml_cgraph * graph, append(layer); append(heads); } - for (const auto & [name, input] : decoder.get_model_inputs()) { + for (const auto & [name, input] : decoder.get_model_input_infos()) { append_string(name); append_string(input.type.get_type_name()); } @@ -249,14 +380,6 @@ static std::string dynamic_graph_signature(const ggml_cgraph * graph, append_string(name); append(tensor->type); } - for (const auto & [name, input] : decoder.get_model_extra_inputs()) { - append_string(name); - append_string(input.type.get_type_name()); - append(input.is_parameter); - if (!input.is_parameter) { - append(input.value); - } - } return key; } @@ -267,8 +390,7 @@ static bool compiled_model_matches_graph(const ov::CompiledModel & model, if (model.inputs().size() != inputs.size() || model.outputs().size() != outputs.size()) { return false; } - const auto & graph_inputs = decoder.get_model_inputs(); - const auto & extra_inputs = decoder.get_model_extra_inputs(); + const auto & graph_inputs = decoder.get_model_input_infos(); const auto & graph_outputs = decoder.get_model_outputs(); for (size_t i = 0; i < inputs.size(); ++i) { const auto & port = model.input(i); @@ -280,9 +402,10 @@ static bool compiled_model_matches_graph(const ov::CompiledModel & model, } continue; } - auto extra_it = extra_inputs.find(inputs[i]); - if (extra_it == extra_inputs.end() || port.get_element_type() != extra_it->second.type || - !port.get_partial_shape().compatible(ov::PartialShape(extra_it->second.shape))) { + // Inputs added after conversion: beam_idx and swa_window by the stateful rewrites, plus runtime + // values the decoder binds per call. + if (inputs[i] != "beam_idx" && inputs[i] != "swa_window" && + !decoder.get_runtime_input_value(inputs[i]).has_value()) { return false; } } @@ -378,6 +501,9 @@ ov::Tensor create_ov_output_tensor(const std::shared_ptr & ggml_d const std::shared_ptr & infer_request, int output_index, const ggml_tensor * ggml_tensor) { + if (auto kv_4d = try_make_kv_4d_tensor(ggml_decoder, ggml_tensor)) { + return *kv_4d; + } if (auto sliced = try_make_kv_sliced_tensor(ggml_decoder, std::string(ggml_tensor->name), ggml_tensor)) { return *sliced; } @@ -482,30 +608,41 @@ ov::Tensor make_contiguous_split_input_tensor(const struct ggml_tensor * ggml_te ov::Tensor convert_ggml_input_to_ov(const std::shared_ptr & ggml_decoder, const std::string & name) { const auto * ggml_tensor = ggml_decoder->get_input_ggml_tensor(name); + if (auto kv_4d = try_make_kv_4d_tensor(ggml_decoder, ggml_tensor)) { + return *kv_4d; + } if (auto sliced = try_make_kv_sliced_tensor(ggml_decoder, name, ggml_tensor)) { return *sliced; } - if (ggml_tensor->extra != nullptr && !ggml_decoder->is_splited_model()) { + // An external view (a slice of a tensor owned by another split) shares its root's extra, which describes + // the whole root rather than the slice, so it is bound from its own data and extent below. + const bool external_view = ggml_decoder->is_external_view_input(ggml_tensor); + if (ggml_tensor->extra != nullptr && !ggml_decoder->is_splited_model() && !external_view) { auto * extra_base = static_cast(ggml_tensor->extra); if (extra_base->type == ggml_openvino_extra_base::Type::TENSOR) { // GGML_LOG_DEBUG("Using ggml_tensor->extra as ov::Tensor for input: %s\n", name.c_str()); auto * tensor_extra = static_cast(extra_base); - return *tensor_extra->tensor; + // Views and reshapes of a backend tensor share its extra; that tensor has the storage's + // shape (e.g. [1, 1, T, 512] for a Vcur viewed as [1, T, 2, 256]), so only use it as is. + if (ggml_tensor->view_src == nullptr || + tensor_extra->tensor->get_shape() == ggml_decoder->get_shape(ggml_tensor)) { + return *tensor_extra->tensor; + } } } // GGML_LOG_DEBUG("Converting ggml tensor to ov::Tensor for input: %s\n", name.c_str()); auto * input_data = ggml_tensor->data; ov::Shape input_shape; - if (ggml_tensor->op == GGML_OP_VIEW && !ggml_decoder->is_splited_model()) { + if (ggml_tensor->op == GGML_OP_VIEW && !ggml_decoder->is_splited_model() && !external_view) { // This case is added to make test-backend-ops work input_shape = GgmlOvDecoder::get_shape(ggml_tensor->view_src); } else { input_shape = GgmlOvDecoder::get_shape(ggml_tensor); } - if (ggml_decoder->is_splited_model() && !ggml_is_contiguous(ggml_tensor)) { + if ((ggml_decoder->is_splited_model() || external_view) && !ggml_is_contiguous(ggml_tensor)) { return make_contiguous_split_input_tensor(ggml_tensor, input_shape); } @@ -514,21 +651,24 @@ ov::Tensor convert_ggml_input_to_ov(const std::shared_ptr & ggml_ } ov::Tensor get_ov_input_tensor(const std::shared_ptr & ggml_decoder, const std::string & param_name) { - ov::Tensor input_tensor; - auto extra_input = ggml_decoder->get_model_extra_inputs().find(param_name); - if (extra_input != ggml_decoder->get_model_extra_inputs().end()) { - input_tensor = ov::Tensor(extra_input->second.type, extra_input->second.shape); - *input_tensor.data() = extra_input->second.value; - } else { - input_tensor = convert_ggml_input_to_ov(ggml_decoder, param_name); + if (param_name == "beam_idx") { + // Added by GGUFMakeStateful to reorder the past KV state; ggml decodes a single beam. + ov::Tensor input_tensor(ov::element::i32, ov::Shape{1}); + *input_tensor.data() = 0; + return input_tensor; } - return input_tensor; + if (const auto value = ggml_decoder->get_runtime_input_value(param_name)) { + ov::Tensor input_tensor(ov::element::i64, ov::Shape{1}); + *input_tensor.data() = *value; + return input_tensor; + } + return convert_ggml_input_to_ov(ggml_decoder, param_name); } ov::Tensor get_ov_input_tensor_static_decode(const std::shared_ptr & ggml_decoder, const std::string & param_name) { // NPU decoding stage - if (ggml_decoder->get_model_extra_inputs().count(param_name)) { + if (ggml_decoder->get_runtime_input_value(param_name).has_value()) { return get_ov_input_tensor(ggml_decoder, param_name); } const auto * ggml_tensor = ggml_decoder->get_input_ggml_tensor(param_name); @@ -585,11 +725,6 @@ ov::Tensor get_ov_input_tensor_static_prefill(const std::shared_ptr() = (int64_t) chunk_valid_size; - return input_tensor; - } if (chunk_index > 0 && param_name == "cache_rs_reset_len") { // The recurrent-state clear belongs to the start of the sequence. Re-applying it on every // chunk would wipe the state accumulated by the preceding chunks, so disable it (a zero @@ -598,7 +733,7 @@ ov::Tensor get_ov_input_tensor_static_prefill(const std::shared_ptr() = 0; return input_tensor; } - if (ggml_decoder->get_model_extra_inputs().count(param_name)) { + if (ggml_decoder->get_runtime_input_value(param_name).has_value()) { return get_ov_input_tensor(ggml_decoder, param_name); } const auto * ggml_tensor = ggml_decoder->get_input_ggml_tensor(param_name); @@ -676,6 +811,15 @@ ov::Tensor get_ov_input_tensor_static_prefill(const std::shared_ptr(); + const auto * src = static_cast(ggml_tensor->data) + chunk_index * chunk_size; + std::copy(src, src + chunk_valid_size, dst); + std::fill(dst + chunk_valid_size, dst + chunk_size, 1.0f); + return input_tensor; + } + if (GgmlOvDecoder::is_inp_mean(ggml_tensor, op)) { const size_t n_seqs = ggml_tensor->ne[1]; const size_t src_stride = ggml_tensor->ne[0]; @@ -732,8 +876,7 @@ enum ggml_status naive_compute(ggml_cgraph * cgraph, bool naive = true; auto model_weights = GgmlOvDecoder::create_weight_nodes(cgraph, naive); auto decoder = std::make_shared(cgraph, model_weights); - auto input_model = std::make_shared(decoder); - auto model = ov::frontend::ggml::FrontEnd::convert(input_model, naive); + auto model = convert_to_ov_model(decoder); if (ggml_openvino_getenv_int("GGML_OPENVINO_DUMP_IR")) { ov::serialize(model, "IR_naive.xml"); } @@ -765,7 +908,6 @@ enum ggml_status naive_compute(ggml_cgraph * cgraph, // Destroy the frontend graph under the compilation lock as well: it can // still own edges into the shared weight nodes. model.reset(); - input_model.reset(); decoder->clear_model_weights(); model_weights.clear(); compile_lock.unlock(); @@ -899,7 +1041,8 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, const std::share ggml_decoder = entry->ptr; old_m_params = ggml_decoder->get_model_params(); if (!ggml_decoder->is_splited_model()) { - cache_hit = old_m_params.can_reuse_dynamically(m_params); + cache_hit = old_m_params.can_reuse_dynamically(m_params) && + ggml_decoder->get_compute_params().same_graph_extents(c_params); } } @@ -913,7 +1056,6 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, const std::share if (old_m_params.kv_buffer_changed(m_params) || !ggml_decoder->is_bound_to(cgraph)) { ggml_decoder->update_io(cgraph); } - ggml_decoder->add_extra_inputs(); { std::lock_guard map_lock(r_ctx->ctx_mutex); infer_request = r_ctx->infer_request_cache.at(key); @@ -924,12 +1066,13 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, const std::share if (stateful_kv_only) { const auto * inp_pos = get_inp_pos_tensor(cgraph); int32_t * pos_data = (int32_t *) inp_pos->data; - auto pos_shape = GgmlOvDecoder::get_shape(inp_pos); + // IMROPE packs several position planes per token; the state grows by tokens. + const auto n_tokens = static_cast(get_inp_pos_n_tokens(cgraph, inp_pos)); if (pos_data[0] == 0) { infer_request->reset_state(); - r_ctx->stateful_kv_size = pos_shape[3]; + r_ctx->stateful_kv_size = n_tokens; } else if (r_ctx->stateful_kv_size == static_cast(pos_data[0])) { - r_ctx->stateful_kv_size += pos_shape[3]; + r_ctx->stateful_kv_size += n_tokens; } else { const size_t pos_begin = static_cast(pos_data[0]); const bool refill = pos_begin > r_ctx->stateful_kv_size; @@ -1013,7 +1156,7 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, const std::share ov::Tensor new_state_tensor(state_tensor, begin, end); state.set_state(new_state_tensor); } - r_ctx->stateful_kv_size = pos_begin + pos_shape[3]; + r_ctx->stateful_kv_size = pos_begin + n_tokens; } } @@ -1187,8 +1330,7 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, const std::share r_ctx->stateful, model_is_splitted); decoder_end_time = ggml_time_us(); - auto input_model = std::make_shared(ggml_decoder); - model = ov::frontend::ggml::FrontEnd::convert(input_model); + model = convert_to_ov_model(ggml_decoder); ggml_decoder->clear_model_weights(); conversion_end_time = ggml_time_us(); @@ -1271,7 +1413,8 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, const std::share if (stateful_kv_only && cache_enabled) { const auto * inp_pos = get_inp_pos_tensor(cgraph); - auto pos_shape = GgmlOvDecoder::get_shape(inp_pos); + // IMROPE packs several position planes per token; the state grows by tokens. + const auto n_tokens = static_cast(get_inp_pos_n_tokens(cgraph, inp_pos)); // A freshly compiled model starts with an empty state, so it can only serve a // sequence from its beginning. A non-zero start position means the KV history was // built elsewhere (a restored ggml cache), which the state cannot adopt. @@ -1284,7 +1427,7 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, const std::share pos_begin); return GGML_STATUS_FAILED; } - r_ctx->stateful_kv_size = pos_shape[3]; + r_ctx->stateful_kv_size = n_tokens; const auto kv_param_res_names = ggml_decoder->get_kv_param_res_names(); for (const auto & pair : kv_param_res_names) { r_ctx->kv_state_input_name_map[pair.first + pair.second] = pair.first; @@ -1431,12 +1574,15 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, const std::shared ComputeParams c_params; std::tie(m_params, c_params) = GgmlOvDecoder::compute_llm_params(cgraph, is_static); - const auto * inp_pos = get_inp_pos_tensor(cgraph); const bool no_kv_cache = m_params.is_cacheless_attn; + // Cacheless encoders with learned position embeddings (e.g. BERT/XLM-R) have no ROPE, so no inp_pos; + // their token count comes from the input tokens. + const auto * inp_pos = no_kv_cache ? find_inp_pos_tensor(cgraph) : get_inp_pos_tensor(cgraph); + const int64_t inp_n_tokens = inp_pos ? get_inp_pos_n_tokens(cgraph, inp_pos) : get_inp_tok_n_tokens(cgraph); const auto is_prefill = no_kv_cache ? true : get_is_prefill(cgraph, inp_pos); const ov::AnyMap compile_config = no_kv_cache ? without_npuw(config) : config; if (m_params.n_heads_kv == -1) { - prefill_chunk_size = inp_pos->ne[0]; + prefill_chunk_size = inp_n_tokens; } graph_key key(cgraph); static const bool cache_enabled = !ggml_openvino_getenv_int("GGML_OPENVINO_DISABLE_CACHE"); @@ -1491,7 +1637,6 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, const std::shared if (old_m_params.kv_buffer_changed(m_params) || !ggml_decoder->is_bound_to(cgraph)) { ggml_decoder->update_io(cgraph); } - ggml_decoder->add_extra_inputs(); { std::lock_guard map_lock(r_ctx->ctx_mutex); infer_request = @@ -1557,8 +1702,7 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, const std::shared std::shared_ptr & model, ov::CompiledModel & compiled_model, std::shared_ptr & infer_request, int64_t & local_conversion_end_time, int64_t & local_compile_end_time) { - auto input_model = std::make_shared(decoder); - model = ov::frontend::ggml::FrontEnd::convert(input_model); + model = convert_to_ov_model(decoder); decoder->clear_model_weights(); local_conversion_end_time = ggml_time_us(); @@ -1629,7 +1773,7 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, const std::shared reset_single_slot_recurrent_cache(cgraph, m_params, c_params); if (is_prefill) { - auto inp_len = get_inp_pos_n_tokens(cgraph, inp_pos); + auto inp_len = inp_n_tokens; for (int chunk_index = 0; chunk_index * prefill_chunk_size < inp_len; chunk_index++) { for (size_t i = 0; i < ov_input_names_local.size(); i++) { const auto & param_name = ov_input_names_local[i]; @@ -1726,6 +1870,25 @@ enum ggml_status ov_graph_compute_static(ggml_cgraph * cgraph, const std::shared } } // namespace +// Nodes on the unselected branches of ggml_build_forward_select() stay in the graph but must not be +// computed. Keep them out of the OV model, or their inputs become parameters with fixed shapes. +static ggml_cgraph * get_compute_graph(ggml_cgraph * cgraph, ov_runtime_context & r_ctx) { + auto is_skipped = [](const ggml_tensor * node) { + return node->op != GGML_OP_NONE && !(node->flags & GGML_TENSOR_FLAG_COMPUTE); + }; + if (std::none_of(cgraph->nodes, cgraph->nodes + cgraph->n_nodes, is_skipped)) { + return cgraph; + } + auto & compute = r_ctx.compute_graphs[cgraph]; + compute.nodes.clear(); + std::copy_if(cgraph->nodes, cgraph->nodes + cgraph->n_nodes, std::back_inserter(compute.nodes), + [&](const ggml_tensor * node) { return !is_skipped(node); }); + compute.graph = *cgraph; + compute.graph.nodes = compute.nodes.data(); + compute.graph.n_nodes = (int) compute.nodes.size(); + return &compute.graph; +} + // Both execution paths use two cache levels: // 1. Reuse this backend's decoder/request via graph_key and compatibility checks. // 2. On a local miss, look up compiled_graph_key in the shared compilation cache, @@ -1744,6 +1907,7 @@ enum ggml_status ov_graph_compute(ggml_cgraph * cgraph, ggml_backend_t backend) GGML_ASSERT(ctx->runtime_context != nullptr); std::shared_ptr r_ctx = std::static_pointer_cast(ctx->runtime_context); std::lock_guard execution_lock(r_ctx->execution_mutex); + cgraph = get_compute_graph(cgraph, *r_ctx); return is_static ? ov_graph_compute_static(cgraph, r_ctx) : ov_graph_compute_dynamic(cgraph, r_ctx); } catch (const ov::Exception & e) { @@ -2007,7 +2171,7 @@ void print_output_tensor_info(const std::string & name, const ov::Tensor & tenso } } -const ggml_tensor * get_inp_pos_tensor(ggml_cgraph * cgraph) { +const ggml_tensor * find_inp_pos_tensor(ggml_cgraph * cgraph) { for (int i = 0; i < cgraph->n_nodes; ++i) { auto * op = cgraph->nodes[i]; for (int j = 0; j < GGML_MAX_SRC; ++j) { @@ -2020,8 +2184,27 @@ const ggml_tensor * get_inp_pos_tensor(ggml_cgraph * cgraph) { } } } - GGML_LOG_ERROR("get_inp_pos_tensor: inp_pos not found in cgraph"); - throw std::runtime_error("get_inp_pos_tensor: inp_pos not found in cgraph"); + return nullptr; +} + +const ggml_tensor * get_inp_pos_tensor(ggml_cgraph * cgraph) { + const auto * inp_pos = find_inp_pos_tensor(cgraph); + if (!inp_pos) { + GGML_LOG_ERROR("get_inp_pos_tensor: inp_pos not found in cgraph"); + throw std::runtime_error("get_inp_pos_tensor: inp_pos not found in cgraph"); + } + return inp_pos; +} + +int64_t get_inp_tok_n_tokens(ggml_cgraph * cgraph) { + for (int i = 0; i < cgraph->n_nodes; ++i) { + auto * op = cgraph->nodes[i]; + if (op->src[1] && GgmlOvDecoder::is_inp_tok(op->src[1], op)) { + return op->src[1]->ne[0]; + } + } + GGML_LOG_ERROR("get_inp_tok_n_tokens: input tokens not found in cgraph"); + throw std::runtime_error("get_inp_tok_n_tokens: input tokens not found in cgraph"); } int64_t get_inp_pos_n_tokens(ggml_cgraph * cgraph, const ggml_tensor * inp_pos) { diff --git a/ggml/src/ggml-openvino/utils.h b/ggml/src/ggml-openvino/utils.h index 491dbadc2e95..782f2e37891e 100644 --- a/ggml/src/ggml-openvino/utils.h +++ b/ggml/src/ggml-openvino/utils.h @@ -119,6 +119,12 @@ struct ov_runtime_context { std::unordered_map, graph_key_hash> ov_output_names_cache; size_t stateful_kv_size; std::map kv_state_input_name_map; + // compute-only copies of graphs that carry unselected ggml_build_forward_select() branches + struct compute_graph { + ggml_cgraph graph; + std::vector nodes; + }; + std::unordered_map compute_graphs; ov_runtime_context() : device("CPU"), stateful(false), stateful_kv_size(0) {} @@ -174,7 +180,9 @@ std::vector pad_input(const ggml_tensor * tensor, size_t padded_rows, size_t padded_rows, padded_cols, pad_value); } +const ggml_tensor * find_inp_pos_tensor(struct ggml_cgraph * cgraph); // nullptr when the graph has no ROPE const ggml_tensor * get_inp_pos_tensor(struct ggml_cgraph * cgraph); +int64_t get_inp_tok_n_tokens(struct ggml_cgraph * cgraph); int64_t get_inp_pos_n_tokens(struct ggml_cgraph * cgraph, const ggml_tensor * inp_pos);