Skip to content
This repository was archived by the owner on Oct 16, 2026. It is now read-only.

ggml-openvino: Fixes a latent cache-key collision in the OpenVINO CPU/GPU dynamic execution path #2

Open
yzhou51 wants to merge 1 commit into
base: dev_backend_openvino
Choose a base branch
from
Open

ggml-openvino: Fixes a latent cache-key collision in the OpenVINO CPU/GPU dynamic execution path #2

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 26 additions & 5 deletions ggml/src/ggml-openvino/utils.cpp
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,10 @@
#include <unordered_map>
#include <vector>

// Non-throwing lookup used where inp_pos may legitimately be absent (e.g. split-model
// fragments); defined alongside get_inp_pos_tensor() further down.
static const ggml_tensor * find_inp_pos_tensor(const ggml_cgraph * cgraph);

namespace {
// For a KV cache input, return an ov::Tensor sized to n_kv (== attention_size
// for that layer) instead of the fully-allocated ctx_per_seq. Pre-conditions:
Expand Down Expand Up @@ -855,7 +859,15 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, const std::share
// graph_key first (a few hundred us), and if the same key is already in decoder_cache
// we know the graph is not splitted (only not-splitted graphs get inserted there).
const int64_t cache_key_start_time = ggml_time_us();
graph_key key(cgraph);
// Prefill and decode graphs can share n_nodes/first/last node name, but
// compute_op_case() bakes shape-dependent structural decisions into the compiled model, so
// the two phases must never share a cache entry. Graphs without inp_pos (e.g. split-model
// fragments) keep the old untagged key, same as before this distinction existed.
bool is_prefill = false;
if (const auto * inp_pos = find_inp_pos_tensor(cgraph)) {
is_prefill = get_is_prefill(cgraph, inp_pos);
}
graph_key key(cgraph, /*include_inputs=*/false, is_prefill);
const int64_t cache_key_compute_time = ggml_time_us() - cache_key_start_time;
int64_t cache_lookup_time = 0;
bool key_seen = false;
Expand Down Expand Up @@ -948,6 +960,7 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, const std::share

if (cache_hit) {
std::map<std::string, std::shared_ptr<ov::Node>> model_weights;
ggml_decoder->m_is_prefill = is_prefill;
ggml_decoder->set_compute_params(c_params);
ggml_decoder->set_model_params(m_params);
if (old_m_params.kv_buffer_changed(m_params) || !ggml_decoder->is_bound_to(cgraph)) {
Expand Down Expand Up @@ -1067,7 +1080,7 @@ enum ggml_status ov_graph_compute_dynamic(ggml_cgraph * cgraph, const std::share
std::unique_lock<std::mutex> compile_lock(shared_cache->mutex);
auto weight_names = get_weight_names(cgraph);
ggml_decoder = std::make_shared<GgmlOvDecoder>(cgraph, m_params, c_params, weight_names,
is_static, r_ctx->stateful, model_is_splitted);
is_static, r_ctx->stateful, model_is_splitted, is_prefill);
const std::string model_cache_dir = ggml_openvino_model_cache_dir();
const std::string exact_key = cache_enabled ? compiled_graph_key(cgraph, *ggml_decoder, device) : "";
uint64_t model_fp = 0;
Expand Down Expand Up @@ -2067,7 +2080,7 @@ void print_output_tensor_info(const std::string & name, const ov::Tensor & tenso
}
}

const ggml_tensor * get_inp_pos_tensor(ggml_cgraph * cgraph) {
static const ggml_tensor * find_inp_pos_tensor(const ggml_cgraph * cgraph) {
for (int i = 0; i < cgraph->n_nodes; ++i) {
auto * op = cgraph->nodes[i];
for (int j = 0; j < GGML_MAX_SRC; ++j) {
Expand All @@ -2080,8 +2093,16 @@ const ggml_tensor * get_inp_pos_tensor(ggml_cgraph * cgraph) {
}
}
}
GGML_LOG_ERROR("get_inp_pos_tensor: inp_pos not found in cgraph");
throw std::runtime_error("get_inp_pos_tensor: inp_pos not found in cgraph");
return nullptr;
}

const ggml_tensor * get_inp_pos_tensor(ggml_cgraph * cgraph) {
const ggml_tensor * inp_pos = find_inp_pos_tensor(cgraph);
if (inp_pos == nullptr) {
GGML_LOG_ERROR("get_inp_pos_tensor: inp_pos not found in cgraph");
throw std::runtime_error("get_inp_pos_tensor: inp_pos not found in cgraph");
}
return inp_pos;
}

int64_t get_inp_pos_n_tokens(ggml_cgraph * cgraph, const ggml_tensor * inp_pos) {
Expand Down
12 changes: 10 additions & 2 deletions ggml/src/ggml-openvino/utils.h
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -23,12 +23,19 @@
struct graph_key {
int n_nodes;
int n_leaves;
// Prefill (n_tokens > 1) and decode (n_tokens == 1) graphs can otherwise share the same
// node count and end names, but compute_op_case() bakes shape-dependent structural
// decisions (e.g. GDN/view slicing) into the translated model at construction time, so a
// decode-shaped call must never reuse a model/infer_request compiled from a prefill-shaped
// one (or vice versa).
bool is_prefill = false;
std::string first_node_name;
std::string last_node_name;
// Each entry: "<node_idx>:<src_idx>:<op_type>"
std::vector<std::string> input_srcs;

graph_key(const ggml_cgraph * cgraph, bool include_inputs = false) : n_nodes(cgraph->n_nodes), n_leaves(cgraph->n_leafs) {
graph_key(const ggml_cgraph * cgraph, bool include_inputs = false, bool is_prefill = false) :
n_nodes(cgraph->n_nodes), n_leaves(cgraph->n_leafs), is_prefill(is_prefill) {
if (n_nodes > 0) {
first_node_name = cgraph->nodes[0]->name;
last_node_name = cgraph->nodes[n_nodes - 1]->name;
Expand Down Expand Up @@ -60,7 +67,7 @@ struct graph_key {
}

bool operator==(const graph_key & other) const {
return n_nodes == other.n_nodes && n_leaves == other.n_leaves &&
return n_nodes == other.n_nodes && n_leaves == other.n_leaves && is_prefill == other.is_prefill &&
first_node_name == other.first_node_name &&
last_node_name == other.last_node_name && input_srcs == other.input_srcs;
}
Expand All @@ -70,6 +77,7 @@ struct graph_key_hash {
size_t operator()(const graph_key & key) const {
size_t hash = std::hash<int>{}(key.n_nodes);
hash ^= std::hash<int>{}(key.n_leaves) + 0x9e3779b9 + (hash << 6) + (hash >> 2);
hash ^= std::hash<bool>{}(key.is_prefill) + 0x9e3779b9 + (hash << 6) + (hash >> 2);
if (key.n_nodes > 0) {
hash ^= std::hash<std::string>{}(key.first_node_name) + 0x9e3779b9 + (hash << 6) + (hash >> 2);
hash ^= std::hash<std::string>{}(key.last_node_name) + 0x9e3779b9 + (hash << 6) + (hash >> 2);
Expand Down