diff --git a/runtime/include/ggmlc/types.h b/runtime/include/ggmlc/types.h index fb91bac..08246ee 100644 --- a/runtime/include/ggmlc/types.h +++ b/runtime/include/ggmlc/types.h @@ -120,6 +120,8 @@ struct SerializedModelGraph { std::unordered_map tensors; std::vector ops; std::vector data_buffer; + std::shared_ptr> data_storage; + uint64_t data_offset = 0; // Metadata key-values std::unordered_map metadata_str; @@ -141,9 +143,15 @@ struct SerializedModelGraph { metadata_int = other.metadata_int; metadata_float = other.metadata_float; metadata_str_arr = other.metadata_str_arr; - - if (!data_buffer.empty() && !other.data_buffer.empty()) { - ptrdiff_t diff = data_buffer.data() - other.data_buffer.data(); + data_storage = other.data_storage; + data_offset = other.data_offset; + + const uint8_t* src_base = other.data_storage ? other.data_storage->data() + : (!other.data_buffer.empty() ? other.data_buffer.data() : nullptr); + const uint8_t* dst_base = data_storage ? data_storage->data() + : (!data_buffer.empty() ? data_buffer.data() : nullptr); + if (src_base && dst_base && dst_base != src_base) { + ptrdiff_t diff = dst_base - src_base; for (auto& pair : tensors) { if (pair.second.data_ptr) { pair.second.data_ptr += diff; @@ -168,9 +176,15 @@ struct SerializedModelGraph { metadata_int = other.metadata_int; metadata_float = other.metadata_float; metadata_str_arr = other.metadata_str_arr; - - if (!data_buffer.empty() && !other.data_buffer.empty()) { - ptrdiff_t diff = data_buffer.data() - other.data_buffer.data(); + data_storage = other.data_storage; + data_offset = other.data_offset; + + const uint8_t* src_base = other.data_storage ? other.data_storage->data() + : (!other.data_buffer.empty() ? other.data_buffer.data() : nullptr); + const uint8_t* dst_base = data_storage ? data_storage->data() + : (!data_buffer.empty() ? data_buffer.data() : nullptr); + if (src_base && dst_base && dst_base != src_base) { + ptrdiff_t diff = dst_base - src_base; for (auto& pair : tensors) { if (pair.second.data_ptr) { pair.second.data_ptr += diff; diff --git a/runtime/src/executor.cpp b/runtime/src/executor.cpp index 14a4503..7f5ef78 100644 --- a/runtime/src/executor.cpp +++ b/runtime/src/executor.cpp @@ -392,19 +392,38 @@ void ModelExecutor::init_weights() { } } - weight_buffer_ = ggml_backend_alloc_ctx_tensors(ctx_w_, backend_); - if (!weight_buffer_) { - throw std::runtime_error("Failed to allocate weight buffer on backend (" + device_ + ")"); - } + if (device_ == "cpu" && model_graph_.data_storage != nullptr) { + for (const auto& pair : model_graph_.tensors) { + uint32_t tid = pair.first; + const auto& t = pair.second; + if ((t.storage == StorageClass::PARAMETER || t.storage == StorageClass::CONSTANT) && t.is_static() && t.data_ptr && t.data_size > 0) { + auto it = weight_tensors_.find(tid); + if (it != weight_tensors_.end()) { + struct ggml_tensor* g_t = it->second; + const size_t need = ggml_nbytes(g_t); + if (t.data_size < need) { + throw std::runtime_error("weight data smaller than tensor for: " + t.name); + } + g_t->data = const_cast(t.data_ptr); + } + } + } + weight_buffer_ = nullptr; + } else { + weight_buffer_ = ggml_backend_alloc_ctx_tensors(ctx_w_, backend_); + if (!weight_buffer_) { + throw std::runtime_error("Failed to allocate weight buffer on backend (" + device_ + ")"); + } - for (const auto& pair : model_graph_.tensors) { - uint32_t tid = pair.first; - const auto& t = pair.second; - if ((t.storage == StorageClass::PARAMETER || t.storage == StorageClass::CONSTANT) && t.is_static() && t.data_ptr && t.data_size > 0) { - auto it = weight_tensors_.find(tid); - if (it != weight_tensors_.end()) { - size_t sz = std::min(t.data_size, ggml_nbytes(it->second)); - ggml_backend_tensor_set(it->second, t.data_ptr, 0, sz); + for (const auto& pair : model_graph_.tensors) { + uint32_t tid = pair.first; + const auto& t = pair.second; + if ((t.storage == StorageClass::PARAMETER || t.storage == StorageClass::CONSTANT) && t.is_static() && t.data_ptr && t.data_size > 0) { + auto it = weight_tensors_.find(tid); + if (it != weight_tensors_.end()) { + size_t sz = std::min(t.data_size, ggml_nbytes(it->second)); + ggml_backend_tensor_set(it->second, t.data_ptr, 0, sz); + } } } } diff --git a/runtime/src/loader.cpp b/runtime/src/loader.cpp index d160718..a641448 100644 --- a/runtime/src/loader.cpp +++ b/runtime/src/loader.cpp @@ -262,27 +262,22 @@ SerializedModelGraph ModelLoader::load_from_file(const std::string& filepath) { std::streamsize size = file.tellg(); file.seekg(0, std::ios::beg); - std::vector buffer(size); - if (!file.read(reinterpret_cast(buffer.data()), size)) { + auto storage = std::make_shared>(size); + if (!file.read(reinterpret_cast(storage->data()), size)) { throw std::runtime_error("Failed to read GGUF file: " + filepath); } - auto graph = load_from_memory(buffer.data(), buffer.size()); - graph.data_buffer = std::move(buffer); + auto graph = load_from_memory(storage->data(), storage->size()); + graph.data_storage = std::move(storage); - // Re-bind data_ptr relative to graph.data_buffer - struct gguf_init_params params = { true, nullptr }; - struct gguf_context* ctx = gguf_init_from_buffer(graph.data_buffer.data(), graph.data_buffer.size(), params); - if (ctx) { - size_t data_offset = gguf_get_data_offset(ctx); - const uint8_t* base_data = graph.data_buffer.data() + data_offset; - for (auto& pair : graph.tensors) { - int64_t t_id = gguf_find_tensor(ctx, pair.second.name.c_str()); - if (t_id >= 0) { - pair.second.data_ptr = base_data + gguf_get_tensor_offset(ctx, t_id); - } + const uint8_t* base = graph.data_storage->data() + graph.data_offset; + for (auto& pair : graph.tensors) { + SerializedTensor& t = pair.second; + if (t.data_size > 0) { + t.data_ptr = base + t.data_offset; + } else { + t.data_ptr = nullptr; } - gguf_free(ctx); } return graph; @@ -380,7 +375,8 @@ SerializedModelGraph ModelLoader::load_from_memory(const uint8_t* data, size_t s } // Tensors - size_t data_offset = gguf_get_data_offset(ctx); + g.data_offset = gguf_get_data_offset(ctx); + size_t data_offset = g.data_offset; const uint8_t* tensor_data_base = data + data_offset; for (const auto& pair : root["tensors"].obj_val) {