Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 20 additions & 6 deletions runtime/include/ggmlc/types.h
Original file line number Diff line number Diff line change
Expand Up @@ -120,6 +120,8 @@ struct SerializedModelGraph {
std::unordered_map<uint32_t, SerializedTensor> tensors;
std::vector<SerializedOp> ops;
std::vector<uint8_t> data_buffer;
std::shared_ptr<std::vector<uint8_t>> data_storage;
uint64_t data_offset = 0;

// Metadata key-values
std::unordered_map<std::string, std::string> metadata_str;
Expand All @@ -141,9 +143,15 @@ struct SerializedModelGraph {
metadata_int = other.metadata_int;
metadata_float = other.metadata_float;
metadata_str_arr = other.metadata_str_arr;

if (!data_buffer.empty() && !other.data_buffer.empty()) {
ptrdiff_t diff = data_buffer.data() - other.data_buffer.data();
data_storage = other.data_storage;
data_offset = other.data_offset;

const uint8_t* src_base = other.data_storage ? other.data_storage->data()
: (!other.data_buffer.empty() ? other.data_buffer.data() : nullptr);
const uint8_t* dst_base = data_storage ? data_storage->data()
: (!data_buffer.empty() ? data_buffer.data() : nullptr);
if (src_base && dst_base && dst_base != src_base) {
ptrdiff_t diff = dst_base - src_base;
for (auto& pair : tensors) {
if (pair.second.data_ptr) {
pair.second.data_ptr += diff;
Expand All @@ -168,9 +176,15 @@ struct SerializedModelGraph {
metadata_int = other.metadata_int;
metadata_float = other.metadata_float;
metadata_str_arr = other.metadata_str_arr;

if (!data_buffer.empty() && !other.data_buffer.empty()) {
ptrdiff_t diff = data_buffer.data() - other.data_buffer.data();
data_storage = other.data_storage;
data_offset = other.data_offset;

const uint8_t* src_base = other.data_storage ? other.data_storage->data()
: (!other.data_buffer.empty() ? other.data_buffer.data() : nullptr);
const uint8_t* dst_base = data_storage ? data_storage->data()
: (!data_buffer.empty() ? data_buffer.data() : nullptr);
if (src_base && dst_base && dst_base != src_base) {
ptrdiff_t diff = dst_base - src_base;
for (auto& pair : tensors) {
if (pair.second.data_ptr) {
pair.second.data_ptr += diff;
Expand Down
43 changes: 31 additions & 12 deletions runtime/src/executor.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -392,19 +392,38 @@ void ModelExecutor::init_weights() {
}
}

weight_buffer_ = ggml_backend_alloc_ctx_tensors(ctx_w_, backend_);
if (!weight_buffer_) {
throw std::runtime_error("Failed to allocate weight buffer on backend (" + device_ + ")");
}
if (device_ == "cpu" && model_graph_.data_storage != nullptr) {
for (const auto& pair : model_graph_.tensors) {
uint32_t tid = pair.first;
const auto& t = pair.second;
if ((t.storage == StorageClass::PARAMETER || t.storage == StorageClass::CONSTANT) && t.is_static() && t.data_ptr && t.data_size > 0) {
auto it = weight_tensors_.find(tid);
if (it != weight_tensors_.end()) {
struct ggml_tensor* g_t = it->second;
const size_t need = ggml_nbytes(g_t);
if (t.data_size < need) {
throw std::runtime_error("weight data smaller than tensor for: " + t.name);
}
g_t->data = const_cast<uint8_t*>(t.data_ptr);
}
}
}
weight_buffer_ = nullptr;
} else {
weight_buffer_ = ggml_backend_alloc_ctx_tensors(ctx_w_, backend_);
if (!weight_buffer_) {
throw std::runtime_error("Failed to allocate weight buffer on backend (" + device_ + ")");
}

for (const auto& pair : model_graph_.tensors) {
uint32_t tid = pair.first;
const auto& t = pair.second;
if ((t.storage == StorageClass::PARAMETER || t.storage == StorageClass::CONSTANT) && t.is_static() && t.data_ptr && t.data_size > 0) {
auto it = weight_tensors_.find(tid);
if (it != weight_tensors_.end()) {
size_t sz = std::min<size_t>(t.data_size, ggml_nbytes(it->second));
ggml_backend_tensor_set(it->second, t.data_ptr, 0, sz);
for (const auto& pair : model_graph_.tensors) {
uint32_t tid = pair.first;
const auto& t = pair.second;
if ((t.storage == StorageClass::PARAMETER || t.storage == StorageClass::CONSTANT) && t.is_static() && t.data_ptr && t.data_size > 0) {
auto it = weight_tensors_.find(tid);
if (it != weight_tensors_.end()) {
size_t sz = std::min<size_t>(t.data_size, ggml_nbytes(it->second));
ggml_backend_tensor_set(it->second, t.data_ptr, 0, sz);
}
}
}
}
Expand Down
30 changes: 13 additions & 17 deletions runtime/src/loader.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -262,27 +262,22 @@ SerializedModelGraph ModelLoader::load_from_file(const std::string& filepath) {
std::streamsize size = file.tellg();
file.seekg(0, std::ios::beg);

std::vector<uint8_t> buffer(size);
if (!file.read(reinterpret_cast<char*>(buffer.data()), size)) {
auto storage = std::make_shared<std::vector<uint8_t>>(size);
if (!file.read(reinterpret_cast<char*>(storage->data()), size)) {
throw std::runtime_error("Failed to read GGUF file: " + filepath);
}

auto graph = load_from_memory(buffer.data(), buffer.size());
graph.data_buffer = std::move(buffer);
auto graph = load_from_memory(storage->data(), storage->size());
graph.data_storage = std::move(storage);

// Re-bind data_ptr relative to graph.data_buffer
struct gguf_init_params params = { true, nullptr };
struct gguf_context* ctx = gguf_init_from_buffer(graph.data_buffer.data(), graph.data_buffer.size(), params);
if (ctx) {
size_t data_offset = gguf_get_data_offset(ctx);
const uint8_t* base_data = graph.data_buffer.data() + data_offset;
for (auto& pair : graph.tensors) {
int64_t t_id = gguf_find_tensor(ctx, pair.second.name.c_str());
if (t_id >= 0) {
pair.second.data_ptr = base_data + gguf_get_tensor_offset(ctx, t_id);
}
const uint8_t* base = graph.data_storage->data() + graph.data_offset;
for (auto& pair : graph.tensors) {
SerializedTensor& t = pair.second;
if (t.data_size > 0) {
t.data_ptr = base + t.data_offset;
} else {
t.data_ptr = nullptr;
}
gguf_free(ctx);
}

return graph;
Expand Down Expand Up @@ -380,7 +375,8 @@ SerializedModelGraph ModelLoader::load_from_memory(const uint8_t* data, size_t s
}

// Tensors
size_t data_offset = gguf_get_data_offset(ctx);
g.data_offset = gguf_get_data_offset(ctx);
size_t data_offset = g.data_offset;
const uint8_t* tensor_data_base = data + data_offset;

for (const auto& pair : root["tensors"].obj_val) {
Expand Down
Loading