Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions ggml/src/ggml-hrx/ggml-hrx.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -789,6 +789,13 @@ static void backend_set_tensor_async(ggml_backend_t backend,
auto * backend_context = static_cast<ggml_backend_hrx_context *>(backend->context);
ggml_backend_hrx_buffer_context * context = nullptr;
size_t tensor_base = 0;
ggml_backend_buffer_t buffer = tensor->view_src != nullptr ? tensor->view_src->buffer : tensor->buffer;
if (buffer != nullptr && buffer->iface.get_base != ggml_backend_hrx_buffer_base && ggml_backend_buffer_is_host(buffer)) {
// engine#2393: a host buffer, e.g. the mapped weight buffer of model_buffer_from_mmap
backend_synchronize(backend);
ggml_backend_tensor_set(tensor, data, offset, size);
return;
}
if (!ggml_backend_hrx_tensor_binding(tensor, &context, &tensor_base) || offset > ggml_nbytes(tensor) ||
size > ggml_nbytes(tensor) - offset) {
GGML_LOG_ERROR("%s: invalid HRX tensor upload\n", __func__);
Expand Down Expand Up @@ -819,6 +826,13 @@ static void backend_get_tensor_async(ggml_backend_t backend,
auto * backend_context = static_cast<ggml_backend_hrx_context *>(backend->context);
ggml_backend_hrx_buffer_context * context = nullptr;
size_t tensor_base = 0;
ggml_backend_buffer_t buffer = tensor->view_src != nullptr ? tensor->view_src->buffer : tensor->buffer;
if (buffer != nullptr && buffer->iface.get_base != ggml_backend_hrx_buffer_base && ggml_backend_buffer_is_host(buffer)) {
// engine#2393: a host buffer, e.g. the mapped weight buffer of model_buffer_from_mmap
backend_synchronize(backend);
ggml_backend_tensor_get(tensor, data, offset, size);
return;
}
if (!ggml_backend_hrx_tensor_binding(tensor, &context, &tensor_base) || offset > ggml_nbytes(tensor) ||
size > ggml_nbytes(tensor) - offset) {
GGML_LOG_ERROR("%s: invalid HRX tensor download\n", __func__);
Expand Down
6 changes: 6 additions & 0 deletions ggml/src/ggml-hrx/model-buffer-from-mmap.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

#include "backend-context.h"
#include "ggml-backend-impl.h"
#include "ggml-impl.h"
#include "runtime/host-memory.h"

#include <iterator>
Expand Down Expand Up @@ -147,6 +148,11 @@ ggml_backend_buffer_t model_buffer_from_mmap(ggml_backend_hrx_device_context * d
// packs once, and it keeps the retagged CPU from-pointer buffer: making it an imported handle (borrowed buffer, no
// pool charge) broke real model loading -- verified by a control run on an unmodified tree that generated tokens
// while this tree crashed in cli_server::wait_ready(). The import path is device_buffer_from_host_ptr's own branch.
if (ptr == nullptr || size == 0 || reinterpret_cast<uintptr_t>(ptr) % TENSOR_ALIGNMENT != 0) {
GGML_LOG_ERROR("%s: refusing host pointer %p (size %zu): it must be non-null and %d-byte aligned\n", __func__,
ptr, size, TENSOR_ALIGNMENT);
return nullptr;
}
ggml_backend_buffer_t buffer = ggml_backend_cpu_buffer_from_ptr(ptr, size);
if (buffer != nullptr) {
buffer->buft = &device->mapped_buft;
Expand Down