Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions ggml/src/ggml-hrx/ggml-hrx.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -781,6 +781,13 @@ static void backend_set_tensor_async(ggml_backend_t backend,
auto * backend_context = static_cast<ggml_backend_hrx_context *>(backend->context);
ggml_backend_hrx_buffer_context * context = nullptr;
size_t tensor_base = 0;
ggml_backend_buffer_t buffer = tensor->view_src != nullptr ? tensor->view_src->buffer : tensor->buffer;
if (buffer != nullptr && buffer->iface.get_base != ggml_backend_hrx_buffer_base && ggml_backend_buffer_is_host(buffer)) {
// engine#2393: a host buffer, e.g. the from-pointer weight buffer of device_buffer_from_host_ptr
backend_synchronize(backend);
ggml_backend_tensor_set(tensor, data, offset, size);
return;
}
if (!ggml_backend_hrx_tensor_binding(tensor, &context, &tensor_base) || offset > ggml_nbytes(tensor) ||
size > ggml_nbytes(tensor) - offset) {
GGML_LOG_ERROR("%s: invalid HRX tensor upload\n", __func__);
Expand Down Expand Up @@ -811,6 +818,13 @@ static void backend_get_tensor_async(ggml_backend_t backend,
auto * backend_context = static_cast<ggml_backend_hrx_context *>(backend->context);
ggml_backend_hrx_buffer_context * context = nullptr;
size_t tensor_base = 0;
ggml_backend_buffer_t buffer = tensor->view_src != nullptr ? tensor->view_src->buffer : tensor->buffer;
if (buffer != nullptr && buffer->iface.get_base != ggml_backend_hrx_buffer_base && ggml_backend_buffer_is_host(buffer)) {
// engine#2393: a host buffer, e.g. the from-pointer weight buffer of device_buffer_from_host_ptr
backend_synchronize(backend);
ggml_backend_tensor_get(tensor, data, offset, size);
return;
}
if (!ggml_backend_hrx_tensor_binding(tensor, &context, &tensor_base) || offset > ggml_nbytes(tensor) ||
size > ggml_nbytes(tensor) - offset) {
GGML_LOG_ERROR("%s: invalid HRX tensor download\n", __func__);
Expand Down
6 changes: 6 additions & 0 deletions ggml/src/ggml-hrx/model-buffer-from-mmap.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,7 @@

#include "backend-context.h"
#include "ggml-backend-impl.h"
#include "ggml-impl.h"
#include "runtime/host-memory.h"

#include <iterator>
Expand Down Expand Up @@ -128,6 +129,11 @@ void model_buffer_from_mmap_init_type(ggml_backend_hrx_device_context * device)
ggml_backend_buffer_t model_buffer_from_mmap(ggml_backend_hrx_device_context * device, void * ptr, size_t size) {
// The CPU from-pointer buffer neither copies nor frees: the loader owns the mapping. Retag it so the scheduler places
// these weights on this device; the binding resolver then reads them as host data and packs them once.
if (ptr == nullptr || size == 0 || reinterpret_cast<uintptr_t>(ptr) % TENSOR_ALIGNMENT != 0) {
GGML_LOG_ERROR("%s: refusing host pointer %p (size %zu): it must be non-null and %d-byte aligned\n", __func__,
ptr, size, TENSOR_ALIGNMENT);
return nullptr;
}
ggml_backend_buffer_t buffer = ggml_backend_cpu_buffer_from_ptr(ptr, size);
if (buffer != nullptr) {
buffer->buft = &device->mapped_buft;
Expand Down
48 changes: 48 additions & 0 deletions tests/hrx-buffer-test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@
#include "ggml-impl.h"
#include "ggml.h"
#include "hrx-interop-utils.h"
#include "model-buffer-from-mmap.h"
#include "runtime/host-memory.h"
#include "testing_suite.h"

Expand Down Expand Up @@ -412,6 +413,48 @@ static void run_imported_buffer_checks(ggml_backend_t backend) {
#endif
}

// engine #856 / #2393: model_buffer_from_mmap, the CPU from-pointer buffer retagged as the mapped type
static void run_mapped_host_ptr_checks(ggml_backend_t backend) {
ggml_backend_dev_t device = ggml_backend_get_device(backend);
alignas(TENSOR_ALIGNMENT) std::array<uint32_t, 128> source = {};
for (size_t i = 0; i < source.size(); ++i) {
source[i] = static_cast<uint32_t>(i + 1);
}
REQUIRE(ggml::hrx::model_buffer_from_mmap(backend_context(backend)->device, source.data() + 1,
sizeof(source) - sizeof(uint32_t)) == nullptr);
REQUIRE(ggml::hrx::model_buffer_from_mmap(backend_context(backend)->device, nullptr, sizeof(source)) == nullptr);

ggml_backend_buffer_t mapped =
ggml::hrx::model_buffer_from_mmap(backend_context(backend)->device, source.data(), sizeof(source));
REQUIRE(mapped != nullptr);
REQUIRE(ggml_backend_buffer_get_base(mapped) == source.data());
REQUIRE(ggml_backend_buffer_is_host(mapped));
REQUIRE(ggml::hrx::buffer_is_file_mapped(mapped));
REQUIRE(ggml_backend_dev_supports_buft(device, ggml_backend_buffer_get_type(mapped)));

ggml_init_params params = {};
params.mem_size = 4096;
params.no_alloc = true;
ggml_context * ggml = ggml_init(params);
REQUIRE(ggml != nullptr);
ggml_tensor * tensor = ggml_new_tensor_1d(ggml, GGML_TYPE_I32, 32);
REQUIRE(ggml_backend_tensor_alloc(mapped, tensor, source.data() + 32) == GGML_STATUS_SUCCESS);

const uint32_t replacement = 999;
ggml_backend_tensor_set_async(backend, tensor, &replacement, sizeof(uint32_t), sizeof(replacement));
ggml_backend_synchronize(backend);
REQUIRE(source[33] == replacement);
std::array<uint32_t, 32> readback = {};
ggml_backend_tensor_get_async(backend, tensor, readback.data(), 0, sizeof(readback));
ggml_backend_synchronize(backend);
REQUIRE(readback[0] == source[32]);
REQUIRE(readback[1] == replacement);

ggml_backend_buffer_free(mapped);
REQUIRE(source[33] == replacement);
ggml_free(ggml);
}

static void run_host_transfer_checks(ggml_backend_hrx_context * context) {
ggml::hrx::HostTransferManager transfers;
ggml::hrx::HostStagingBuffer staging;
Expand Down Expand Up @@ -729,6 +772,11 @@ static void register_hrx_buffer_cases(test_runner::Suite & suite) {
run_imported_buffer_checks(backend);
});
});
suite.device_case("mapped_host_ptr", [] {
run_with_hrx_backend([](ggml_backend_t backend, ggml_backend_hrx_context *) {
run_mapped_host_ptr_checks(backend);
});
});
suite.device_case("host_transfer", [] {
run_with_hrx_backend([](ggml_backend_t, ggml_backend_hrx_context * context) {
run_host_transfer_checks(context);
Expand Down