diff --git a/ggml/src/ggml-hrx/ggml-hrx.cpp b/ggml/src/ggml-hrx/ggml-hrx.cpp index 1c9c119710aa..db431897c00d 100644 --- a/ggml/src/ggml-hrx/ggml-hrx.cpp +++ b/ggml/src/ggml-hrx/ggml-hrx.cpp @@ -781,6 +781,13 @@ static void backend_set_tensor_async(ggml_backend_t backend, auto * backend_context = static_cast(backend->context); ggml_backend_hrx_buffer_context * context = nullptr; size_t tensor_base = 0; + ggml_backend_buffer_t buffer = tensor->view_src != nullptr ? tensor->view_src->buffer : tensor->buffer; + if (buffer != nullptr && buffer->iface.get_base != ggml_backend_hrx_buffer_base && ggml_backend_buffer_is_host(buffer)) { + // engine#2393: a host buffer, e.g. the from-pointer weight buffer of device_buffer_from_host_ptr + backend_synchronize(backend); + ggml_backend_tensor_set(tensor, data, offset, size); + return; + } if (!ggml_backend_hrx_tensor_binding(tensor, &context, &tensor_base) || offset > ggml_nbytes(tensor) || size > ggml_nbytes(tensor) - offset) { GGML_LOG_ERROR("%s: invalid HRX tensor upload\n", __func__); @@ -811,6 +818,13 @@ static void backend_get_tensor_async(ggml_backend_t backend, auto * backend_context = static_cast(backend->context); ggml_backend_hrx_buffer_context * context = nullptr; size_t tensor_base = 0; + ggml_backend_buffer_t buffer = tensor->view_src != nullptr ? tensor->view_src->buffer : tensor->buffer; + if (buffer != nullptr && buffer->iface.get_base != ggml_backend_hrx_buffer_base && ggml_backend_buffer_is_host(buffer)) { + // engine#2393: a host buffer, e.g. the from-pointer weight buffer of device_buffer_from_host_ptr + backend_synchronize(backend); + ggml_backend_tensor_get(tensor, data, offset, size); + return; + } if (!ggml_backend_hrx_tensor_binding(tensor, &context, &tensor_base) || offset > ggml_nbytes(tensor) || size > ggml_nbytes(tensor) - offset) { GGML_LOG_ERROR("%s: invalid HRX tensor download\n", __func__); diff --git a/ggml/src/ggml-hrx/model-buffer-from-mmap.cpp b/ggml/src/ggml-hrx/model-buffer-from-mmap.cpp index e35d1fa77cb3..0e5b1b286b87 100644 --- a/ggml/src/ggml-hrx/model-buffer-from-mmap.cpp +++ b/ggml/src/ggml-hrx/model-buffer-from-mmap.cpp @@ -17,6 +17,7 @@ #include "backend-context.h" #include "ggml-backend-impl.h" +#include "ggml-impl.h" #include "runtime/host-memory.h" #include @@ -128,6 +129,11 @@ void model_buffer_from_mmap_init_type(ggml_backend_hrx_device_context * device) ggml_backend_buffer_t model_buffer_from_mmap(ggml_backend_hrx_device_context * device, void * ptr, size_t size) { // The CPU from-pointer buffer neither copies nor frees: the loader owns the mapping. Retag it so the scheduler places // these weights on this device; the binding resolver then reads them as host data and packs them once. + if (ptr == nullptr || size == 0 || reinterpret_cast(ptr) % TENSOR_ALIGNMENT != 0) { + GGML_LOG_ERROR("%s: refusing host pointer %p (size %zu): it must be non-null and %d-byte aligned\n", __func__, + ptr, size, TENSOR_ALIGNMENT); + return nullptr; + } ggml_backend_buffer_t buffer = ggml_backend_cpu_buffer_from_ptr(ptr, size); if (buffer != nullptr) { buffer->buft = &device->mapped_buft; diff --git a/tests/hrx-buffer-test.cpp b/tests/hrx-buffer-test.cpp index 83d5ac7a4bb9..f7574bf91f8a 100644 --- a/tests/hrx-buffer-test.cpp +++ b/tests/hrx-buffer-test.cpp @@ -7,6 +7,7 @@ #include "ggml-impl.h" #include "ggml.h" #include "hrx-interop-utils.h" +#include "model-buffer-from-mmap.h" #include "runtime/host-memory.h" #include "testing_suite.h" @@ -412,6 +413,48 @@ static void run_imported_buffer_checks(ggml_backend_t backend) { #endif } +// engine #856 / #2393: model_buffer_from_mmap, the CPU from-pointer buffer retagged as the mapped type +static void run_mapped_host_ptr_checks(ggml_backend_t backend) { + ggml_backend_dev_t device = ggml_backend_get_device(backend); + alignas(TENSOR_ALIGNMENT) std::array source = {}; + for (size_t i = 0; i < source.size(); ++i) { + source[i] = static_cast(i + 1); + } + REQUIRE(ggml::hrx::model_buffer_from_mmap(backend_context(backend)->device, source.data() + 1, + sizeof(source) - sizeof(uint32_t)) == nullptr); + REQUIRE(ggml::hrx::model_buffer_from_mmap(backend_context(backend)->device, nullptr, sizeof(source)) == nullptr); + + ggml_backend_buffer_t mapped = + ggml::hrx::model_buffer_from_mmap(backend_context(backend)->device, source.data(), sizeof(source)); + REQUIRE(mapped != nullptr); + REQUIRE(ggml_backend_buffer_get_base(mapped) == source.data()); + REQUIRE(ggml_backend_buffer_is_host(mapped)); + REQUIRE(ggml::hrx::buffer_is_file_mapped(mapped)); + REQUIRE(ggml_backend_dev_supports_buft(device, ggml_backend_buffer_get_type(mapped))); + + ggml_init_params params = {}; + params.mem_size = 4096; + params.no_alloc = true; + ggml_context * ggml = ggml_init(params); + REQUIRE(ggml != nullptr); + ggml_tensor * tensor = ggml_new_tensor_1d(ggml, GGML_TYPE_I32, 32); + REQUIRE(ggml_backend_tensor_alloc(mapped, tensor, source.data() + 32) == GGML_STATUS_SUCCESS); + + const uint32_t replacement = 999; + ggml_backend_tensor_set_async(backend, tensor, &replacement, sizeof(uint32_t), sizeof(replacement)); + ggml_backend_synchronize(backend); + REQUIRE(source[33] == replacement); + std::array readback = {}; + ggml_backend_tensor_get_async(backend, tensor, readback.data(), 0, sizeof(readback)); + ggml_backend_synchronize(backend); + REQUIRE(readback[0] == source[32]); + REQUIRE(readback[1] == replacement); + + ggml_backend_buffer_free(mapped); + REQUIRE(source[33] == replacement); + ggml_free(ggml); +} + static void run_host_transfer_checks(ggml_backend_hrx_context * context) { ggml::hrx::HostTransferManager transfers; ggml::hrx::HostStagingBuffer staging; @@ -729,6 +772,11 @@ static void register_hrx_buffer_cases(test_runner::Suite & suite) { run_imported_buffer_checks(backend); }); }); + suite.device_case("mapped_host_ptr", [] { + run_with_hrx_backend([](ggml_backend_t backend, ggml_backend_hrx_context *) { + run_mapped_host_ptr_checks(backend); + }); + }); suite.device_case("host_transfer", [] { run_with_hrx_backend([](ggml_backend_t, ggml_backend_hrx_context * context) { run_host_transfer_checks(context);