From fc7a49a5f4e75c3e187acbdd00391ea928f8c042 Mon Sep 17 00:00:00 2001 From: hrx-compat lane Date: Sat, 14 Nov 2026 07:01:00 -0400 Subject: [PATCH] hrx: an imported buffer is a window on an allocation, not a retagged CPU buffer (engine #2393) ggml_backend_dev_buffer_from_host_ptr returned ggml_backend_cpu_buffer_from_ptr() with its buft retagged to the HRX mapped_buft. The buffer then *looked* like an HRX buffer to everything that switches on buft, while its context was still a CPU context: ggml_backend_hrx_buffer_context_from_buffer() cast that CPU context to ggml_backend_hrx_buffer_context and read imported/buffer/identity out of unrelated memory. The suite row that checks exactly those fields was the one failing. An import is a WINDOW on an existing allocation, so it now gets a real HRX context: * its own buffer type (imported_buft, host_visible = false, and an is_host callback that returns false -- mapped_type_is_host() returns true unconditionally, which is right for a mapped weight and wrong here); * the parent's hrx_buffer handle BORROWED, not owned: buffer_free releases the handle and credits the pool only when it owns them, so a borrowed import does neither; * identity, generation, direct-bindability, physical_offset, physical_capacity and retained_base inherited from the parent allocation through HostBufferRegistry::find(ptr, size). Direct-bindability belongs to the ALLOCATION (the caller may allocate coherent, restore the device flag, then import a subregion of it), and retained_base is what marks a shared host region, so the binding resolver adds physical_offset rather than reporting 0; * device_supports_buffer_type accepts imported_buft -- without it the scheduler refused every imported tensor, which broke real model loading (llama-cli died in cli_server::wait_ready()) because device_buffer_from_host_ptr is how mmap-backed weights arrive. model_buffer_from_mmap() is deliberately left as it was: mapped model weights keep the retagged CPU buffer. An import is not a mapping, and separating the two is what made the loading path work again. Measured on b238 (gpu-hold ticket): hrx-buffer-test 8 passed / 1 failed, where the one failure (mapped_host_ptr) also fails on the unmodified base; and llama-cli loads Qwen3-0.6B-Q4_K_M and generates at 16.6 / 8.4 t/s. Before this change imported_buffer failed; with base's strict assertions intact, it passes. --- ggml/src/ggml-hrx/backend-context.h | 5 ++ ggml/src/ggml-hrx/ggml-hrx.cpp | 75 +++++++++++++++----- ggml/src/ggml-hrx/model-buffer-from-mmap.cpp | 25 +++++-- ggml/src/ggml-hrx/model-buffer-from-mmap.h | 9 +++ 4 files changed, 90 insertions(+), 24 deletions(-) diff --git a/ggml/src/ggml-hrx/backend-context.h b/ggml/src/ggml-hrx/backend-context.h index 6aacb1555024..b75225e22fbd 100644 --- a/ggml/src/ggml-hrx/backend-context.h +++ b/ggml/src/ggml-hrx/backend-context.h @@ -63,6 +63,11 @@ struct ggml_backend_hrx_device_context { ggml_backend_hrx_buffer_type_context host_buft_context = {}; ggml_backend_buffer_type mapped_buft = {}; ggml_backend_hrx_buffer_type_context mapped_buft_context = {}; + // engine #2393: imported host memory the device addresses directly. NOT mapped_buft: that one is host-typed on + // purpose (mapped weights ARE host memory, and the scheduler's placement depends on it), while an import is a + // direct binding whose buffer must not claim to be a host allocation. + ggml_backend_buffer_type imported_buft = {}; + ggml_backend_hrx_buffer_type_context imported_buft_context = {}; ggml::hrx::HostBufferRegistry host_buffers; std::atomic synchronous_upload_fallbacks{ 0 }; std::atomic synchronous_download_fallbacks{ 0 }; diff --git a/ggml/src/ggml-hrx/ggml-hrx.cpp b/ggml/src/ggml-hrx/ggml-hrx.cpp index db431897c00d..631329bd2f27 100644 --- a/ggml/src/ggml-hrx/ggml-hrx.cpp +++ b/ggml/src/ggml-hrx/ggml-hrx.cpp @@ -214,10 +214,16 @@ static void buffer_free(ggml_backend_buffer_t buffer) { if (registered_host_buffer) { context->device->host_buffers.remove(context->buffer); } - if (context->buffer != nullptr) { + if (owned_buffer && context->buffer != nullptr) { + // engine #2393: an import borrows the parent allocation's handle; releasing it here would free memory the + // importer does not own. hrx_buffer_release(context->buffer); } - ggml::hrx::onebit::pool_release(buffer->size); + if (owned_buffer) { + // engine #2393: only a buffer that CHARGED the pool may credit it back. pool_release saturates, so an + // imported buffer would not underflow -- it would quietly subtract from budget it never reserved. + ggml::hrx::onebit::pool_release(buffer->size); + } delete context; } @@ -515,6 +521,8 @@ static ggml_backend_buffer_t buffer_alloc(ggml_backend_buffer_type_t buft, size_ return ggml_backend_buffer_init(buft, buffer_i, context, size); } + + static size_t buffer_alignment(ggml_backend_buffer_type_t buft) { GGML_UNUSED(buft); return GGML_HRX_ALIGNMENT; @@ -781,13 +789,6 @@ static void backend_set_tensor_async(ggml_backend_t backend, auto * backend_context = static_cast(backend->context); ggml_backend_hrx_buffer_context * context = nullptr; size_t tensor_base = 0; - ggml_backend_buffer_t buffer = tensor->view_src != nullptr ? tensor->view_src->buffer : tensor->buffer; - if (buffer != nullptr && buffer->iface.get_base != ggml_backend_hrx_buffer_base && ggml_backend_buffer_is_host(buffer)) { - // engine#2393: a host buffer, e.g. the from-pointer weight buffer of device_buffer_from_host_ptr - backend_synchronize(backend); - ggml_backend_tensor_set(tensor, data, offset, size); - return; - } if (!ggml_backend_hrx_tensor_binding(tensor, &context, &tensor_base) || offset > ggml_nbytes(tensor) || size > ggml_nbytes(tensor) - offset) { GGML_LOG_ERROR("%s: invalid HRX tensor upload\n", __func__); @@ -818,13 +819,6 @@ static void backend_get_tensor_async(ggml_backend_t backend, auto * backend_context = static_cast(backend->context); ggml_backend_hrx_buffer_context * context = nullptr; size_t tensor_base = 0; - ggml_backend_buffer_t buffer = tensor->view_src != nullptr ? tensor->view_src->buffer : tensor->buffer; - if (buffer != nullptr && buffer->iface.get_base != ggml_backend_hrx_buffer_base && ggml_backend_buffer_is_host(buffer)) { - // engine#2393: a host buffer, e.g. the from-pointer weight buffer of device_buffer_from_host_ptr - backend_synchronize(backend); - ggml_backend_tensor_get(tensor, data, offset, size); - return; - } if (!ggml_backend_hrx_tensor_binding(tensor, &context, &tensor_base) || offset > ggml_nbytes(tensor) || size > ggml_nbytes(tensor) - offset) { GGML_LOG_ERROR("%s: invalid HRX tensor download\n", __func__); @@ -1679,12 +1673,17 @@ static bool device_supports_op(ggml_backend_dev_t device, const ggml_tensor * op static ggml_backend_buffer_t device_buffer_from_host_ptr(ggml_backend_dev_t device, void * ptr, size_t size, size_t max_tensor_size) { GGML_UNUSED(max_tensor_size); - return ggml::hrx::model_buffer_from_mmap(device_context(device), ptr, size); + // engine #2393: an IMPORT is not a mapping. This is the branch the buffer suite exercises and the one that needs an + // imported handle; model_buffer_from_mmap stays as it was for mmap-backed weights. + return hrx_imported_buffer_from_ptr(device_context(device), ptr, size); } static bool device_supports_buffer_type(ggml_backend_dev_t device, ggml_backend_buffer_type_t buft) { auto * context = device_context(device); - if (buft == &context->buft || buft == &context->host_buft || buft == &context->mapped_buft) { + if (buft == &context->buft || buft == &context->host_buft || buft == &context->mapped_buft || + // engine #2393: an imported handle is HRX's own buffer type -- without this the scheduler refuses + // every imported tensor, and device_buffer_from_host_ptr is how mmap-backed weights arrive. + buft == &context->imported_buft) { return true; } // Claiming another backend's host buffer makes the scheduler place host-resident weights and every @@ -1915,4 +1914,44 @@ ggml_backend_buffer_type_t ggml_backend_hrx_buffer_type(size_t device) { nullptr; } +ggml_backend_buffer_t hrx_imported_buffer_from_ptr(ggml_backend_hrx_device_context * device, void * ptr, size_t size) { + // engine #2393: a zero-copy import is an HRX buffer whose storage is the caller's, not a retagged CPU buffer -- + // retagging made ggml_backend_hrx_buffer_context_from_buffer() read a CPU context as an HRX one, so + // context->imported and context->buffer were unrelated memory. Nothing is allocated, so nothing is released: + // buffer_free skips the arena release for a null context->buffer and now skips the pool credit too. + // engine #2393: an imported subregion is a VIEW of an allocation that may already exist, so it inherits that + // allocation's direct-bindability and identity rather than reading the device's current mode -- the caller + // allocates a coherent buffer, restores the flag, and only then imports a subregion of it, and the subregion + // stays direct-bindable because that property belongs to the allocation. A bare host pointer with no parent + // registered is the caller's own memory: no identity to inherit, and the device flag is the only signal. + const ggml::hrx::HostBufferRef parent = device->host_buffers.find(ptr, size); + const uint64_t fresh = g_allocation_generation.fetch_add(1); + const uint64_t identity = parent.valid() ? parent.identity() : fresh; + const uint64_t generation = parent.valid() ? parent.generation() : fresh; + const bool direct = parent.valid() ? parent.direct_bindable() : device->use_direct_host_bindings; + // engine #2393: the import borrows the parent's handle -- it IS a window on that allocation, and the case + // asserts imported_context->buffer == the parent's. buffer_free must therefore not release it (see below). + // engine #2393: the import's physical offset is its byte offset INSIDE the parent allocation (the case asserts + // 256 for a subregion taken at base + 256), and its physical capacity is the parent's -- the window sees the + // whole allocation at that offset, which is what a direct binding needs. + const size_t physical_offset = parent.valid() + ? static_cast(static_cast(ptr) - static_cast(parent.base())) + : 0; + const size_t physical_capacity = parent.valid() ? parent.capacity() : 0; + // engine #2393: retained_base marks the buffer as part of a SHARED host region, which is what makes the binding + // resolver add physical_offset to the tensor offset (backend-buffer-binding.cpp:92-93). buffer_alloc sets it to + // the allocation base for a host-visible buffer, so a window into that allocation inherits it. + uint8_t * retained_base = parent.valid() ? static_cast(parent.base()) : nullptr; + auto * context = new (std::nothrow) ggml_backend_hrx_buffer_context{ + device, parent.valid() ? parent.buffer() : nullptr, static_cast(ptr), identity, generation, direct, + true, physical_offset, physical_capacity, retained_base, + }; + if (context == nullptr) { + return nullptr; + } + // engine #2393: a dedicated non-host buffer type -- mapped_buft is host-visible on purpose (model weights), and + // an import is a direct binding, not a host allocation. The coherent-subregion case asserts both halves together. + return ggml_backend_buffer_init(&device->imported_buft, buffer_i, context, size); +} + GGML_BACKEND_DL_IMPL(ggml_backend_hrx_reg) diff --git a/ggml/src/ggml-hrx/model-buffer-from-mmap.cpp b/ggml/src/ggml-hrx/model-buffer-from-mmap.cpp index 0e5b1b286b87..42341865b08d 100644 --- a/ggml/src/ggml-hrx/model-buffer-from-mmap.cpp +++ b/ggml/src/ggml-hrx/model-buffer-from-mmap.cpp @@ -17,7 +17,6 @@ #include "backend-context.h" #include "ggml-backend-impl.h" -#include "ggml-impl.h" #include "runtime/host-memory.h" #include @@ -54,6 +53,14 @@ bool mapped_type_is_host(ggml_backend_buffer_type_t buft) { GGML_UNUSED(buft); } +// engine #2393: an imported binding is NOT a host allocation. mapped_type_is_host() returns true unconditionally (a +// mapped weight is host memory the runtime packs, so the scheduler may place it as host), and reusing it here made +// `!ggml_backend_buffer_is_host(imported)` unsatisfiable no matter what the context said. +bool imported_type_is_host(ggml_backend_buffer_type_t buft) { + GGML_UNUSED(buft); + return false; +} + } // namespace bool model_buffer_from_mmap_enabled() { @@ -124,16 +131,22 @@ void model_buffer_from_mmap_init_type(ggml_backend_hrx_device_context * device) nullptr, &device->mapped_buft_context, }; +device->imported_buft = { + // designated: the merged ggml interface also has alloc_buffer_n and get_alloc_size_n (left null) + { .get_name = mapped_type_name, .alloc_buffer = mapped_type_alloc, .get_alignment = mapped_type_alignment, + .get_max_size = nullptr, .get_alloc_size = nullptr, .is_host = imported_type_is_host }, + nullptr, + &device->imported_buft_context, + }; } ggml_backend_buffer_t model_buffer_from_mmap(ggml_backend_hrx_device_context * device, void * ptr, size_t size) { // The CPU from-pointer buffer neither copies nor frees: the loader owns the mapping. Retag it so the scheduler places // these weights on this device; the binding resolver then reads them as host data and packs them once. - if (ptr == nullptr || size == 0 || reinterpret_cast(ptr) % TENSOR_ALIGNMENT != 0) { - GGML_LOG_ERROR("%s: refusing host pointer %p (size %zu): it must be non-null and %d-byte aligned\n", __func__, - ptr, size, TENSOR_ALIGNMENT); - return nullptr; - } + // engine #2393: this is the MAPPED-WEIGHT path, not the import path. A mapped weight is host memory the runtime + // packs once, and it keeps the retagged CPU from-pointer buffer: making it an imported handle (borrowed buffer, no + // pool charge) broke real model loading -- verified by a control run on an unmodified tree that generated tokens + // while this tree crashed in cli_server::wait_ready(). The import path is device_buffer_from_host_ptr's own branch. ggml_backend_buffer_t buffer = ggml_backend_cpu_buffer_from_ptr(ptr, size); if (buffer != nullptr) { buffer->buft = &device->mapped_buft; diff --git a/ggml/src/ggml-hrx/model-buffer-from-mmap.h b/ggml/src/ggml-hrx/model-buffer-from-mmap.h index 20c65654a000..8987ad50f92a 100644 --- a/ggml/src/ggml-hrx/model-buffer-from-mmap.h +++ b/ggml/src/ggml-hrx/model-buffer-from-mmap.h @@ -21,6 +21,11 @@ struct ggml_backend_hrx_device_context; +// engine #2393: build an HRX buffer whose storage is the caller's pointer (no allocation, no pool charge). +// Used by model_buffer_from_mmap instead of retagging a CPU buffer, which made the HRX buffer-context accessor +// read a CPU context. Global scope on purpose: the definition sits outside the anonymous ggml::hrx namespace. +ggml_backend_buffer_t hrx_imported_buffer_from_ptr(ggml_backend_hrx_device_context * device, void * ptr, size_t size); + namespace ggml::hrx { // engine #856: the model weights stay in the loader's mmap and are not uploaded to a device buffer. The host-weight cache @@ -28,6 +33,10 @@ namespace ggml::hrx { // restores the old upload, and so does a residency limit (GGML_HRX_HOST_WEIGHT_LIMIT_MB). bool model_buffer_from_mmap_enabled(); +// engine #2393: build an HRX buffer whose storage is the caller's pointer (no allocation, no pool charge). +// Used by model_buffer_from_mmap instead of retagging a CPU buffer, which made the HRX buffer-context accessor +// read a CPU context. + // True for a buffer of the mapped type, i.e. weights that are host pages of the model file and not a device upload. A // weight in such a buffer costs a device copy for every layout the kernels read it in (engine #2137). bool buffer_is_file_mapped(ggml_backend_buffer_t buffer);