Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
14 changes: 14 additions & 0 deletions ggml/src/ggml-hrx/ggml-hrx.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1804,10 +1804,24 @@ static bool ggml_backend_hrx_buffer_export_dmabuf(ggml_backend_buffer_t buffer,
ggml::hrx::export_dmabuf(device_ptr, buffer->size, fd, offset);
}

// engine #2500: the loader's file mapping is a MAPPED weight (host memory the runtime packs once, model_buffer_from_mmap),
// not an import. Since #371, device_buffer_from_host_ptr is the import path, and it is the generic entry the loader used,
// so every mmap-backed weight became a non-host imported buffer: a weight view an HRX matcher declines (ZAYA's grouped
// CCA conv) then had no backend at all ("pre-allocated tensor ... in a buffer that cannot run the operation").
// The loader asks for this by name for HRX devices (src/llama-model.cpp); other callers keep the import semantics.
static ggml_backend_buffer_t ggml_backend_hrx_model_buffer_from_mmap(ggml_backend_dev_t device, void * ptr, size_t size,
size_t max_tensor_size) {
GGML_UNUSED(max_tensor_size);
return ggml::hrx::model_buffer_from_mmap(device_context(device), ptr, size);
}

static void * registry_proc(ggml_backend_reg_t registry, const char * name) {
if (std::strcmp(name, "ggml_backend_buffer_export_dmabuf") == 0) {
return (void *) ggml_backend_hrx_buffer_export_dmabuf;
}
if (std::strcmp(name, "ggml_backend_hrx_model_buffer_from_mmap") == 0) {
return (void *) ggml_backend_hrx_model_buffer_from_mmap;
}
GGML_UNUSED(registry);
GGML_UNUSED(name);
return nullptr;
Expand Down
3 changes: 3 additions & 0 deletions ggml/src/ggml-hrx/model-buffer-from-mmap.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -125,6 +125,9 @@ bool buffer_is_file_mapped(ggml_backend_buffer_t buffer) {

void model_buffer_from_mmap_init_type(ggml_backend_hrx_device_context * device) {
device->mapped_buft_context = { device, device->name + "_Mapped", true };
// engine #2500: the imported type had no context (name "", device null) since #371, so every imported weight buffer
// was a nameless, device-less type: "pre-allocated tensor (...) in a buffer () that cannot run the operation".
device->imported_buft_context = { device, device->name + "_Imported", false };
device->mapped_buft = {
// designated: the merged ggml interface also has alloc_buffer_n and get_alloc_size_n (left null)
{ .get_name = mapped_type_name, .alloc_buffer = mapped_type_alloc, .get_alignment = mapped_type_alignment,
Expand Down
11 changes: 10 additions & 1 deletion src/llama-model.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2007,6 +2007,13 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
GGML_ASSERT(!ml.no_alloc);
const size_t max_size = ggml_get_max_tensor_size(ctx);

// engine #2500: an HRX device wraps the loader's mapping as a mapped (host) weight buffer, not an import
using hrx_mmap_buffer_fn = ggml_backend_buffer_t (*)(ggml_backend_dev_t, void *, size_t, size_t);
hrx_mmap_buffer_fn hrx_mmap_buffer = nullptr;
if (ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(dev);
reg != nullptr && std::strcmp(ggml_backend_reg_name(reg), "HRX") == 0) {
hrx_mmap_buffer = (hrx_mmap_buffer_fn) ggml_backend_reg_get_proc_address(reg, "ggml_backend_hrx_model_buffer_from_mmap");
}
for (uint32_t idx = 0; idx < ml.files.size(); idx++) {
// map the mmap regions that contain the tensors of this context
// this is important for metal with apple silicon: a backend buffer pins everything it
Expand All @@ -2017,7 +2024,9 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) {
void * addr = ml.mappings.at(idx)->addr();

for (const auto & range : ml.get_mapping_ranges(idx, ctx)) {
ggml_backend_buffer_t buf = ggml_backend_dev_buffer_from_host_ptr(dev, (char *) addr + range.first, range.last - range.first, max_size);
ggml_backend_buffer_t buf = hrx_mmap_buffer != nullptr
? hrx_mmap_buffer(dev, (char *) addr + range.first, range.last - range.first, max_size)
: ggml_backend_dev_buffer_from_host_ptr(dev, (char *) addr + range.first, range.last - range.first, max_size);
if (buf == nullptr) {
throw std::runtime_error(format("unable to allocate %s buffer", ggml_backend_buft_name(buft)));
}
Expand Down
33 changes: 33 additions & 0 deletions tests/hrx-buffer-test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -13,6 +13,7 @@

#include <algorithm>
#include <array>
#include <string>
#include <cstdint>
#include <cstdio>
#include <cstdlib>
Expand Down Expand Up @@ -774,6 +775,33 @@ static void run_with_hrx_backend(F && f) {
ggml_backend_free(backend);
}

// engine #2500: the loader wraps its file mapping through the HRX proc "ggml_backend_hrx_model_buffer_from_mmap"
// (src/llama-model.cpp), which must give a MAPPED weight buffer (host, file-mapped), while the generic
// buffer_from_host_ptr stays an import (non-host, #2393). Since #371 the loader got the import, and a weight view an
// HRX matcher declines (ZAYA's grouped CCA conv) then had no backend at all. The imported type also needs a name.
static void run_loader_mmap_proc_checks(ggml_backend_t backend) {
ggml_backend_dev_t device = ggml_backend_get_device(backend);
ggml_backend_reg_t reg = ggml_backend_dev_backend_reg(device);
REQUIRE(reg != nullptr);
using fn_t = ggml_backend_buffer_t (*)(ggml_backend_dev_t, void *, size_t, size_t);
auto fn = (fn_t) ggml_backend_reg_get_proc_address(reg, "ggml_backend_hrx_model_buffer_from_mmap");
REQUIRE(fn != nullptr);
alignas(TENSOR_ALIGNMENT) std::array<uint32_t, 128> source = {};
ggml_backend_buffer_t mapped = fn(device, source.data(), sizeof(source), sizeof(source));
REQUIRE(mapped != nullptr);
REQUIRE(ggml_backend_buffer_is_host(mapped));
REQUIRE(ggml::hrx::buffer_is_file_mapped(mapped));
REQUIRE(ggml_backend_dev_supports_buft(device, ggml_backend_buffer_get_type(mapped)));
ggml_backend_buffer_t imported = ggml_backend_dev_buffer_from_host_ptr(device, source.data(), sizeof(source), sizeof(source));
REQUIRE(imported != nullptr);
REQUIRE(!ggml_backend_buffer_is_host(imported));
const std::string imported_name = ggml_backend_buft_name(ggml_backend_buffer_get_type(imported));
REQUIRE(!imported_name.empty());
REQUIRE(imported_name != ggml_backend_buft_name(ggml_backend_buffer_get_type(mapped)));
ggml_backend_buffer_free(imported);
ggml_backend_buffer_free(mapped);
}

static void register_hrx_buffer_cases(test_runner::Suite & suite) {
suite.device_case("backend_buffer", [] {
run_with_hrx_backend([](ggml_backend_t backend, ggml_backend_hrx_context *) {
Expand All @@ -795,6 +823,11 @@ static void register_hrx_buffer_cases(test_runner::Suite & suite) {
run_mapped_host_ptr_checks(backend);
});
});
suite.device_case("loader_mmap_proc", [] {
run_with_hrx_backend([](ggml_backend_t backend, ggml_backend_hrx_context *) {
run_loader_mmap_proc_checks(backend);
});
});
suite.device_case("host_transfer", [] {
run_with_hrx_backend([](ggml_backend_t, ggml_backend_hrx_context * context) {
run_host_transfer_checks(context);
Expand Down