diff --git a/ggml/src/ggml-hrx/backend-context.h b/ggml/src/ggml-hrx/backend-context.h index ea92a06e07f5..29b60d02bd12 100644 --- a/ggml/src/ggml-hrx/backend-context.h +++ b/ggml/src/ggml-hrx/backend-context.h @@ -34,6 +34,9 @@ struct ggml_backend_hrx_buffer_context { uint64_t identity; uint64_t generation; bool direct_host_binding; + // engine #856: the storage is a caller-owned host region (the loader's mapping) that was imported, not allocated + // from the device, so it is not charged to the pool and must not release pool bytes on free. + bool imported = false; }; struct ggml_backend_hrx_device_context { diff --git a/ggml/src/ggml-hrx/ggml-hrx.cpp b/ggml/src/ggml-hrx/ggml-hrx.cpp index b32d7c341a50..1d5ca8d81377 100644 --- a/ggml/src/ggml-hrx/ggml-hrx.cpp +++ b/ggml/src/ggml-hrx/ggml-hrx.cpp @@ -157,6 +157,16 @@ static bool buffer_type_is_host(ggml_backend_buffer_type_t buft) { return static_cast(buft->context)->host_visible; } +// engine #856: the A/B switch for importing the loader's mapping (see device_buffer_from_host_ptr). +static bool import_host_pointer_enabled() { + static const bool disabled = [] { + const char * env = std::getenv("GGML_HRX_WEIGHT_HOST_PLACEMENT"); + return env != nullptr && env[0] == '0'; + }(); + return !disabled; +} + + static bool buffer_submit_and_wait(ggml_backend_hrx_device_context * device, hrx_status_t (*submit)(hrx_stream_t, void *), void * user_data) { @@ -209,7 +219,9 @@ static void buffer_free(ggml_backend_buffer_t buffer) { if (context->buffer != nullptr) { hrx_buffer_release(context->buffer); } - ggml::hrx::onebit::pool_release(buffer->size); + if (!context->imported) { + ggml::hrx::onebit::pool_release(buffer->size); + } delete context; } @@ -849,7 +861,7 @@ static void device_props(ggml_backend_dev_t device, ggml_backend_dev_props * pro device_memory(device, &props->memory_free, &props->memory_total); props->type = GGML_BACKEND_DEVICE_TYPE_GPU; props->device_id = nullptr; - props->caps = { true, true, false, false }; + props->caps = { true, true, import_host_pointer_enabled(), false }; } static ggml_backend_t device_init(ggml_backend_dev_t device, const char * parameters) { @@ -886,6 +898,58 @@ static ggml_backend_buffer_type_t device_host_buffer_type(ggml_backend_dev_t dev return &device_context(device)->host_buft; } +// engine #856: at -ngl 99 the loader places every offloaded weight in one device buffer, and this backend serves a dense +// MUL_MAT weight from the repacked resident copy it materialises on first use - so the loader's GGUF-layout copy is a +// second copy of the same bytes that nothing reads (5.9 GiB of GTT for a 2.38 GiB model, 2376 MiB of it that buffer). +// +// Importing the loader's mapping removes that copy without moving the weight off the device: the tensor stays in this +// backend's own buffer type, so the scheduler keeps the op here, while the pages are the mapping's own file pages and +// cost no device memory. Moving the weight to a host buffer type instead would hand the node to the CPU, because that +// path needs offload_op (ggml-backend.cpp:929-935) which this backend does not implement. GGML_HRX_WEIGHT_HOST_PLACEMENT=0 +// turns the capability off, which is the A/B control. +static ggml_backend_buffer_t device_buffer_from_host_ptr(ggml_backend_dev_t device, void * ptr, size_t size, + size_t max_tensor_size) { + GGML_UNUSED(max_tensor_size); + if (!import_host_pointer_enabled() || ptr == nullptr || size == 0) { + return nullptr; + } + auto * device_ctx = device_context(device); + hrx_buffer_params_t params = { + HRX_MEMORY_TYPE_HOST_LOCAL | HRX_MEMORY_TYPE_DEVICE_VISIBLE | HRX_MEMORY_TYPE_HOST_COHERENT, + HRX_MEMORY_ACCESS_ALL, + HRX_BUFFER_USAGE_DEFAULT, + 0, + }; + // the region the loader hands over starts at a tensor offset, and hsa_amd_memory_lock wants a page-aligned address + // (an unaligned one fails with a bare HSA_STATUS_ERROR - measured 2026-11-06, first import attempt). Import the + // page-aligned superset and keep the aligned address as the buffer base; a tensor's offset inside it then includes + // the small delta, which is exactly what the host-mapped lookup needs. + const uintptr_t page = 4096; + const uintptr_t unaligned = reinterpret_cast(ptr); + const uintptr_t aligned = unaligned & ~(page - 1); + const size_t delta = unaligned - aligned; + const size_t import_size = size + delta; + hrx_buffer_t allocation = nullptr; + if (!HRX_CHECK(hrx_allocator_import_buffer(hrx_device_allocator(device_ctx->device), params, + reinterpret_cast(aligned), import_size, &allocation))) { + GGML_LOG_WARN("%s: cannot import a %zu MiB host region, falling back to a device buffer (engine #856)\n", + __func__, size / 1024 / 1024); + return nullptr; + } + const uint64_t generation = g_allocation_generation.fetch_add(1); + auto * context = new (std::nothrow) ggml_backend_hrx_buffer_context{ + device_ctx, allocation, reinterpret_cast(aligned), generation, generation, false, true, + }; + if (context == nullptr) { + hrx_buffer_release(allocation); + return nullptr; + } + // register it as host-mapped storage so the in-place get/set paths and direct bindings find it, exactly as an + // allocated host buffer does (ggml-hrx.cpp buffer_alloc) + device_ctx->host_buffers.add(allocation, context->base, import_size); + return ggml_backend_buffer_init(device_buffer_type(device), buffer_i, context, import_size); +} + static bool eager_capability_declared(enum ggml_op op) { switch (op) { // The scheduler probes preallocated weight tensors as NONE operations when deciding whether their buffer type is @@ -1421,7 +1485,7 @@ static const ggml_backend_device_i device_i = { device_init, device_buffer_type, device_host_buffer_type, - nullptr, + device_buffer_from_host_ptr, device_supports_op, device_supports_buffer_type, nullptr, diff --git a/src/llama-mmap.cpp b/src/llama-mmap.cpp index 4d183cbc9c45..10f63fe577e9 100644 --- a/src/llama-mmap.cpp +++ b/src/llama-mmap.cpp @@ -465,9 +465,19 @@ struct llama_mmap::impl { #ifdef _POSIX_MAPPED_FILES std::vector> mapped_fragments; - impl(struct llama_file * file, size_t prefetch, bool numa, const llama_mmap::ranges & lazy_ranges) { + impl(struct llama_file * file, size_t prefetch, bool numa, const llama_mmap::ranges & lazy_ranges, + bool writable) { size = file->size(); int fd = file->file_id(); + // engine #856: a backend that can import a host region into its own buffers (HRX) needs the mapping to be + // writable, because a read-write lock of the pages is the only form the HSA API offers - importing a read-only + // mapping fails with a bare generic error (measured 2026-11-06). + // + // It must be MAP_SHARED, not MAP_PRIVATE: MAP_POPULATE and mlock fault pages in with FOLL_WRITE, and on a + // private writable mapping that breaks CoW, so the kernel copies the whole file into anonymous memory at load + // time (measured: MAP_PRIVATE|PROT_RW + populate = +1024 MiB of RssAnon for a 1 GiB file, versus +0 with the + // same mapping read page by page, and +0 for MAP_SHARED|PROT_READ). A shared writable mapping keeps every page + // in the page cache, still shared between processes, and HSA can lock it without copying anything. int flags = MAP_SHARED; if (numa) { prefetch = 0; } #ifdef __linux__ @@ -478,7 +488,7 @@ struct llama_mmap::impl { // MAP_POPULATE would fault in the lazy ranges too if (prefetch && lazy_ranges.empty()) { flags |= MAP_POPULATE; } #endif - addr = mmap(NULL, file->size(), PROT_READ, flags, fd, 0); + addr = mmap(NULL, file->size(), writable ? (PROT_READ | PROT_WRITE) : PROT_READ, flags, fd, 0); if (addr == MAP_FAILED) { throw std::runtime_error(format("mmap failed: %s", strerror(errno))); } @@ -663,7 +673,8 @@ struct llama_mmap::impl { }; llama_mmap::llama_mmap(struct llama_file * file, size_t prefetch, bool numa, - const ranges & lazy_ranges) : pimpl(std::make_unique(file, prefetch, numa, lazy_ranges)) {} + const ranges & lazy_ranges, bool writable) + : pimpl(std::make_unique(file, prefetch, numa, lazy_ranges, writable)) {} llama_mmap::~llama_mmap() = default; size_t llama_mmap::size() const { return pimpl->size; } diff --git a/src/llama-mmap.h b/src/llama-mmap.h index cc28c8a73fa5..bfacece16fd0 100644 --- a/src/llama-mmap.h +++ b/src/llama-mmap.h @@ -47,7 +47,7 @@ struct llama_mmap { llama_mmap(const llama_mmap &) = delete; llama_mmap(struct llama_file * file, size_t prefetch = (size_t) -1, bool numa = false, - const ranges & lazy_ranges = {}); + const ranges & lazy_ranges = {}, bool writable = false); ~llama_mmap(); size_t size() const; diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index dc19169a8381..834da6185036 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -537,6 +537,7 @@ llama_model_loader::llama_model_loader( bool check_tensors, bool no_alloc, bool load_mtp, + bool map_weights_writable, const llama_model_kv_override * param_overrides_p, const llama_model_tensor_buft_override * param_tensor_buft_overrides_p) : metadata(meta), set_tensor_data(set_tensor_data), set_tensor_data_ud(set_tensor_data_ud) { @@ -573,7 +574,9 @@ llama_model_loader::llama_model_loader( get_key(llm_kv(LLM_KV_GENERAL_ARCHITECTURE), arch_name, false); llm_kv = LLM_KV(llm_arch_from_string(arch_name)); - files.emplace_back(new llama_file(fname.c_str(), "rb", use_direct_io)); + // engine #856: a shared writable mapping needs the file opened for writing; llama.cpp only sets this flag when + // every file of the model passed its W_OK check + files.emplace_back(new llama_file(fname.c_str(), map_weights_writable ? "r+b" : "rb", use_direct_io)); contexts.emplace_back(ctx); // Save tensors data offset of the main file. @@ -641,7 +644,7 @@ llama_model_loader::llama_model_loader( } } - files.emplace_back(new llama_file(fname_split, "rb", use_direct_io)); + files.emplace_back(new llama_file(fname_split, map_weights_writable ? "r+b" : "rb", use_direct_io)); contexts.emplace_back(ctx); // Save tensors data offset info of the shard. @@ -826,6 +829,7 @@ llama_model_loader::llama_model_loader( this->check_tensors = check_tensors; this->no_alloc = no_alloc; this->load_mtp = load_mtp; + this->map_weights_writable = map_weights_writable; } std::string llama_model_loader::get_arch_name() const { @@ -1429,7 +1433,7 @@ void llama_model_loader::init_mappings(bool prefetch, llama_mlocks * mlock_mmaps const size_t prefetch_size = prefetch && use_mmap ? -1 : 0; std::unique_ptr mapping = std::make_unique(file.get(), prefetch_size, is_numa, - lazy.for_file(idx)); + lazy.for_file(idx), map_weights_writable); mmaps_used.emplace_back(mapping->size(), 0); if (mlock_mmaps) { std::unique_ptr mlock_mmap(new llama_mlock()); diff --git a/src/llama-model-loader.h b/src/llama-model-loader.h index f39bd154058a..78554fbc0c72 100644 --- a/src/llama-model-loader.h +++ b/src/llama-model-loader.h @@ -84,6 +84,9 @@ struct llama_model_loader { bool no_alloc; bool load_mtp; + // engine #856: map the weight files writable so a backend that imports host regions (HRX) can zero-copy them + bool map_weights_writable; + // handle TENSOR_READ_LAZY // use case: keep PLE / engrams embd tensors on disk, read them on demand struct lazy_read { @@ -182,6 +185,7 @@ struct llama_model_loader { bool check_tensors, bool no_alloc, bool load_mtp, + bool map_weights_writable, const llama_model_kv_override * param_overrides_p, const llama_model_tensor_buft_override * param_tensor_buft_overrides_p); diff --git a/src/llama-model.cpp b/src/llama-model.cpp index 667fdc12b7c7..65350b6a5b4c 100644 --- a/src/llama-model.cpp +++ b/src/llama-model.cpp @@ -1623,7 +1623,34 @@ bool llama_model_base::load_tensors(llama_model_loader & ml) { // a lazy context is mapped whatever the load mode, but the memory-fit pass maps nothing const bool is_lazy_mapped = ctx_key.lazy && !ml.no_alloc; - if ((ml.use_mmap || is_lazy_mapped) && use_mmap_buffer && buffer_from_host_ptr_supported && is_default_buft) { + // engine #856: a backend that imports this region into its own buffers pins every page of it. That only pays + // when the region is (close to) this ctx's own tensors; with CPU-placed tensors interleaved in the same file - + // a MoE under -ot exps=CPU - the range spans the whole file and pinning it costs far more than the device copy + // it would replace (measured: 18.6 GB model, >14 min stuck in load, against 31 s for the same model with the + // import disabled). So the import is taken only for a tight region, per file, and the whole ctx falls back to a + // normal device buffer otherwise. + bool host_ptr_region_tight = buffer_from_host_ptr_supported && is_default_buft; + if (host_ptr_region_tight && (ml.use_mmap || is_lazy_mapped) && use_mmap_buffer) { + size_t ctx_bytes = 0; + for (ggml_tensor * t = ggml_get_first_tensor(ctx); t != nullptr; t = ggml_get_next_tensor(ctx, t)) { + ctx_bytes += ggml_nbytes(t); + } + // a little slack for alignment and index tensors, but not a factor of the model + const size_t slack = ctx_bytes / 8 + ((size_t) 1 << 20); // NB: << binds looser than +, parentheses matter + for (uint32_t idx = 0; idx < ml.files.size() && host_ptr_region_tight; idx++) { + void * addr = nullptr; + size_t first, last; // NOLINT + ml.get_mapping_range(&first, &last, &addr, idx, ctx); + if (first >= last) { + continue; + } + if (last - first > ctx_bytes + slack) { + host_ptr_region_tight = false; + } + } + } + + if (host_ptr_region_tight && (ml.use_mmap || is_lazy_mapped) && use_mmap_buffer) { GGML_ASSERT(!ml.no_alloc); for (uint32_t idx = 0; idx < ml.files.size(); idx++) { // only the mmap region containing the tensors in the model is mapped to the backend buffer diff --git a/src/llama-quant.cpp b/src/llama-quant.cpp index fe2184551375..12c03bb4e110 100644 --- a/src/llama-quant.cpp +++ b/src/llama-quant.cpp @@ -934,7 +934,7 @@ static void llama_model_quantize_impl(const std::string & fname_inp, const std:: const llama_model_kv_override * kv_overrides = params->kv_overrides; std::vector splits = {}; llama_model_loader ml(/*metadata*/ nullptr, /*set_tensor_data*/ nullptr, /*set_tensor_data_ud*/ nullptr, - fname_inp, splits, /*file*/ nullptr, /*load_mode*/ load_mode, /*check_tensors*/ true, /*no_alloc*/ false, /*load_mtp*/ true, kv_overrides, nullptr); + fname_inp, splits, /*file*/ nullptr, /*load_mode*/ load_mode, /*check_tensors*/ true, /*no_alloc*/ false, /*load_mtp*/ true, /*map_weights_writable*/ false, kv_overrides, nullptr); ml.init_mappings(false); // no prefetching auto mparams = llama_model_default_params(); diff --git a/src/llama.cpp b/src/llama.cpp index ee7595ae22fc..0a9a4005cdd9 100644 --- a/src/llama.cpp +++ b/src/llama.cpp @@ -21,6 +21,7 @@ #include #include #include +#include #include #include #include @@ -304,8 +305,43 @@ static bool llama_prepare_model_devices(const llama_model_params & params, llama static std::pair llama_model_load(struct gguf_context * metadata, llama_model_set_tensor_data_t set_tensor_data, void * set_tensor_data_ud, const std::string & fname, std::vector & splits, FILE * file, llama_model_params & params) { try { + // engine #856: if any device can import a host region into its own buffers, map the weights writable so it can + // do that instead of copying them (a read-write lock is the only form the HSA API offers). + // params.devices == NULL means "all available devices", which is what llama-cli and the server pass, so the + // capability has to be read from the registry in that case (the first version only walked params.devices and + // therefore never saw the capability at all). + bool map_weights_writable = false; + if (params.devices != nullptr) { + for (ggml_backend_dev_t * dev = params.devices; *dev != nullptr && !map_weights_writable; ++dev) { + ggml_backend_dev_props props; + ggml_backend_dev_get_props(*dev, &props); + // the CPU backend also advertises this capability, and it does not want a writable weight mapping + map_weights_writable = props.type != GGML_BACKEND_DEVICE_TYPE_CPU && props.caps.buffer_from_host_ptr; + } + } else { + for (size_t i = 0, n = ggml_backend_dev_count(); i < n && !map_weights_writable; ++i) { + ggml_backend_dev_props props; + ggml_backend_dev_get_props(ggml_backend_dev_get(i), &props); + map_weights_writable = props.type != GGML_BACKEND_DEVICE_TYPE_CPU && props.caps.buffer_from_host_ptr; + } + } + if (map_weights_writable) { + // a shared writable mapping needs the file opened for writing, and an import over a read-only mapping fails; + // check every file of the model up front and fall back to the shared read-only mapping + auto can_write = [](const std::string & path) { return !path.empty() && access(path.c_str(), W_OK) == 0; }; + for (const auto & path : splits) { + if (!can_write(path)) { + map_weights_writable = false; + break; + } + } + if (map_weights_writable && !can_write(fname)) { + map_weights_writable = false; + } + } llama_model_loader ml(metadata, set_tensor_data, set_tensor_data_ud, fname, splits, file, params.load_mode, - params.check_tensors, params.no_alloc, params.load_mtp, params.kv_overrides, params.tensor_buft_overrides); + params.check_tensors, params.no_alloc, params.load_mtp, map_weights_writable, params.kv_overrides, + params.tensor_buft_overrides); ml.lazy.mode = params.lazy_mode;