From 5cfa6a875e8cce499ef1ab350712da6d51d3c71d Mon Sep 17 00:00:00 2001 From: Inovello Date: Tue, 1 Sep 2026 20:20:20 -0700 Subject: [PATCH 1/2] llama : keep explicit host buffer overrides when using mmap Testing revealed that a host buffer type, specifically CUDA_Host was being replaced with CPU buffer type under mmap. Patch makes it so that when the buffer type comes from an explicit -ot rule, it skips the safety check it does (The loader's rule that essentially replaces a host buffer type with a plain CPU one when a model is memory mapped) and keeps the type you asked for which enables CPU resident experts to live in the pinned memory for op offload while the rest of the model stays memory mapped. Prefill of a 26k prompt went from 166 t/s to 379 t/s on 2 x RTX 3090 with 40 expert layers on the host. The change lives in the check in create_tensor() in src/llama-model-loader.cpp, which now skips the host buffer downgrade when the new buft_overridden flag is set, and in parse_tensor_buffer_overrides() in common/arg.cpp, which now accepts CUDA_Host as a target. Assisted-by: Claude Fable 5.1 --- common/arg.cpp | 5 +++++ src/llama-model-loader.cpp | 6 ++++-- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/common/arg.cpp b/common/arg.cpp index 79405b59e076..758bb1f4d430 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -259,6 +259,11 @@ static void parse_tensor_buffer_overrides(const std::string & value, std::vector if (buft) { buft_list[ggml_backend_buft_name(buft)] = buft; } + // host buffer types (e.g. CUDA_Host) so CPU-resident tensors can use pinned memory + auto * host_buft = ggml_backend_dev_host_buffer_type(dev); + if (host_buft) { + buft_list[ggml_backend_buft_name(host_buft)] = host_buft; + } } for (const auto & override : string_split(value, ',')) { diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index 7663797ba001..50554af7e596 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -1223,6 +1223,7 @@ struct ggml_tensor * llama_model_loader::create_tensor( } ggml_backend_buffer_type_t buft = nullptr; + bool buft_overridden = false; // check overrides if (tensor_buft_overrides) { @@ -1241,6 +1242,7 @@ struct ggml_tensor * llama_model_loader::create_tensor( } } else { buft = overrides->buft; + buft_overridden = true; } LLAMA_LOG_DEBUG("tensor %s (%zu MiB %s) buffer type overridden to %s\n", @@ -1259,9 +1261,9 @@ struct ggml_tensor * llama_model_loader::create_tensor( } } - // avoid using a host buffer when using mmap + // avoid using a host buffer when using mmap, unless an override asks for it: the tensor is then copied into pinned memory auto * buft_dev = ggml_backend_buft_get_device(buft); - if (use_mmap && buft_dev && buft == ggml_backend_dev_host_buffer_type(buft_dev)) { + if (use_mmap && !buft_overridden && buft_dev && buft == ggml_backend_dev_host_buffer_type(buft_dev)) { auto * cpu_dev = ggml_backend_dev_by_type(GGML_BACKEND_DEVICE_TYPE_CPU); if (!cpu_dev) { throw std::runtime_error("no CPU backend found"); From c59754bfc185f64b67b1cd59fb912e94be5ede23 Mon Sep 17 00:00:00 2001 From: Inovello Date: Thu, 3 Sep 2026 11:13:02 -0700 Subject: [PATCH 2/2] llama : loading issue caused by copying tensors through mmap resolved 1. Issue and cause: Load time was incredibly long and it was caused by load_all_data copying each expert tensor from the mmap into the pinned buffer. The mmap carries POSIX_MADV_RANDOM under --numa distribute, so the memcpy faulted one 4 KiB page at a time with no readahead. 2. What changed: When the destination buffer is the host memory, the tensor will be read from the file with read_raw instead of being copied from the mmap. llama-mmap.cpp and the normal mmap path are untouched. 3. Result: Cold load went from 512 s to 168 s on 2 x RTX 3090 with 101.75 GiB of experts on CUDA_Host; the copy phase went from 450 s to 104 s. This doesn't affect the prefill or the output, just the load times. Assisted-by: Claude Fable 5.1 --- src/llama-model-loader.cpp | 9 +++++++++ 1 file changed, 9 insertions(+) diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index 50554af7e596..bcfad76fe0c6 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -1644,6 +1644,15 @@ bool llama_model_loader::load_all_data( auto & mmap_used = mmaps_used[weight->idx]; mmap_used.first = std::min(mmap_used.first, weight->offs); mmap_used.second = std::max(mmap_used.second, weight->offs + n_size); + } else if (ggml_backend_buffer_is_host(cur->buffer)) { + // The destination is host memory (e.g. a pinned CUDA_Host buffer selected by -ot), so it can be + // filled directly from the file. Copying from the mapping instead would fault in every page one + // by one at queue depth 1 - and with --numa distribute the whole mapping carries POSIX_MADV_RANDOM, + // which disables readahead entirely. A plain read gets full readahead and is an order of magnitude + // faster. Non-host destinations keep the memcpy from the mapping below. + const auto & file = files.at(weight->idx); + file->seek(weight->offs, SEEK_SET); + file->read_raw(cur->data, n_size); } else { ggml_backend_tensor_set(cur, data, 0, n_size); }