From 6ad78b2ae793945641c9364f14473464289f8989 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 02:45:07 -0400 Subject: [PATCH] ggml-hrx: charge the UNBOUNDED host-weight path too, so the default mode cannot pierce the pool (engine #1823) #269 charged the host-weight *arena*, which is the mode an operator opts into with GGML_HRX_HOST_WEIGHT_LIMIT_MB. The default is that variable UNSET (0 = off = per-weight residency), and that branch allocated each materialised weight its own device buffer at host-memory.cpp:2009 with no charge, freed in ~HostWeightLease::Entry with nothing to release. So the earlier claim that all three allocators were charged held for the opt-in mode and not for the mode everybody runs - the wrong way round for a guardrail. The weight is now charged against the pool before the allocation (a pool that cannot hold it is a refusal naming the weight, a refusal this path has never had), returned if the allocation fails, and released once in ~Entry, which owns exactly one buffer. Observed on hardware with the default mode (no GGML_HRX_HOST_WEIGHT_LIMIT_MB) at GGML_HRX_POOL_LIMIT_MB=700: 'a materialized host weight of 121.7 MiB does not fit (674.9 MiB of 700.0 MiB already in use)' then 'refused: a 121.7 MiB host weight does not fit the pool'. Before this change that run materialised it uncharged. --- ggml/src/ggml-hrx/runtime/host-memory.cpp | 24 +++++++++++++++++++++++ 1 file changed, 24 insertions(+) diff --git a/ggml/src/ggml-hrx/runtime/host-memory.cpp b/ggml/src/ggml-hrx/runtime/host-memory.cpp index 7c6211b1d365..e0b1d70d60b1 100644 --- a/ggml/src/ggml-hrx/runtime/host-memory.cpp +++ b/ggml/src/ggml-hrx/runtime/host-memory.cpp @@ -1384,8 +1384,17 @@ struct HostWeightLease::Entry { if (owns_buffer && buffer != nullptr) { hrx_buffer_release(buffer); } + // engine#1823: the unbounded branch's per-weight buffer is device memory like any other. Before this it was + // allocated uncharged and freed with nothing to release, so the DEFAULT mode (`GGML_HRX_HOST_WEIGHT_LIMIT_MB=0` + // = per-weight residency) pierced the guardrail by however much the materialised weights held. Released here + // because Entry owns exactly one buffer. + if (charged_bytes_ > 0) { + ggml::hrx::onebit::pool_release(charged_bytes_); + charged_bytes_ = 0; + } } bool owns_buffer = false; + size_t charged_bytes_ = 0; // engine#1823 hrx_buffer_t buffer = nullptr; size_t offset = 0; @@ -2007,8 +2016,23 @@ HostWeightAcquireResult HostWeightCache::acquire(hrx_device_t device return result; } } else { + // engine#1823: this is the DEFAULT mode (no GGML_HRX_HOST_WEIGHT_LIMIT_MB), and until now it was the one + // path that allocated device memory without asking the pool. Charge the weight before the allocation, so a + // pool that cannot hold it is a refusal naming the weight rather than an over-commit that faults later. + if (pool_budget > 0 && materialized_length > 0) { + if (!ggml::hrx::onebit::pool_reserve(pool_budget, materialized_length, "a materialized host weight")) { + result.status.log("refused: a %.1f MiB host weight does not fit the pool", + materialized_length / (1024.0 * 1024.0)); + return result; + } + entry->charged_bytes_ = materialized_length; + } result.status = allocate_device_buffer(device, materialized_length, entry->buffer); if (!result.status.success()) { + if (entry->charged_bytes_ > 0) { + ggml::hrx::onebit::pool_release(entry->charged_bytes_); + entry->charged_bytes_ = 0; + } return result; } entry->owns_buffer = true;