diff --git a/app/q4nx_cache.cpp b/app/q4nx_cache.cpp index 361974546..3bcb727c6 100644 --- a/app/q4nx_cache.cpp +++ b/app/q4nx_cache.cpp @@ -16,8 +16,10 @@ #include "q4nx_cache.h" #include +#include #include #include +#include #include #include @@ -83,3 +85,22 @@ Q4nxCacheSweep q4nx_cache_evict(const std::string& root, const std::string& keep } return r; } + +int64_t q4nx_cache_bound_bytes(const char* text, int64_t default_gib) { + const int64_t max_gib = INT64_MAX >> 30; // the largest GiB count whose byte count still fits + long long gib = default_gib; + if (text && *text) { + char* end = nullptr; + errno = 0; + const long long v = std::strtoll(text, &end, 10); + if (end != text) gib = v; // a number (ERANGE saturates, which the clamp below handles) + } + if (gib <= 0) return 0; + if (gib > max_gib) gib = max_gib; + return int64_t(gib) << 30; +} + +void q4nx_cache_touch(const std::string& entry) { + std::error_code ec; + if (fs::is_directory(entry, ec)) fs::last_write_time(entry, fs::file_time_type::clock::now(), ec); +} diff --git a/app/q4nx_cache.h b/app/q4nx_cache.h index 53783bac3..f5d987511 100644 --- a/app/q4nx_cache.h +++ b/app/q4nx_cache.h @@ -44,3 +44,16 @@ Q4nxCacheSweep q4nx_cache_evict(const std::string& root, const std::string& keep // Directory bytes under `path` (0 when it does not exist). int64_t q4nx_dir_bytes(const std::string& path); + +// The byte bound for ONEBIT_Q4NX_CACHE_MAX_GIB's value (`text` may be null). Unset, empty or not a number gives the default +// (`default_gib`); zero or negative gives 0, which q4nx_cache_evict() reads as "no sweep". The result never exceeds +// INT64_MAX: the plain `gib << 30` this replaces overflowed from 2^33 GiB up, so a very large number written to mean "no +// limit" could come out as a 1 GiB bound, and a typo ("sixty") silently turned the sweep off. +int64_t q4nx_cache_bound_bytes(const char* text, int64_t default_gib = 60); + +// Mark `entry` (a directory in the cache) as used now. The sweep orders by the entry directory's mtime, which the filesystem +// only moves when a file is created or removed inside it -- so without this the order is by CREATION, and a container +// imported last week and served every day is evicted before yesterday's one-off. Call it on every cache hit. It also puts a +// just-opened entry inside the sweep's age window, which is what makes "a serve may be reading it" true for a reused entry +// and not only for a new one. Failure is ignored (a read-only cache still serves). +void q4nx_cache_touch(const std::string& entry); diff --git a/app/serve.cpp b/app/serve.cpp index 146ef794e..9556af962 100644 --- a/app/serve.cpp +++ b/app/serve.cpp @@ -2958,12 +2958,12 @@ std::string q4nx_cache_root() { // engine#1956: the cache is regenerable scratch, not user data, and nothing ever removed an old entry -- measured at 104 G in // 29 entries on one box, which closed two lanes' build gates the same day. The bound is a size, not an age, because a busy // day imports more packs than any age window expires; ONEBIT_Q4NX_CACHE_MAX_GIB=0 restores the old unbounded behaviour. +// The order is least-recently-USED: a cache hit touches its entry (q4nx_cache_touch), here and in npu/q4nx_device.cpp. // The entry just written is never a candidate, and neither is one younger than the sweep's age window (a serve may be reading // it) -- see app/q4nx_cache.h for why that is a heuristic rather than a guarantee. static void q4nx_cache_sweep(const std::string& cache, const std::string& keep) { - const char* env = std::getenv("ONEBIT_Q4NX_CACHE_MAX_GIB"); - long long gib = env && *env ? std::atoll(env) : 60; - q4nx_cache_evict(cache, keep, gib > 0 ? gib * (1LL << 30) : 0); + q4nx_cache_touch(keep); // a reused entry counts as used now: the sweep is LRU by this, not by creation + q4nx_cache_evict(cache, keep, q4nx_cache_bound_bytes(std::getenv("ONEBIT_Q4NX_CACHE_MAX_GIB"))); } std::string repack_gguf(const std::string& gguf) { @@ -3041,9 +3041,13 @@ std::string repack_gguf(const std::string& gguf) { "1bit serve: ============================================================\n\n", checker.empty() ? "(unresolved)" : checker.c_str()); } + q4nx_cache_touch(out); // a hit is a use: without this the sweep's order is by creation + return out; + } + if (run_program({ python, checker, gguf, out }, true) == 0) { + q4nx_cache_touch(out); return out; } - if (run_program({ python, checker, gguf, out }, true) == 0) return out; std::fprintf(stderr, "1bit serve: cached Q4NX dir %s FAILED the declared-scale scan; " "discarding and re-repacking\n", out.c_str()); fs::remove_all(out); diff --git a/docs/npu.md b/docs/npu.md index bff42342d..727a20037 100644 --- a/docs/npu.md +++ b/docs/npu.md @@ -262,6 +262,16 @@ cache (`$ONEBIT_Q4NX_CACHE`, else `~/.cache/1bit/q4nx`) and hands the loader tha device streams its 5,120-byte bf16 q4_1 while the container on disk stays exact. The repack is where the bf16 rounding reappears for the device, never in the container. +**The cache is bounded, and the bound is least-recently-used (engine#1956).** After writing an +entry, `1bit serve` removes the least-recently-used entries until the cache is within +`ONEBIT_Q4NX_CACHE_MAX_GIB` (default 60; `0` or a negative number turns the sweep off). The entry +just written and any entry used in the last ten minutes are never removed. "Used" means reused, +not only written: a cache hit — a repacked GGUF found in the cache, or a device-form conversion +reused at load — marks the entry's directory as used now, so a container imported once and served +every day outlives a one-off imported yesterday. The value is read as a whole number of GiB: a +number too large to be a byte count is the largest bound that is one (it does not wrap to a small +one), and a value that is not a number gets the default rather than an unbounded cache. + **The packed lm-head is cached beside it (engine#1716).** A model whose head runs as int8 tiles packs those tiles on the host — gather `kpad × T` weights per tile, quantise each column to int8 with its own scale — and until this cache that happened on every serve. The image is a function diff --git a/npu/q4nx_device.cpp b/npu/q4nx_device.cpp index b6e014028..794e476a2 100644 --- a/npu/q4nx_device.cpp +++ b/npu/q4nx_device.cpp @@ -185,6 +185,9 @@ DeviceForm device_form(const std::string& model_path, const std::string& cache_d out.model = model; out.converted = true; out.reused = true; + // A reuse is a use. The cache's sweep (app/q4nx_cache.h) orders entries by this directory's mtime, which nothing + // else moves after the conversion is written; the stamp compares the SOURCE's mtime, so this does not disturb it. + fs::last_write_time(dir, fs::file_time_type::clock::now(), ec); std::fprintf(stderr, "1bit q4nx: reusing the device-form conversion of %s at %s\n", model_path.c_str(), dir.c_str()); return out; diff --git a/tests/q4nx_cache_test.cpp b/tests/q4nx_cache_test.cpp index ba5ddbcb8..32cd463a0 100644 --- a/tests/q4nx_cache_test.cpp +++ b/tests/q4nx_cache_test.cpp @@ -87,8 +87,42 @@ int main() { check(r.removed == 0 && fs::exists(root / "keepme-1") && fs::exists(root / "keepme-2"), "a non-positive bound disables the sweep"); + // (f) a hit is a use: the older entry, touched, outlives a younger one nobody opened. Without the touch the order is + // by creation and the entry served every day goes first. fs::remove_all(root, ec); - std::printf("q4nx cache: %d arm(s) over the sweep (bound, LRU order, keep, age window, disabled)\n", - failures ? 0 : 12); + make_entry(root, "served-daily", 4, 7 * 24 * 60); + make_entry(root, "one-off", 4, 24 * 60); + q4nx_cache_touch((root / "served-daily").string()); + r = q4nx_cache_evict(root.string(), "", 6LL << 20, /*min_age_seconds=*/0); + check(r.removed == 1, "one entry has to go (removed " + std::to_string(r.removed) + ")"); + check(fs::exists(root / "served-daily"), "the entry that was just used survives"); + check(!fs::exists(root / "one-off"), "and the unused one is the one removed"); + // ... and a touched entry is inside the age window, so a serve that just opened it is covered like a new one + make_entry(root, "one-off", 4, 24 * 60); + make_entry(root, "opened-now", 4, 7 * 24 * 60); + fs::remove_all(root / "served-daily", ec); + q4nx_cache_touch((root / "opened-now").string()); + r = q4nx_cache_evict(root.string(), "", 1LL << 20); + check(fs::exists(root / "opened-now") && !fs::exists(root / "one-off"), + "a touched entry is protected by the age window"); + q4nx_cache_touch((root / "absent").string()); + check(!fs::exists(root / "absent"), "touching a missing entry creates nothing"); + + // (g) the bound from ONEBIT_Q4NX_CACHE_MAX_GIB's text: never wraps, and a typo is not "unbounded" + const int64_t gib = 1LL << 30, top = (INT64_MAX >> 30) << 30; + check(q4nx_cache_bound_bytes(nullptr) == 60 * gib && q4nx_cache_bound_bytes("") == 60 * gib, "unset is the default"); + check(q4nx_cache_bound_bytes("100") == 100 * gib, "a plain number is that many GiB"); + check(q4nx_cache_bound_bytes("0") == 0 && q4nx_cache_bound_bytes("-5") == 0, "zero or negative disables the sweep"); + check(q4nx_cache_bound_bytes("8589934592") == top, "2^33 GiB clamps instead of overflowing"); + check(q4nx_cache_bound_bytes("17179869184") == top, "2^34 GiB is not a bound of 0"); + check(q4nx_cache_bound_bytes("17179869185") == top, "2^34+1 GiB is not a bound of 1 GiB"); + check(q4nx_cache_bound_bytes("99999999999999999999999") == top, "a number past long long clamps too"); + check(q4nx_cache_bound_bytes("sixty") == 60 * gib, "a value that is not a number is the default, not unbounded"); + check(q4nx_cache_bound_bytes("40G") == 40 * gib, "a number with a suffix keeps its number"); + check(q4nx_cache_bound_bytes("7", 9) == 7 * gib && q4nx_cache_bound_bytes(nullptr, 9) == 9 * gib, "the default is the caller's"); + + fs::remove_all(root, ec); + std::printf("q4nx cache: %d arm(s) over the sweep (bound, LRU order, keep, age window, disabled, touch on a hit, bound text)\n", + failures ? 0 : 26); return failures ? 1 : 0; }