Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
21 changes: 21 additions & 0 deletions app/q4nx_cache.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -16,8 +16,10 @@
#include "q4nx_cache.h"

#include <algorithm>
#include <cerrno>
#include <chrono>
#include <cstdio>
#include <cstdlib>
#include <filesystem>
#include <vector>

Expand Down Expand Up @@ -83,3 +85,22 @@ Q4nxCacheSweep q4nx_cache_evict(const std::string& root, const std::string& keep
}
return r;
}

int64_t q4nx_cache_bound_bytes(const char* text, int64_t default_gib) {
const int64_t max_gib = INT64_MAX >> 30; // the largest GiB count whose byte count still fits
long long gib = default_gib;
if (text && *text) {
char* end = nullptr;
errno = 0;
const long long v = std::strtoll(text, &end, 10);
if (end != text) gib = v; // a number (ERANGE saturates, which the clamp below handles)
}
if (gib <= 0) return 0;
if (gib > max_gib) gib = max_gib;
return int64_t(gib) << 30;
}

void q4nx_cache_touch(const std::string& entry) {
std::error_code ec;
if (fs::is_directory(entry, ec)) fs::last_write_time(entry, fs::file_time_type::clock::now(), ec);
}
13 changes: 13 additions & 0 deletions app/q4nx_cache.h
Original file line number Diff line number Diff line change
Expand Up @@ -44,3 +44,16 @@ Q4nxCacheSweep q4nx_cache_evict(const std::string& root, const std::string& keep

// Directory bytes under `path` (0 when it does not exist).
int64_t q4nx_dir_bytes(const std::string& path);

// The byte bound for ONEBIT_Q4NX_CACHE_MAX_GIB's value (`text` may be null). Unset, empty or not a number gives the default
// (`default_gib`); zero or negative gives 0, which q4nx_cache_evict() reads as "no sweep". The result never exceeds
// INT64_MAX: the plain `gib << 30` this replaces overflowed from 2^33 GiB up, so a very large number written to mean "no
// limit" could come out as a 1 GiB bound, and a typo ("sixty") silently turned the sweep off.
int64_t q4nx_cache_bound_bytes(const char* text, int64_t default_gib = 60);

// Mark `entry` (a directory in the cache) as used now. The sweep orders by the entry directory's mtime, which the filesystem
// only moves when a file is created or removed inside it -- so without this the order is by CREATION, and a container
// imported last week and served every day is evicted before yesterday's one-off. Call it on every cache hit. It also puts a
// just-opened entry inside the sweep's age window, which is what makes "a serve may be reading it" true for a reused entry
// and not only for a new one. Failure is ignored (a read-only cache still serves).
void q4nx_cache_touch(const std::string& entry);
12 changes: 8 additions & 4 deletions app/serve.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2958,12 +2958,12 @@ std::string q4nx_cache_root() {
// engine#1956: the cache is regenerable scratch, not user data, and nothing ever removed an old entry -- measured at 104 G in
// 29 entries on one box, which closed two lanes' build gates the same day. The bound is a size, not an age, because a busy
// day imports more packs than any age window expires; ONEBIT_Q4NX_CACHE_MAX_GIB=0 restores the old unbounded behaviour.
// The order is least-recently-USED: a cache hit touches its entry (q4nx_cache_touch), here and in npu/q4nx_device.cpp.
// The entry just written is never a candidate, and neither is one younger than the sweep's age window (a serve may be reading
// it) -- see app/q4nx_cache.h for why that is a heuristic rather than a guarantee.
static void q4nx_cache_sweep(const std::string& cache, const std::string& keep) {
const char* env = std::getenv("ONEBIT_Q4NX_CACHE_MAX_GIB");
long long gib = env && *env ? std::atoll(env) : 60;
q4nx_cache_evict(cache, keep, gib > 0 ? gib * (1LL << 30) : 0);
q4nx_cache_touch(keep); // a reused entry counts as used now: the sweep is LRU by this, not by creation
q4nx_cache_evict(cache, keep, q4nx_cache_bound_bytes(std::getenv("ONEBIT_Q4NX_CACHE_MAX_GIB")));
}

std::string repack_gguf(const std::string& gguf) {
Expand Down Expand Up @@ -3041,9 +3041,13 @@ std::string repack_gguf(const std::string& gguf) {
"1bit serve: ============================================================\n\n",
checker.empty() ? "(unresolved)" : checker.c_str());
}
q4nx_cache_touch(out); // a hit is a use: without this the sweep's order is by creation
return out;
}
if (run_program({ python, checker, gguf, out }, true) == 0) {
q4nx_cache_touch(out);
return out;
}
if (run_program({ python, checker, gguf, out }, true) == 0) return out;
std::fprintf(stderr, "1bit serve: cached Q4NX dir %s FAILED the declared-scale scan; "
"discarding and re-repacking\n", out.c_str());
fs::remove_all(out);
Expand Down
10 changes: 10 additions & 0 deletions docs/npu.md
Original file line number Diff line number Diff line change
Expand Up @@ -262,6 +262,16 @@ cache (`$ONEBIT_Q4NX_CACHE`, else `~/.cache/1bit/q4nx`) and hands the loader tha
device streams its 5,120-byte bf16 q4_1 while the container on disk stays exact. The repack is where
the bf16 rounding reappears for the device, never in the container.

**The cache is bounded, and the bound is least-recently-used (engine#1956).** After writing an
entry, `1bit serve` removes the least-recently-used entries until the cache is within
`ONEBIT_Q4NX_CACHE_MAX_GIB` (default 60; `0` or a negative number turns the sweep off). The entry
just written and any entry used in the last ten minutes are never removed. "Used" means reused,
not only written: a cache hit — a repacked GGUF found in the cache, or a device-form conversion
reused at load — marks the entry's directory as used now, so a container imported once and served
every day outlives a one-off imported yesterday. The value is read as a whole number of GiB: a
number too large to be a byte count is the largest bound that is one (it does not wrap to a small
one), and a value that is not a number gets the default rather than an unbounded cache.

**The packed lm-head is cached beside it (engine#1716).** A model whose head runs as int8 tiles
packs those tiles on the host — gather `kpad × T` weights per tile, quantise each column to int8
with its own scale — and until this cache that happened on every serve. The image is a function
Expand Down
3 changes: 3 additions & 0 deletions npu/q4nx_device.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -185,6 +185,9 @@ DeviceForm device_form(const std::string& model_path, const std::string& cache_d
out.model = model;
out.converted = true;
out.reused = true;
// A reuse is a use. The cache's sweep (app/q4nx_cache.h) orders entries by this directory's mtime, which nothing
// else moves after the conversion is written; the stamp compares the SOURCE's mtime, so this does not disturb it.
fs::last_write_time(dir, fs::file_time_type::clock::now(), ec);
std::fprintf(stderr, "1bit q4nx: reusing the device-form conversion of %s at %s\n",
model_path.c_str(), dir.c_str());
return out;
Expand Down
38 changes: 36 additions & 2 deletions tests/q4nx_cache_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -87,8 +87,42 @@ int main() {
check(r.removed == 0 && fs::exists(root / "keepme-1") && fs::exists(root / "keepme-2"),
"a non-positive bound disables the sweep");

// (f) a hit is a use: the older entry, touched, outlives a younger one nobody opened. Without the touch the order is
// by creation and the entry served every day goes first.
fs::remove_all(root, ec);
std::printf("q4nx cache: %d arm(s) over the sweep (bound, LRU order, keep, age window, disabled)\n",
failures ? 0 : 12);
make_entry(root, "served-daily", 4, 7 * 24 * 60);
make_entry(root, "one-off", 4, 24 * 60);
q4nx_cache_touch((root / "served-daily").string());
r = q4nx_cache_evict(root.string(), "", 6LL << 20, /*min_age_seconds=*/0);
check(r.removed == 1, "one entry has to go (removed " + std::to_string(r.removed) + ")");
check(fs::exists(root / "served-daily"), "the entry that was just used survives");
check(!fs::exists(root / "one-off"), "and the unused one is the one removed");
// ... and a touched entry is inside the age window, so a serve that just opened it is covered like a new one
make_entry(root, "one-off", 4, 24 * 60);
make_entry(root, "opened-now", 4, 7 * 24 * 60);
fs::remove_all(root / "served-daily", ec);
q4nx_cache_touch((root / "opened-now").string());
r = q4nx_cache_evict(root.string(), "", 1LL << 20);
check(fs::exists(root / "opened-now") && !fs::exists(root / "one-off"),
"a touched entry is protected by the age window");
q4nx_cache_touch((root / "absent").string());
check(!fs::exists(root / "absent"), "touching a missing entry creates nothing");

// (g) the bound from ONEBIT_Q4NX_CACHE_MAX_GIB's text: never wraps, and a typo is not "unbounded"
const int64_t gib = 1LL << 30, top = (INT64_MAX >> 30) << 30;
check(q4nx_cache_bound_bytes(nullptr) == 60 * gib && q4nx_cache_bound_bytes("") == 60 * gib, "unset is the default");
check(q4nx_cache_bound_bytes("100") == 100 * gib, "a plain number is that many GiB");
check(q4nx_cache_bound_bytes("0") == 0 && q4nx_cache_bound_bytes("-5") == 0, "zero or negative disables the sweep");
check(q4nx_cache_bound_bytes("8589934592") == top, "2^33 GiB clamps instead of overflowing");
check(q4nx_cache_bound_bytes("17179869184") == top, "2^34 GiB is not a bound of 0");
check(q4nx_cache_bound_bytes("17179869185") == top, "2^34+1 GiB is not a bound of 1 GiB");
check(q4nx_cache_bound_bytes("99999999999999999999999") == top, "a number past long long clamps too");
check(q4nx_cache_bound_bytes("sixty") == 60 * gib, "a value that is not a number is the default, not unbounded");
check(q4nx_cache_bound_bytes("40G") == 40 * gib, "a number with a suffix keeps its number");
check(q4nx_cache_bound_bytes("7", 9) == 7 * gib && q4nx_cache_bound_bytes(nullptr, 9) == 9 * gib, "the default is the caller's");

fs::remove_all(root, ec);
std::printf("q4nx cache: %d arm(s) over the sweep (bound, LRU order, keep, age window, disabled, touch on a hit, bound text)\n",
failures ? 0 : 26);
return failures ? 1 : 0;
}
Loading