From bcc9ade1f2011036c2b363c7bf1cabb12b0c3fa4 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 08:03:30 -0400 Subject: [PATCH] npu-pack: sweep every fconst19 scratch directory, not only the one written to Fixes #1907. Since #1801 the 1.9 canonical copy is written to the on-disk cache first, and evict_stale_canonical_copies() (stale siblings + the 6 h age rule, #1770) ran only on that chosen directory. The copies older binaries left in /tmp -- tmpfs, i.e. RAM -- were never looked at again: 188 files, 17.0 GiB on strixhalo. sweep_other_fconst_scratch() applies the same rules to every other fconst_scratch_dirs() entry after each write; the current directory (a write fallback) is never swept. New arm test_sweep_other_fconst_scratch (an aged copy and a stale sibling in a non-chosen dir go; a fresh copy, a non-stem file, the chosen dir and the cwd are kept). Mutants: a no-op sweep fails 3 checks, dropping the cwd guard fails 2. Co-Authored-By: Claude Opus 5.5 --- npu/pack_import.cpp | 26 +++++++++++++++++++++ npu/pack_import.h | 5 ++++ tests/npu_pack_import_test.cpp | 42 +++++++++++++++++++++++++++++++++- 3 files changed, 72 insertions(+), 1 deletion(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 248c2f6ce..cba6b161c 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -753,6 +753,8 @@ std::string materialise_token_fconst(const std::string& pack_dir, const Dims& di const std::string legacy_name = std::string(kScratchStem) + "-" + tag + "-" + std::to_string(dims.hidden) + "-" + std::to_string(dims.heads) + "-" + std::to_string(dims.layers) + ".fconst"; evict_stale_canonical_copies(d.string(), name, ident_prefix, legacy_name, kScratchMaxAgeSeconds); + // engine#1907: and the scratch directories this run did NOT write to -- above all /tmp, which older binaries used. + sweep_other_fconst_scratch(fconst_scratch_dirs(), d.string(), name, ident_prefix, legacy_name, kScratchMaxAgeSeconds); // Say where it went whenever that is not the ordinary place: the copy is a few hundred MB and it is scratch. std::error_code ec2; const fs::path tmpdir = [&] { try { return fs::temp_directory_path(); } catch (...) { return fs::path(); } }(); @@ -4908,4 +4910,28 @@ void evict_stale_canonical_copies(const std::string& dir, const std::string& kee } } +// engine#1907: the sweep above ran only on the directory the copy was WRITTEN to. Since #1801 that is the on-disk cache, so +// the copies older binaries left in /tmp (tmpfs, i.e. RAM) were never looked at again -- 188 files, 17.0 GiB measured on +// strixhalo 2026-11-10, 2-9 h old. Every other scratch directory gets the same rules (stale siblings of this pack and geometry, +// and anything of the stem untouched for `max_age_seconds`). The current directory is never swept: it is the last-resort +// WRITE location, not a scratch directory, and a user's working directory is not this importer's to clean. +int sweep_other_fconst_scratch(const std::vector& dirs, const std::string& chosen_dir, + const std::string& keep_name, const std::string& prefix, const std::string& legacy_name, + int64_t max_age_seconds) { + std::error_code ec; + const fs::path chosen = chosen_dir.empty() ? fs::path() : fs::weakly_canonical(fs::path(chosen_dir), ec); + const fs::path cwd = fs::weakly_canonical(fs::current_path(ec), ec); + int swept = 0; + for (const std::string& d : dirs) { + if (d.empty() || d == ".") continue; + std::error_code e2; + if (!fs::is_directory(fs::path(d), e2)) continue; + const fs::path c = fs::weakly_canonical(fs::path(d), e2); + if (e2 || c == chosen || c == cwd) continue; + evict_stale_canonical_copies(d, keep_name, prefix, legacy_name, max_age_seconds); + ++swept; + } + return swept; +} + } // namespace onebit::npu::pack_import diff --git a/npu/pack_import.h b/npu/pack_import.h index 01541c3d7..b6d640806 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -492,6 +492,11 @@ const Fconst19Rule* fconst19_rules(int* count); // directory, then the temp directory and the cwd. Exposed so a test can pin the order -- the bug that shipped once was a cache // directory that was never created, so every pack silently fell through to tmpfs. std::vector fconst_scratch_dirs(); +// engine#1907: run evict_stale_canonical_copies() over every directory in `dirs` except `chosen_dir` (already swept by the +// write) and the current directory (a write fallback, never swept). Returns how many directories were swept. +int sweep_other_fconst_scratch(const std::vector& dirs, const std::string& chosen_dir, + const std::string& keep_name, const std::string& prefix, const std::string& legacy_name, + int64_t max_age_seconds); // True when `dir` is, or can be made into, a directory the scratch copy may be written to. Exposed with the order above because the // bug #1801 shipped once was exactly this: the cache directory existed only in the comment. bool fconst_scratch_ready(const std::string& dir); diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index c63b3fb93..aaca6df3f 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -640,6 +640,46 @@ void test_canonical_copy_reusable() { // engine#1770: the canonical copy is scratch that used to only ever accumulate -- reuse writes a new file when the source // changes, so 699 files / 8.7 GiB of tmpfs (RAM) had built up on one box. The eviction rule is "same pack and geometry, // different source identity", and what can go wrong is deleting one of the neighbours that must survive, so each is an arm. +// engine#1907: the eviction must reach the scratch directories a run did NOT write to. Since the canonical copy moved to the +// on-disk cache, /tmp (tmpfs) was never swept again and 17.0 GiB of copies from older binaries stayed in RAM. The chosen +// directory is left to the writer's own sweep, and the current directory is never swept. +void test_sweep_other_fconst_scratch() { + const fs::path base = fs::temp_directory_path() / ("1bit-sweep-" + std::to_string(::getpid())); + std::error_code ec; + fs::remove_all(base, ec); + const fs::path chosen = base / "cache", other = base / "tmp", cwd = base / "cwd"; + for (const auto& d : {chosen, other, cwd}) fs::create_directories(d); + const std::string prefix = "1bit-fconst19v1-76263210-896-14-24-"; + const std::string keep = prefix + "291653952-1.fconst"; + const auto old = fs::file_time_type::clock::now() - std::chrono::hours(7); + auto mk = [&](const fs::path& p, bool aged) { + put_file(p.string(), "x"); + if (aged) fs::last_write_time(p, old, ec); + return p; + }; + const fs::path o_old = mk(other / "1bit-fconst19v1-111-4096-32-32-9-9.fconst", true); // another pack, 7 h old + const fs::path o_new = mk(other / "1bit-fconst19v1-222-4096-32-32-9-9.fconst", false); // another pack, fresh + const fs::path o_sib = mk(other / (prefix + "291653952-0.fconst"), false); // THIS pack, stale identity + const fs::path o_unrel = mk(other / "notes.txt", true); // not the stem + const fs::path c_old = mk(chosen / "1bit-fconst19v1-333-4096-32-32-9-9.fconst", true); // chosen dir: not this sweep's + const fs::path w_old = mk(cwd / "1bit-fconst19v1-444-4096-32-32-9-9.fconst", true); // cwd: never swept + const fs::path prev = fs::current_path(ec); + fs::current_path(cwd, ec); + const int swept = pi::sweep_other_fconst_scratch({chosen.string(), other.string(), ".", cwd.string()}, chosen.string(), + keep, prefix, "", 6 * 3600); + fs::current_path(prev, ec); + check(swept == 1, "scratch sweep: exactly the one directory that is neither chosen nor the cwd is swept (" + + std::to_string(swept) + ")"); + check(!fs::exists(o_old), "scratch sweep: a 7 h old copy in a directory the run did not write to goes (the #1907 RAM)"); + check(fs::exists(o_new), "scratch sweep: a fresh copy of another pack there is kept -- a lane may be using it"); + check(!fs::exists(o_sib), "scratch sweep: this pack's stale-identity sibling there goes too"); + check(fs::exists(o_unrel), "scratch sweep: a file that is not fconst19 scratch is kept, however old"); + check(fs::exists(c_old), "scratch sweep: the chosen directory is left to the writer's own sweep"); + check(fs::exists(w_old), "scratch sweep: the current directory is never swept, however old the file"); + fs::remove_all(base, ec); + std::printf(" scratch sweep: other scratch dirs swept by age and sibling, chosen dir and cwd left alone\n"); +} + void test_evict_stale_canonical_copies() { const fs::path dir = fs::temp_directory_path() / ("1bit-evict-" + std::to_string(::getpid())); std::error_code ec; @@ -3682,7 +3722,7 @@ int main(int argc, char** argv) { test_external_roles(); test_external_container_names(); test_external_packed_genre(golden_dir.empty() ? "." : golden_dir); - test_canonical_copy_reusable(); test_evict_stale_canonical_copies(); + test_canonical_copy_reusable(); test_evict_stale_canonical_copies(); test_sweep_other_fconst_scratch(); test_crafted_refusals(golden_dir.empty() ? "." : golden_dir); test_modelbin_genre_refusal(golden_dir.empty() ? "." : golden_dir); test_1671_plan_head_arm(golden_dir.empty() ? "." : golden_dir);