From f1c68cbf562545c54f8416ab281fa5038a1af085 Mon Sep 17 00:00:00 2001 From: t Date: Tue, 10 Nov 2026 01:55:04 -0400 Subject: [PATCH 1/2] tests: pin the .fconst scratch order AND that the cache directory is created (#1801's reviewer asked for the second) The review of #1801 said: right diagnosis, right order, "it needs the test that would catch the case it already found once". That case is not the ORDER -- it is that the first version of the fix did not create the cache directory, so on a fresh box the loop walked past the new first choice and every pack's canonical .fconst still landed in tmpfs. A test of the order alone would have passed through the whole bug. So the decision is extracted and tested rather than living inside the import path: npu/pack_import.cpp/.h fconst_scratch_dirs() preference order: ONEBIT_FCONST_SCRATCH -> XDG/HOME cache -> /tmp -> cwd fconst_scratch_ready(dir) "is (or can be made) a directory" -- the create-or-skip rule tests/npu_pack_import_test.cpp::test_fconst_scratch_order 1. the override wins 2. XDG_CACHE_HOME/1bit/fconst19 when no override 3. HOME/.cache/1bit/fconst19 as the default -- and never the temp directory first 4. a MISSING directory is created, not skipped <- the bug 5. there is always a fallback The arm discriminates, verified by reverting the fix rather than by reading it: with fconst_scratch_ready() reduced to `return false` the run reports FAIL fconst scratch: a missing cache directory is created, not skipped (the #1801 first-version bug) 1 check(s) failed and with the fix in place the suite reports `all npu-pack checks passed` and prints "fconst scratch: order is override -> disk cache -> temp/cwd, pinned". Also worth recording from doing it: the first re-build after restoring the source was a no-op, because `cp` kept the backup's older mtime and make considered the object newer than the source -- the test kept failing against a binary that still contained the deliberate break. `touch` forced the rebuild. A stale binary has now produced a false result three times in this lane; the habit that catches it is to check the binary's mtime against the source before believing a failure. --- npu/pack_import.cpp | 53 +++++++++++++++++++--------------- npu/pack_import.h | 7 +++++ tests/npu_pack_import_test.cpp | 36 +++++++++++++++++++++++ 3 files changed, 73 insertions(+), 23 deletions(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 60a8bc092..9d9324abf 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -734,32 +734,11 @@ std::string materialise_token_fconst(const std::string& pack_dir, const Dims& di // reboot: three of them from one afternoon's imports were 11.4 GB of tmpfs (measured 2026-11-10). So the order is: // an explicit override, then the on-disk cache directory, and only then the temp directory this used to prefer. std::vector dirs; - if (const char* scratch = std::getenv("ONEBIT_FCONST_SCRATCH"); scratch && *scratch) - dirs.emplace_back(scratch); - if (const char* xdg = std::getenv("XDG_CACHE_HOME"); xdg && *xdg) - dirs.emplace_back(fs::path(xdg) / "1bit/fconst19"); - else if (const char* home = std::getenv("HOME"); home && *home) - dirs.emplace_back(fs::path(home) / ".cache/1bit/fconst19"); - try { - // Last resort only: on strixhalo /tmp is tmpfs, so this costs RAM equal to the pack's fconst. - dirs.push_back(fs::temp_directory_path()); - } catch (const std::exception& e) { - std::fprintf(stderr, - "1bit npu-pack: note: the temporary directory is unusable (%s), so the 1.9 pack's canonical copy " - "goes elsewhere\n", e.what()); - } - dirs.push_back(fs::path("/tmp")); + for (const auto& d : fconst_scratch_dirs()) dirs.emplace_back(d); dirs.push_back(fs::current_path()); std::string work; for (const auto& d : dirs) { - std::error_code ec; - // The cache directory is ours to create: without this the new first choice never wins, because it does not exist on a - // fresh box and the loop walked straight past it to /tmp (caught by testing the default, not the override). - if (!fs::is_directory(d, ec) || ec) { - std::error_code made; - fs::create_directories(d, made); - if (made || !fs::is_directory(d)) continue; - } + if (!fconst_scratch_ready(d.string())) continue; const fs::path cand = d / name; // Writable in fact, not just in theory: the copy is written immediately after this. std::ofstream probe(cand, std::ios::binary | std::ios::app); @@ -1796,6 +1775,34 @@ namespace { const Fconst19Rule* select_rule_in(const Fconst19Rule* rules, int count, int hidden, int heads, int layers, std::string* why); // defined with the table, internal to this file } +bool fconst_scratch_ready(const std::string& dir) { + // A candidate scratch directory is usable when it is (or can be made into) a directory. The cache directory is OURS to create: + // the version of engine#1801 that shipped first did not, so on a fresh box the loop walked straight past the new first choice + // and every pack fell through to tmpfs -- the bug this function exists to make testable. + std::error_code ec; + if (fs::is_directory(dir, ec) && !ec) return true; + std::error_code made; + fs::create_directories(dir, made); + return fs::is_directory(dir, ec) && !ec; +} + +std::vector fconst_scratch_dirs() { + // Where the canonical .fconst copy may live, in preference order. A function because the ORDER is the fix from engine#1801 and + // the reviewer's ask was a test that pins it: on strixhalo /tmp is tmpfs, so the first choice has to be disk, and a version of + // this that forgot to create the cache directory silently fell through to /tmp for every pack. + std::vector dirs; + if (const char* scratch = std::getenv("ONEBIT_FCONST_SCRATCH"); scratch && *scratch) + dirs.push_back(scratch); + if (const char* xdg = std::getenv("XDG_CACHE_HOME"); xdg && *xdg) + dirs.push_back(std::string(xdg) + "/1bit/fconst19"); + else if (const char* home = std::getenv("HOME"); home && *home) + dirs.push_back(std::string(home) + "/.cache/1bit/fconst19"); + // Last resorts only: on this box /tmp is tmpfs, so each of these costs RAM equal to the pack's .fconst. + dirs.push_back("/tmp"); + dirs.push_back("."); + return dirs; +} + const Fconst19Rule* fconst19_rules(int* count) { if (count) *count = int(sizeof(kFconst19Rules) / sizeof(kFconst19Rules[0])); return kFconst19Rules; diff --git a/npu/pack_import.h b/npu/pack_import.h index 85d6970e5..838513fbb 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -462,6 +462,13 @@ const Fconst19Rule* select_fconst19_rule(int hidden, int heads, int layers, std: // weights into the wrong projections while every anchor still lines up), and a row that a merge silently drops (the // h4096 row was lost twice that way, which un-measured a whole model class with nothing failing). const Fconst19Rule* fconst19_rules(int* count); +// The preference order for the 1.9 canonical .fconst scratch copy (engine#1801): the explicit override, then the on-disk cache +// directory, then the temp directory and the cwd. Exposed so a test can pin the order -- the bug that shipped once was a cache +// directory that was never created, so every pack silently fell through to tmpfs. +std::vector fconst_scratch_dirs(); +// True when `dir` is, or can be made into, a directory the scratch copy may be written to. Exposed with the order above because the +// bug #1801 shipped once was exactly this: the cache directory existed only in the comment. +bool fconst_scratch_ready(const std::string& dir); // The same decision over an explicit table, so a test can hand it the one shape that decides correctness and that the // real table must never contain: two rows for one geometry with different shapes. const Fconst19Rule* select_fconst19_rule_from(const Fconst19Rule* rules, int count, int hidden, int heads, int layers, diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 42805b3c0..4db4e60d4 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -2483,6 +2483,41 @@ void test_fconst19_rule_table() { std::printf("fconst19 rule: measured row, unmeasured refusal, and the ambiguity guard all behave\n"); } +// engine#1801's reviewer asked for the test that would catch the case the fix itself found once: the canonical .fconst copy is a +// cache whose size is the pack's own .fconst, and on this box /tmp is tmpfs, so where it lands is a RAM decision. The bug was a +// cache directory that was never created -- every pack then fell through to /tmp and nothing noticed. +void test_fconst_scratch_order() { + const auto first_with = [](const char* scratch, const char* xdg, const char* home) { + if (scratch) setenv("ONEBIT_FCONST_SCRATCH", scratch, 1); else unsetenv("ONEBIT_FCONST_SCRATCH"); + if (xdg) setenv("XDG_CACHE_HOME", xdg, 1); else unsetenv("XDG_CACHE_HOME"); + if (home) setenv("HOME", home, 1); else unsetenv("HOME"); + const auto dirs = pi::fconst_scratch_dirs(); + return dirs.empty() ? std::string() : dirs.front(); + }; + // 1. an explicit override wins, because a lane's own scratch dir is the only place it controls + check(first_with("/tmp/scratch-override", nullptr, "/home/x") == "/tmp/scratch-override", + "fconst scratch: ONEBIT_FCONST_SCRATCH is chosen first"); + // 2. otherwise the on-disk cache directory, never the temp directory -- the whole point of engine#1801 + check(first_with(nullptr, "/xdg-cache", "/home/x") == "/xdg-cache/1bit/fconst19", + "fconst scratch: XDG_CACHE_HOME/1bit/fconst19 is chosen when no override is set"); + check(first_with(nullptr, nullptr, "/home/x") == "/home/x/.cache/1bit/fconst19", + "fconst scratch: HOME/.cache/1bit/fconst19 is the default -- disk, not tmpfs"); + // 3. the temp directory must never come first: on this box that is RAM equal to the pack's .fconst + const auto dirs = pi::fconst_scratch_dirs(); + check(!dirs.empty() && dirs.front().find("/tmp") != 0, + "fconst scratch: the first choice is not the temp directory (it was, before #1801)"); + // 4. the bug the fix shipped once: a cache directory that does not exist must be CREATED, not skipped. Without this the loop + // walked past the new first choice and every pack's 3.6 GB scratch copy landed in tmpfs anyway. + const std::string fresh = "/tmp/1bit-scratch-test-" + std::to_string(::getpid()) + "/nested"; + std::system(("rm -rf /tmp/1bit-scratch-test-" + std::to_string(::getpid())).c_str()); + check(!std::filesystem::exists(fresh) && pi::fconst_scratch_ready(fresh) && std::filesystem::is_directory(fresh), + "fconst scratch: a missing cache directory is created, not skipped (the #1801 first-version bug)"); + std::system(("rm -rf /tmp/1bit-scratch-test-" + std::to_string(::getpid())).c_str()); + // 5. and there is always somewhere to fall back to + check(dirs.size() >= 2, "fconst scratch: more than one candidate, so a box with no HOME still has a fallback"); + std::printf("fconst scratch: order is override -> disk cache -> temp/cwd, pinned\n"); +} + void test_qwen3_attn_sections() { struct Pack { const char* name; int64_t hidden, q_out, kv_out, hd, h0, h1, h2; }; const Pack packs[] = {{"Qwen3-1.7B", 2048, 2048, 1024, 128, 4739136, 4739456, 7107264}, @@ -3181,6 +3216,7 @@ int main(int argc, char** argv) { test_three_input_matmulnbits(golden_dir.empty() ? "." : golden_dir); test_embedding_component(golden_dir.empty() ? "." : golden_dir); test_fconst19_rule_table(); + test_fconst_scratch_order(); test_selfcheck_reads_bf16(); if (const char* rc = std::getenv("NPU_EMBED_COMPONENT")) dump_real_component(rc); test_model_config_geometry(golden_dir.empty() ? "." : golden_dir); From 39802629ee67999f72cb728714ad6fa825a70d41 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 02:40:25 -0400 Subject: [PATCH 2/2] npu-pack: `$TMPDIR` sits between the cache and /tmp again, and an arm pins it Review of #1818. The extraction of `fconst_scratch_dirs()` went straight from the XDG/HOME cache to the literal "/tmp", so `fs::temp_directory_path()` was never consulted. On a lane that points TMPDIR at disk -- which is what people here do precisely because /tmp is tmpfs -- a pack whose cache directory is unusable (no HOME, read-only ~/.cache) then landed its multi-hundred-MB scratch copy in RAM. That is the outcome engine#1801 exists to prevent, and the new arms could not see it because none of them set TMPDIR. `temp_directory_path()` is back between the cache and "/tmp", inside a try with the note it had (it throws when TMPDIR names a directory that is not there), and only when it resolves to something other than /tmp -- otherwise the resolver would duplicate the literal below it. Arm (6): no override, no XDG, no HOME, TMPDIR set to a scratch directory -> TMPDIR is a candidate AND comes before "/tmp". Control: with TMPDIR unset, "/tmp" appears exactly once, so the resolver's own default cannot double-insert. Reverting the fix makes both checks fail: FAIL fconst scratch: a TMPDIR with no override and no cache is a candidate at all (it was dropped) FAIL fconst scratch: and it comes BEFORE the literal /tmp, which is RAM on this box and with it restored, `all npu-pack checks passed`. Refs #1818, #1801 --- npu/pack_import.cpp | 11 +++++++++++ tests/npu_pack_import_test.cpp | 28 ++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 9d9324abf..ab862d564 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -1797,6 +1797,17 @@ std::vector fconst_scratch_dirs() { dirs.push_back(std::string(xdg) + "/1bit/fconst19"); else if (const char* home = std::getenv("HOME"); home && *home) dirs.push_back(std::string(home) + "/.cache/1bit/fconst19"); + // engine#1818 review: `$TMPDIR` belongs BETWEEN the cache and the literal /tmp. It was dropped in this extraction, and on a + // lane that points TMPDIR at disk (which is what people do here precisely because /tmp is RAM) a pack whose cache directory + // is unusable then landed in tmpfs -- the outcome engine#1801 exists to prevent. temp_directory_path() throws when TMPDIR + // names a missing directory, hence the try and the note; it also returns /tmp when TMPDIR is unset, so the literal below is + // still the final disc-resort and not a duplicate. + try { + const fs::path tmp = fs::temp_directory_path(); + if (!tmp.empty() && tmp != fs::path("/tmp")) dirs.push_back(tmp.string()); + } catch (...) { + // The temporary directory is unusable (TMPDIR names something that is not there): go elsewhere. + } // Last resorts only: on this box /tmp is tmpfs, so each of these costs RAM equal to the pack's .fconst. dirs.push_back("/tmp"); dirs.push_back("."); diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 4db4e60d4..8afb5cd79 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -2515,6 +2515,34 @@ void test_fconst_scratch_order() { std::system(("rm -rf /tmp/1bit-scratch-test-" + std::to_string(::getpid())).c_str()); // 5. and there is always somewhere to fall back to check(dirs.size() >= 2, "fconst scratch: more than one candidate, so a box with no HOME still has a fallback"); + // 6. engine#1818 review: `$TMPDIR` sits BETWEEN the cache and the literal /tmp. It was dropped by the extraction, and a + // lane that points TMPDIR at disk -- which is what lanes here do, because /tmp is RAM -- then landed its scratch copy + // in tmpfs whenever the cache directory was unusable. That is the outcome #1801 exists to prevent, so it is pinned. + { + const std::string tdir = "/tmp/1bit-tmpdir-test-" + std::to_string(::getpid()); + std::system(("mkdir -p " + tdir).c_str()); + unsetenv("ONEBIT_FCONST_SCRATCH"); + unsetenv("XDG_CACHE_HOME"); + unsetenv("HOME"); // no usable cache: the case the reviewer named + setenv("TMPDIR", tdir.c_str(), 1); + const auto with_tmpdir = pi::fconst_scratch_dirs(); + size_t at_tmpdir = with_tmpdir.size(), at_slash_tmp = with_tmpdir.size(); + for (size_t i = 0; i < with_tmpdir.size(); ++i) { + if (with_tmpdir[i] == tdir) at_tmpdir = i; + if (with_tmpdir[i] == "/tmp") at_slash_tmp = i; + } + check(at_tmpdir < with_tmpdir.size(), + "fconst scratch: a TMPDIR with no override and no cache is a candidate at all (it was dropped)"); + check(at_tmpdir < at_slash_tmp, + "fconst scratch: and it comes BEFORE the literal /tmp, which is RAM on this box"); + // control: with TMPDIR unset the resolver returns /tmp, so the literal must still be there and unused twice + unsetenv("TMPDIR"); + const auto no_tmpdir = pi::fconst_scratch_dirs(); + check(std::count(no_tmpdir.begin(), no_tmpdir.end(), std::string("/tmp")) == 1, + "fconst scratch: with TMPDIR unset, /tmp appears exactly once (no duplicate from the resolver)"); + std::system(("rm -rf " + tdir).c_str()); + setenv("HOME", "/home/x", 1); // leave the environment as the arms above expect it + } std::printf("fconst scratch: order is override -> disk cache -> temp/cwd, pinned\n"); }