From dc64f40ae8ebaed3bb87c18d2331a0d8dc01e301 Mon Sep 17 00:00:00 2001 From: bong-water-water-bong Date: Tue, 10 Nov 2026 03:24:30 -0400 Subject: [PATCH] pack_import: one text copier for both import genres, refusing a failed copy by name Refs #1751. #1751 merged with two copiers: copy_text_artifacts() (external-data path) counted successes and swallowed copy errors, so an import could finish without tokenizer.json and serve then refused the directory for a missing tokenizer; the .fconst path kept its own inline loop. Both paths now call the one copier, which skips files the pack does not ship and throws, naming the file, on any copy that fails. New host arm test_copy_text_artifacts; the mutant that restores the swallow fails it. Co-Authored-By: Claude Opus 5.5 --- npu/pack_import.cpp | 19 ++++++++++++++----- npu/pack_import.h | 4 ++++ tests/npu_pack_import_test.cpp | 29 +++++++++++++++++++++++++++++ 3 files changed, 47 insertions(+), 5 deletions(-) diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 4ffb990d8..0fb65d63f 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -3171,6 +3171,11 @@ static const char* const kTextFiles[] = {"tokenizer.json", "generation_config.json"}; // The same list every path copies (see kTextFiles): a container without tokenizer.json is not an NPU model directory. +// engine#1751 follow-up: the ONE copier. Both import paths call it (the .fconst path used to keep its own loop, which +// threw a bare filesystem_error, while this one counted successes and swallowed failures), and a file the pack HAS that +// fails to copy is refused by name -- a counter nobody compared let an import "succeed" without tokenizer.json, and serve +// then refused the directory for a missing tokenizer, the exact misleading report #1751 existed to remove. A file the pack +// does not ship stays a skip: a pack without a chat template is legitimate. static int copy_text_artifacts(const std::string& pack_dir, const std::string& out_dir) { int copied = 0; for (const char* f : kTextFiles) { @@ -3178,10 +3183,16 @@ static int copy_text_artifacts(const std::string& pack_dir, const std::string& o std::error_code cec; if (!fs::exists(src, cec)) continue; fs::copy_file(src, out_dir + "/" + f, fs::copy_options::overwrite_existing, cec); - if (!cec) ++copied; + if (cec) + throw std::runtime_error("cannot copy the pack's " + std::string(f) + " into " + out_dir + " (" + cec.message() + + "); without it the output is not a servable model directory"); + ++copied; } return copied; } +int copy_text_artifacts_exported(const std::string& pack_dir, const std::string& out_dir) { + return copy_text_artifacts(pack_dir, out_dir); +} TextArtifacts scan_text_artifacts(const std::string& pack_dir) { TextArtifacts t; @@ -4582,10 +4593,8 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons std::ofstream out(out_dir + "/config.json"); out << c.dump(2) << "\n"; } - for (const char* f : kTextFiles) { - const std::string src = pack_dir + "/" + f; - if (fs::exists(src)) fs::copy_file(src, out_dir + "/" + f, fs::copy_options::overwrite_existing); - } + // engine#1751 follow-up: the same copier the external-data path uses, so the two genres cannot drift apart again. + copy_text_artifacts(pack_dir, out_dir); for (const auto& [name, m] : ext) unmap_file(m, ext_size[name]); return r; } diff --git a/npu/pack_import.h b/npu/pack_import.h index 838513fbb..8ffe7a16f 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -202,6 +202,10 @@ struct TextArtifacts { bool tokenizer = false; // tokenizer.json present: what `1bit serve` opens }; TextArtifacts scan_text_artifacts(const std::string& pack_dir); +// engine#1751 follow-up: the one text-artifact copier both import paths use. Copies every kTextFiles entry the pack has +// into out_dir and returns how many; a file the pack has that cannot be copied throws, naming it (a skipped file is not +// a failure -- packs legitimately ship subsets). Exported so a test can drive the failure path without a pack. +int copy_text_artifacts_exported(const std::string& pack_dir, const std::string& out_dir); // engine#1740 (W1a): where the 1.9 Token .fconst keeps its output layer. The arithmetic is exposed because it is the // half that can be wrong silently -- a one-tile error reads a plausible region and only a correlation against the diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 25051340b..1e0a036fc 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -2757,6 +2757,34 @@ void test_kernel_set_refusal() { "kernels: a file that is not an .elf is not a kernel"); } +// engine#1751 follow-up: the one copier both import paths use. #1751 added a second copier that counted successes and +// swallowed failures, so an import could "succeed" without tokenizer.json and serve then refused the directory for a +// missing tokenizer -- the misleading report #1751 was written to remove. A file the pack has and cannot copy is refused +// BY NAME; a file the pack does not ship is not a failure. +void test_copy_text_artifacts() { + const fs::path base = fs::temp_directory_path() / ("1bit-textcopy-" + std::to_string(::getpid())); + std::error_code ec; + fs::remove_all(base, ec); + const fs::path pack = base / "pack", out = base / "out"; + fs::create_directories(pack); + fs::create_directories(out); + put_file((pack / "tokenizer.json").string(), "{}"); + put_file((pack / "tokenizer_config.json").string(), "{}"); + // (a) the files the pack has are copied, the rest skipped without complaint + int copied = -1; + std::string why; + try { copied = pi::copy_text_artifacts_exported(pack.string(), out.string()); } catch (const std::exception& e) { why = e.what(); } + check(why.empty() && copied == 2 && fs::exists(out / "tokenizer.json") && fs::exists(out / "tokenizer_config.json"), + "text copy: the two files the pack ships are copied, the six it does not are skipped (" + why + ")"); + // (b) a copy that FAILS is refused, naming the file -- not counted down to a smaller number nobody reads + why.clear(); + try { (void)pi::copy_text_artifacts_exported(pack.string(), (base / "no-such-dir").string()); } catch (const std::exception& e) { why = e.what(); } + check(why.find("tokenizer.json") != std::string::npos, + "text copy: a file the pack has that cannot be copied is refused by name (" + why + ")"); + fs::remove_all(base, ec); + std::printf(" text copy: present files copied, absent ones skipped, a failed copy refused by name\n"); +} + void test_text_artifacts(const std::string& root) { const fs::path dir = fs::path(root) / "text-artifacts"; fs::remove_all(dir); @@ -3289,6 +3317,7 @@ int main(int argc, char** argv) { test_final_norm_envelope(golden_dir.empty() ? "." : golden_dir); test_import_fp32_params(golden_dir.empty() ? "." : golden_dir); test_text_artifacts(golden_dir.empty() ? "." : golden_dir); + test_copy_text_artifacts(); test_kernel_set_refusal(); test_weight_layout(); test_null_optional_dims();