diff --git a/npu/pack_import.cpp b/npu/pack_import.cpp index 400e4fe29..ea8a61d5a 100644 --- a/npu/pack_import.cpp +++ b/npu/pack_import.cpp @@ -3171,6 +3171,11 @@ static const char* const kTextFiles[] = {"tokenizer.json", "generation_config.json"}; // The same list every path copies (see kTextFiles): a container without tokenizer.json is not an NPU model directory. +// engine#1751 follow-up: the ONE copier. Both import paths call it (the .fconst path used to keep its own loop, which +// threw a bare filesystem_error, while this one counted successes and swallowed failures), and a file the pack HAS that +// fails to copy is refused by name -- a counter nobody compared let an import "succeed" without tokenizer.json, and serve +// then refused the directory for a missing tokenizer, the exact misleading report #1751 existed to remove. A file the pack +// does not ship stays a skip: a pack without a chat template is legitimate. static int copy_text_artifacts(const std::string& pack_dir, const std::string& out_dir) { int copied = 0; for (const char* f : kTextFiles) { @@ -3178,10 +3183,16 @@ static int copy_text_artifacts(const std::string& pack_dir, const std::string& o std::error_code cec; if (!fs::exists(src, cec)) continue; fs::copy_file(src, out_dir + "/" + f, fs::copy_options::overwrite_existing, cec); - if (!cec) ++copied; + if (cec) + throw std::runtime_error("cannot copy the pack's " + std::string(f) + " into " + out_dir + " (" + cec.message() + + "); without it the output is not a servable model directory"); + ++copied; } return copied; } +int copy_text_artifacts_exported(const std::string& pack_dir, const std::string& out_dir) { + return copy_text_artifacts(pack_dir, out_dir); +} TextArtifacts scan_text_artifacts(const std::string& pack_dir) { TextArtifacts t; @@ -4600,10 +4611,8 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons std::ofstream out(out_dir + "/config.json"); out << c.dump(2) << "\n"; } - for (const char* f : kTextFiles) { - const std::string src = pack_dir + "/" + f; - if (fs::exists(src)) fs::copy_file(src, out_dir + "/" + f, fs::copy_options::overwrite_existing); - } + // engine#1751 follow-up: the same copier the external-data path uses, so the two genres cannot drift apart again. + copy_text_artifacts(pack_dir, out_dir); for (const auto& [name, m] : ext) unmap_file(m, ext_size[name]); return r; } diff --git a/npu/pack_import.h b/npu/pack_import.h index 838513fbb..8ffe7a16f 100644 --- a/npu/pack_import.h +++ b/npu/pack_import.h @@ -202,6 +202,10 @@ struct TextArtifacts { bool tokenizer = false; // tokenizer.json present: what `1bit serve` opens }; TextArtifacts scan_text_artifacts(const std::string& pack_dir); +// engine#1751 follow-up: the one text-artifact copier both import paths use. Copies every kTextFiles entry the pack has +// into out_dir and returns how many; a file the pack has that cannot be copied throws, naming it (a skipped file is not +// a failure -- packs legitimately ship subsets). Exported so a test can drive the failure path without a pack. +int copy_text_artifacts_exported(const std::string& pack_dir, const std::string& out_dir); // engine#1740 (W1a): where the 1.9 Token .fconst keeps its output layer. The arithmetic is exposed because it is the // half that can be wrong silently -- a one-tile error reads a plausible region and only a correlation against the diff --git a/tests/npu_pack_import_test.cpp b/tests/npu_pack_import_test.cpp index 019347612..6e0ba47d3 100644 --- a/tests/npu_pack_import_test.cpp +++ b/tests/npu_pack_import_test.cpp @@ -2887,6 +2887,34 @@ void test_kernel_set_refusal() { "kernels: a file that is not an .elf is not a kernel"); } +// engine#1751 follow-up: the one copier both import paths use. #1751 added a second copier that counted successes and +// swallowed failures, so an import could "succeed" without tokenizer.json and serve then refused the directory for a +// missing tokenizer -- the misleading report #1751 was written to remove. A file the pack has and cannot copy is refused +// BY NAME; a file the pack does not ship is not a failure. +void test_copy_text_artifacts() { + const fs::path base = fs::temp_directory_path() / ("1bit-textcopy-" + std::to_string(::getpid())); + std::error_code ec; + fs::remove_all(base, ec); + const fs::path pack = base / "pack", out = base / "out"; + fs::create_directories(pack); + fs::create_directories(out); + put_file((pack / "tokenizer.json").string(), "{}"); + put_file((pack / "tokenizer_config.json").string(), "{}"); + // (a) the files the pack has are copied, the rest skipped without complaint + int copied = -1; + std::string why; + try { copied = pi::copy_text_artifacts_exported(pack.string(), out.string()); } catch (const std::exception& e) { why = e.what(); } + check(why.empty() && copied == 2 && fs::exists(out / "tokenizer.json") && fs::exists(out / "tokenizer_config.json"), + "text copy: the two files the pack ships are copied, the six it does not are skipped (" + why + ")"); + // (b) a copy that FAILS is refused, naming the file -- not counted down to a smaller number nobody reads + why.clear(); + try { (void)pi::copy_text_artifacts_exported(pack.string(), (base / "no-such-dir").string()); } catch (const std::exception& e) { why = e.what(); } + check(why.find("tokenizer.json") != std::string::npos, + "text copy: a file the pack has that cannot be copied is refused by name (" + why + ")"); + fs::remove_all(base, ec); + std::printf(" text copy: present files copied, absent ones skipped, a failed copy refused by name\n"); +} + void test_text_artifacts(const std::string& root) { const fs::path dir = fs::path(root) / "text-artifacts"; fs::remove_all(dir); @@ -3420,6 +3448,7 @@ int main(int argc, char** argv) { test_final_norm_envelope(golden_dir.empty() ? "." : golden_dir); test_import_fp32_params(golden_dir.empty() ? "." : golden_dir); test_text_artifacts(golden_dir.empty() ? "." : golden_dir); + test_copy_text_artifacts(); test_kernel_set_refusal(); test_weight_layout(); test_null_optional_dims();