Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
19 changes: 14 additions & 5 deletions npu/pack_import.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3171,17 +3171,28 @@ static const char* const kTextFiles[] = {"tokenizer.json",
"generation_config.json"};

// The same list every path copies (see kTextFiles): a container without tokenizer.json is not an NPU model directory.
// engine#1751 follow-up: the ONE copier. Both import paths call it (the .fconst path used to keep its own loop, which
// threw a bare filesystem_error, while this one counted successes and swallowed failures), and a file the pack HAS that
// fails to copy is refused by name -- a counter nobody compared let an import "succeed" without tokenizer.json, and serve
// then refused the directory for a missing tokenizer, the exact misleading report #1751 existed to remove. A file the pack
// does not ship stays a skip: a pack without a chat template is legitimate.
static int copy_text_artifacts(const std::string& pack_dir, const std::string& out_dir) {
int copied = 0;
for (const char* f : kTextFiles) {
const std::string src = pack_dir + "/" + f;
std::error_code cec;
if (!fs::exists(src, cec)) continue;
fs::copy_file(src, out_dir + "/" + f, fs::copy_options::overwrite_existing, cec);
if (!cec) ++copied;
if (cec)
throw std::runtime_error("cannot copy the pack's " + std::string(f) + " into " + out_dir + " (" + cec.message() +
"); without it the output is not a servable model directory");
++copied;
}
return copied;
}
int copy_text_artifacts_exported(const std::string& pack_dir, const std::string& out_dir) {
return copy_text_artifacts(pack_dir, out_dir);
}

TextArtifacts scan_text_artifacts(const std::string& pack_dir) {
TextArtifacts t;
Expand Down Expand Up @@ -4600,10 +4611,8 @@ Report import_pack(const std::string& pack_dir, const std::string& out_dir, cons
std::ofstream out(out_dir + "/config.json");
out << c.dump(2) << "\n";
}
for (const char* f : kTextFiles) {
const std::string src = pack_dir + "/" + f;
if (fs::exists(src)) fs::copy_file(src, out_dir + "/" + f, fs::copy_options::overwrite_existing);
}
// engine#1751 follow-up: the same copier the external-data path uses, so the two genres cannot drift apart again.
copy_text_artifacts(pack_dir, out_dir);
for (const auto& [name, m] : ext) unmap_file(m, ext_size[name]);
return r;
}
Expand Down
4 changes: 4 additions & 0 deletions npu/pack_import.h
Original file line number Diff line number Diff line change
Expand Up @@ -202,6 +202,10 @@ struct TextArtifacts {
bool tokenizer = false; // tokenizer.json present: what `1bit serve` opens
};
TextArtifacts scan_text_artifacts(const std::string& pack_dir);
// engine#1751 follow-up: the one text-artifact copier both import paths use. Copies every kTextFiles entry the pack has
// into out_dir and returns how many; a file the pack has that cannot be copied throws, naming it (a skipped file is not
// a failure -- packs legitimately ship subsets). Exported so a test can drive the failure path without a pack.
int copy_text_artifacts_exported(const std::string& pack_dir, const std::string& out_dir);

// engine#1740 (W1a): where the 1.9 Token .fconst keeps its output layer. The arithmetic is exposed because it is the
// half that can be wrong silently -- a one-tile error reads a plausible region and only a correlation against the
Expand Down
29 changes: 29 additions & 0 deletions tests/npu_pack_import_test.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2887,6 +2887,34 @@ void test_kernel_set_refusal() {
"kernels: a file that is not an .elf is not a kernel");
}

// engine#1751 follow-up: the one copier both import paths use. #1751 added a second copier that counted successes and
// swallowed failures, so an import could "succeed" without tokenizer.json and serve then refused the directory for a
// missing tokenizer -- the misleading report #1751 was written to remove. A file the pack has and cannot copy is refused
// BY NAME; a file the pack does not ship is not a failure.
void test_copy_text_artifacts() {
const fs::path base = fs::temp_directory_path() / ("1bit-textcopy-" + std::to_string(::getpid()));
std::error_code ec;
fs::remove_all(base, ec);
const fs::path pack = base / "pack", out = base / "out";
fs::create_directories(pack);
fs::create_directories(out);
put_file((pack / "tokenizer.json").string(), "{}");
put_file((pack / "tokenizer_config.json").string(), "{}");
// (a) the files the pack has are copied, the rest skipped without complaint
int copied = -1;
std::string why;
try { copied = pi::copy_text_artifacts_exported(pack.string(), out.string()); } catch (const std::exception& e) { why = e.what(); }
check(why.empty() && copied == 2 && fs::exists(out / "tokenizer.json") && fs::exists(out / "tokenizer_config.json"),
"text copy: the two files the pack ships are copied, the six it does not are skipped (" + why + ")");
// (b) a copy that FAILS is refused, naming the file -- not counted down to a smaller number nobody reads
why.clear();
try { (void)pi::copy_text_artifacts_exported(pack.string(), (base / "no-such-dir").string()); } catch (const std::exception& e) { why = e.what(); }
check(why.find("tokenizer.json") != std::string::npos,
"text copy: a file the pack has that cannot be copied is refused by name (" + why + ")");
fs::remove_all(base, ec);
std::printf(" text copy: present files copied, absent ones skipped, a failed copy refused by name\n");
}

void test_text_artifacts(const std::string& root) {
const fs::path dir = fs::path(root) / "text-artifacts";
fs::remove_all(dir);
Expand Down Expand Up @@ -3420,6 +3448,7 @@ int main(int argc, char** argv) {
test_final_norm_envelope(golden_dir.empty() ? "." : golden_dir);
test_import_fp32_params(golden_dir.empty() ? "." : golden_dir);
test_text_artifacts(golden_dir.empty() ? "." : golden_dir);
test_copy_text_artifacts();
test_kernel_set_refusal();
test_weight_layout();
test_null_optional_dims();
Expand Down
Loading