From cab6fadaf8a250851a59b83c8a8097db60d2edf7 Mon Sep 17 00:00:00 2001 From: "Piotr Wilkin (ilintar)" Date: Mon, 9 Nov 2026 19:41:01 +0100 Subject: [PATCH] feat: external PLE file loading (#31185) * model : allow loading external PLE files Assisted-by: Codex * convert : add separate PLE exports Assisted-by: Codex * tests : fix PLE export test lint Assisted-by: Codex * model : make --ple an alias of --shard, allow multiple shard locations Reuse the multi-shard loading path for external PLE files instead of a separate option. * --ple is an alias of --shard: both take a comma separated list and may be given more than once, and the shards may live anywhere * metadata comes from the main file; numbered shards are still auto-discovered and strictly validated * explicit files outside the discovered set are appended as extra shards, exempt from split.no and split.count checks * -isnm/--ignore-shard-num-mismatch loads only the given files, in order, with no auto-discovery and no split metadata validation * ple_path, path_ple and llama_model_loader::add_ple_file are removed; common_model_load() uses llama_model_load_from_splits for two or more paths * common/fit.{h,cpp} take the shard path vector, so --fit and the router memory measurement load every shard Assisted-by: GPT-6-Astra * rm -isnm * make sure to dedup paths * rm test_convert_ple * address comments * allow ple shard to override tensor --------- Co-authored-by: Xuan Son Nguyen (cherry picked from commit 5f490d7bfe78844e6390c869454c149fd7e003a8, ggml-org/llama.cpp) [1bit] Port onto 1bit/hrx-vulkan-patched for the ORCH 11-16 ask (Unsloth Flash-Next / DeepSeek-V4.1-Flash READMEs swap the Engram table with --ple). Two conflicts resolved: common/fit.cpp passes upstream's hp_ngl_skip/hp_nsk to common_get_device_memory_data_impl, an argument from an upstream change this fork does not carry -- the fork's own call sites are kept with path_model -> paths_model; conversion/base.py keeps the fork's numpy() conversion and adds upstream's PLE filter in front of it. Co-Authored-By: Claude Opus 5.5 --- common/arg.cpp | 10 ++ common/common.cpp | 19 ++- common/common.h | 14 +++ common/fit.cpp | 40 +++---- common/fit.h | 12 +- common/speculative.cpp | 2 +- conversion/base.py | 28 ++++- convert_hf_to_gguf.py | 14 +++ docs/models.md | 20 ++++ gguf-py/gguf/gguf_writer.py | 9 +- gguf-py/tests/test_gguf_writer.py | 23 ++++ src/llama-model-loader.cpp | 188 +++++++++++++++++++++--------- tests/test-arg-parser.cpp | 20 ++++ tools/fit-params/fit-params.cpp | 4 +- tools/llama-bench/llama-bench.cpp | 2 +- tools/server/server-models.cpp | 1 + 16 files changed, 313 insertions(+), 93 deletions(-) create mode 100644 gguf-py/tests/test_gguf_writer.py diff --git a/common/arg.cpp b/common/arg.cpp index c9ac797fe64d..0c84ced44c2c 100644 --- a/common/arg.cpp +++ b/common/arg.cpp @@ -3091,6 +3091,16 @@ common_params_context common_params_parser_init(common_params & params, llama_ex params.model.path = value; } ).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_EXPORT_LORA, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_MODEL")); + add_opt(common_arg( + {"--ple", "--shard"}, "FNAME,...", + "extra GGUF shards for the model, comma separated; may be given more than once\n" + "paths may live anywhere, including on different drives; --ple is an alias for --shard", + [](common_params & params, const std::string & value) { + for (const auto & item : parse_csv_row(value)) { + params.model.shards.push_back(item); + } + } + ).set_examples({LLAMA_EXAMPLE_COMMON}).set_env("LLAMA_ARG_SHARD")); add_opt(common_arg( {"-mu", "--model-url"}, "MODEL_URL", "model download url (default: unused)", diff --git a/common/common.cpp b/common/common.cpp index c142fb4b580b..9d59d6357273 100644 --- a/common/common.cpp +++ b/common/common.cpp @@ -1278,13 +1278,13 @@ void common_fit_apply(common_params & params, llama_model_params & mparams, llam cparams_dft.n_rs_seq = 0; const common_fit_extra_model extra = { - /*.path_model =*/ params_dft.model.path.c_str(), + /*.paths_model =*/ params_dft.model.get_paths(), /*.mparams =*/ &mparams_dft, /*.cparams =*/ &cparams_dft, /*.shares_model =*/ !has_draft, // an MTP context runs on the weights of the main model }; - common_fit_params(params.model.path.c_str(), &mparams, &cparams, + common_fit_params(params.model.get_paths(), &mparams, &cparams, params.tensor_split, params.tensor_buft_overrides.data(), params.fit_params_target.data(), @@ -1300,7 +1300,7 @@ common_init_result::common_init_result(common_params & params, bool model_only) common_fit_apply(params, mparams, cparams); - llama_model * model = llama_model_load_from_file(params.model.path.c_str(), mparams); + llama_model * model = common_model_load(params.model.get_paths(), mparams); if (model == NULL) { return; } @@ -1736,6 +1736,19 @@ struct llama_model_params common_model_params_to_llama(common_params & params) { return mparams; } +struct llama_model * common_model_load(const std::vector & paths, const struct llama_model_params & mparams) { + if (paths.size() == 1) { + return llama_model_load_from_file(paths[0].c_str(), mparams); + } + + std::vector c_paths; + c_paths.reserve(paths.size()); + for (const auto & path : paths) { + c_paths.push_back(path.c_str()); + } + return llama_model_load_from_splits(c_paths.data(), c_paths.size(), mparams); +} + struct llama_context_params common_context_params_to_llama(const common_params & params) { auto cparams = llama_context_default_params(); diff --git a/common/common.h b/common/common.h index 58338f0d7d6a..add3f2c646f5 100644 --- a/common/common.h +++ b/common/common.h @@ -311,6 +311,18 @@ struct common_params_model { std::string hf_file = ""; // HF file std::string docker_repo = ""; // Docker repo + std::vector shards; + + std::vector get_paths() const { + std::vector paths = { path }; + for (const auto & shard : shards) { + if (std::find(paths.begin(), paths.end(), shard) == paths.end()) { + paths.push_back(shard); + } + } + return paths; + } + std::string get_name() const { if (!hf_repo.empty()) { return hf_repo; @@ -1028,6 +1040,8 @@ struct llama_context_params common_context_params_to_llama(const common_params & // apply --fit to mparams and cparams, no-op if params.fit_params is false void common_fit_apply(common_params & params, struct llama_model_params & mparams, struct llama_context_params & cparams); +struct llama_model * common_model_load(const std::vector & paths, const struct llama_model_params & mparams); + // clear LoRA adapters from context, then apply new list of adapters void common_set_adapter_lora(struct llama_context * ctx, std::vector & lora); diff --git a/common/fit.cpp b/common/fit.cpp index 55b457e1060d..92d4c0a6a0aa 100644 --- a/common/fit.cpp +++ b/common/fit.cpp @@ -56,7 +56,7 @@ struct common_fit_log_scope { }; static std::vector common_get_device_memory_data_impl( - const char * path_model, + const std::vector & paths_model, const llama_model_params * mparams, const llama_context_params * cparams, std::vector & devs, @@ -70,7 +70,7 @@ static std::vector common_get_device_memory_data_impl( mparams_copy.no_alloc = true; mparams_copy.load_mode = LLAMA_LOAD_MODE_NONE; - llama_model * model = llama_model_load_from_file(path_model, mparams_copy); + llama_model * model = common_model_load(paths_model, mparams_copy); if (model == nullptr) { throw std::runtime_error("failed to load model"); } @@ -164,7 +164,7 @@ static std::vector common_get_device_memory_data_impl( } std::vector common_fit_measure( - const char * path_model, + const std::vector & paths_model, const llama_model_params * mparams, const llama_context_params * cparams, uint32_t * n_ctx_train, @@ -176,7 +176,7 @@ std::vector common_fit_measure( mparams_copy.no_alloc = true; mparams_copy.load_mode = LLAMA_LOAD_MODE_NONE; - llama_model_ptr model(llama_model_load_from_file(path_model, mparams_copy)); + llama_model_ptr model(common_model_load(paths_model, mparams_copy)); if (!model) { throw std::runtime_error("failed to load model"); } @@ -198,7 +198,7 @@ std::vector common_fit_measure( common_memory_reports common_measure_memory(common_params & params) { const ggml_log_level log_level = params.verbosity >= LOG_LEVEL_DEBUG ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR; - const char * path_model = params.model.path.c_str(); + const std::vector paths_model = params.model.get_paths(); common_memory_reports res; @@ -210,7 +210,7 @@ common_memory_reports common_measure_memory(common_params & params) { auto mparams_fit = mparams; auto cparams_fit = cparams; common_fit_apply(params, mparams_fit, cparams_fit); - res.info_max = common_fit_measure(path_model, &mparams_fit, &cparams_fit, &res.n_ctx_train, &res.n_ctx_max, log_level); + res.info_max = common_fit_measure(paths_model, &mparams_fit, &cparams_fit, &res.n_ctx_train, &res.n_ctx_max, log_level); } // min: the smallest context --fit may pick (--fit only changes n_ctx when it is 0) @@ -228,14 +228,14 @@ common_memory_reports common_measure_memory(common_params & params) { res.info_min = res.info_max; } else { cparams.n_ctx = n_ctx_min; - res.info_min = common_fit_measure(path_model, &mparams, &cparams, nullptr, &res.n_ctx_min, log_level); + res.info_min = common_fit_measure(paths_model, &mparams, &cparams, nullptr, &res.n_ctx_min, log_level); } return res; } common_device_memory_data_vec common_get_device_memory_data( - const char * path_model, + const std::vector & paths_model, const llama_model_params * mparams, const llama_context_params * cparams, std::vector & devs, @@ -244,7 +244,7 @@ common_device_memory_data_vec common_get_device_memory_data( uint32_t & hp_n_expert, ggml_log_level log_level) { std::vector impl = common_get_device_memory_data_impl( - path_model, mparams, cparams, devs, hp_ngl, hp_n_ctx_train, hp_n_expert, log_level); + paths_model, mparams, cparams, devs, hp_ngl, hp_n_ctx_train, hp_n_expert, log_level); common_device_memory_data_vec ret(impl.size()); for (size_t i = 0; i < impl.size(); i++) { @@ -258,7 +258,7 @@ common_device_memory_data_vec common_get_device_memory_data( } static void common_params_fit_impl( - const char * path_model, struct llama_model_params * mparams, struct llama_context_params * cparams, + const std::vector & paths_model, struct llama_model_params * mparams, struct llama_context_params * cparams, float * tensor_split, struct llama_model_tensor_buft_override * tensor_buft_overrides, size_t * margins_s, uint32_t n_ctx_min, const common_fit_extra_model * extra, enum ggml_log_level log_level) { if (mparams->split_mode == LLAMA_SPLIT_MODE_TENSOR) { @@ -302,7 +302,7 @@ static void common_params_fit_impl( dmds_t measured; try { measured = common_get_device_memory_data_impl( - extra->path_model, extra->mparams, extra->cparams, devs_extra, ngl_extra, nct_extra, nex_extra, log_level); + extra->paths_model, extra->mparams, extra->cparams, devs_extra, ngl_extra, nct_extra, nex_extra, log_level); } catch (const std::runtime_error & e) { // the extra model is optional, fit the main model alone rather than giving up LOG_WRN("%s: failed to measure the memory of the extra model, fitting without it: %s\n", __func__, e.what()); @@ -342,7 +342,7 @@ static void common_params_fit_impl( // step 1: get data for default parameters and check whether any changes are necessary in the first place LOG_TRC("%s: getting device memory data for initial parameters:\n", __func__); - dmds_t dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level); + dmds_t dmds_full = common_get_device_memory_data_impl(paths_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level); // saturate instead of overflowing, this also preserves the UINT32_MAX sentinel of n_ctx_min: const uint32_t n_ctx_max = (uint32_t) std::min(uint64_t(hp_nct) * n_streams, UINT32_MAX); @@ -354,7 +354,7 @@ static void common_params_fit_impl( if (n_streams > 1) { LOG_TRC("%s: context size unset and KV cache not unified -> using %" PRIu32 " for %" PRIu32 " sequences:\n", __func__, n_ctx_max, n_streams); - dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level); + dmds_full = common_get_device_memory_data_impl(paths_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level); } } add_extra_memory(dmds_full); @@ -493,7 +493,7 @@ static void common_params_fit_impl( int64_t sum_projected_used_min_ctx = 0; cparams->n_ctx = n_ctx_min_total; - dmds_t dmds_min_ctx = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level); + dmds_t dmds_min_ctx = common_get_device_memory_data_impl(paths_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level); add_extra_memory(dmds_min_ctx); if (nd == 0) { sum_projected_used_min_ctx = dmds_min_ctx.back().mb.total(); @@ -675,7 +675,7 @@ static void common_params_fit_impl( set_ngl_tensor_split_tbo(ngl_per_device, overflow_bufts, mparams_copy); dmds_t dmd_nl = common_get_device_memory_data_impl( - path_model, &mparams_copy, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level); + paths_model, &mparams_copy, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level); add_extra_memory(dmd_nl); LOG_TRC("%s: memory for test allocation by device:\n", func_name); @@ -704,7 +704,7 @@ static void common_params_fit_impl( LOG_TRC("%s: getting device memory data with all MoE tensors moved to system memory:\n", __func__); dmds_t dmds_cpu_moe = common_get_device_memory_data_impl( - path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level); + paths_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level); add_extra_memory(dmds_cpu_moe); for (size_t id = 0; id < nd; id++) { @@ -958,7 +958,7 @@ static void common_params_fit_impl( } enum common_params_fit_status common_fit_params( - const char * path_model, + const std::vector & paths_model, llama_model_params * mparams, llama_context_params * cparams, float * tensor_split, @@ -970,7 +970,7 @@ enum common_params_fit_status common_fit_params( const int64_t t0_us = llama_time_us(); common_params_fit_status status = COMMON_PARAMS_FIT_STATUS_SUCCESS; try { - common_params_fit_impl(path_model, mparams, cparams, tensor_split, tensor_buft_overrides, margins, n_ctx_min, extra, log_level); + common_params_fit_impl(paths_model, mparams, cparams, tensor_split, tensor_buft_overrides, margins, n_ctx_min, extra, log_level); LOG_TRC("%s: successfully fit params to free device memory\n", __func__); } catch (const common_params_fit_exception & e) { LOG_WRN("%s: failed to fit params to free device memory: %s\n", __func__, e.what()); @@ -1180,7 +1180,7 @@ void common_memory_breakdown_print(const struct llama_context * ctx) { } void common_fit_print( - const char * path_model, + const std::vector & paths_model, llama_model_params * mparams, llama_context_params * cparams) { std::vector devs; @@ -1188,7 +1188,7 @@ void common_fit_print( uint32_t hp_nct = 0; // hparams.n_ctx_train uint32_t hp_nex = 0; // hparams.n_expert - auto dmd = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, GGML_LOG_LEVEL_ERROR); + auto dmd = common_get_device_memory_data_impl(paths_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, GGML_LOG_LEVEL_ERROR); GGML_ASSERT(dmd.size() == devs.size() + 1); for (size_t id = 0; id < devs.size(); id++) { diff --git a/common/fit.h b/common/fit.h index 47478bba12e0..27d8195aea00 100644 --- a/common/fit.h +++ b/common/fit.h @@ -16,7 +16,7 @@ enum common_params_fit_status { // - its context follows the context of the main model, so its memory is measured again whenever that context changes // - shares_model tells the fit that the weights are already counted in the main model, as for an MTP context struct common_fit_extra_model { - const char * path_model; + std::vector paths_model; llama_model_params * mparams; llama_context_params * cparams; bool shares_model; @@ -28,19 +28,19 @@ struct common_fit_extra_model { // - only parameters that have the same value as in llama_default_model_params are modified // with the exception of the context size which is modified if and only if equal to 0 common_params_fit_status common_fit_params( - const char * path_model, + const std::vector & paths_model, llama_model_params * mparams, llama_context_params * cparams, float * tensor_split, // writable buffer for tensor split, needs at least llama_max_devices elements llama_model_tensor_buft_override * tensor_buft_overrides, // writable buffer for overrides, needs at least llama_max_tensor_buft_overrides elements size_t * margins, // margins of memory to leave per device in bytes uint32_t n_ctx_min, // minimum context size to set when trying to reduce memory use - const common_fit_extra_model * extra, // model to fit alongside the main one, nullptr if there is none + const common_fit_extra_model * extra, // model to fit alongside the main one, nullptr if there is none ggml_log_level log_level); // minimum log level to print during fitting, lower levels go to debug log // print estimated memory to stdout void common_fit_print( - const char * path_model, + const std::vector & paths_model, llama_model_params * mparams, llama_context_params * cparams); @@ -75,7 +75,7 @@ std::vector common_memory_breakdown_rows(const llam // load model + context with no_alloc and return the memory breakdown, params are used as-is // n_ctx_train and n_ctx are optional outputs; throws on failure std::vector common_fit_measure( - const char * path_model, + const std::vector & paths_model, const llama_model_params * mparams, const llama_context_params * cparams, uint32_t * n_ctx_train, @@ -99,7 +99,7 @@ using common_device_memory_data_vec = std::vector; // Load a model + context with no_alloc and return the per-device memory breakdown. common_device_memory_data_vec common_get_device_memory_data( - const char * path_model, + const std::vector & paths_model, const llama_model_params * mparams, const llama_context_params * cparams, std::vector & devs, diff --git a/common/speculative.cpp b/common/speculative.cpp index db3c3c0fa8ce..2fa4e0766119 100644 --- a/common/speculative.cpp +++ b/common/speculative.cpp @@ -2815,7 +2815,7 @@ common_speculative_init_result::common_speculative_init_result( model_path = params.speculative.draft.mparams.path; LOG_INF("%s: loading draft model '%s'\n", __func__, model_path.c_str()); - llama_model * model_dft = llama_model_load_from_file(model_path.c_str(), mparams); + llama_model * model_dft = common_model_load(params.speculative.draft.mparams.get_paths(), mparams); if (model_dft == NULL) { LOG_ERR("%s: failed to load draft model, '%s'\n", __func__, model_path.c_str()); return; diff --git a/conversion/base.py b/conversion/base.py index ba4f85408265..dfb5b81b3454 100644 --- a/conversion/base.py +++ b/conversion/base.py @@ -132,7 +132,8 @@ def __init__(self, dir_model: Path, ftype: gguf.LlamaFileType, fname_out: Path, target_model_dir: Path | None = None, fuse_gate_up_exps: bool = False, fp8_as_q8: bool = False, - fuse_qkv: bool = False): + fuse_qkv: bool = False, + ple_only: bool = False, no_ple: bool = False): if type(self) is ModelBase or \ type(self) is TextModel or \ type(self) is MmprojModel: @@ -141,6 +142,11 @@ def __init__(self, dir_model: Path, ftype: gguf.LlamaFileType, fname_out: Path, if self.is_mistral_format and not _mistral_common_installed: raise ImportError(_mistral_import_error_msg) + if ple_only and no_ple: + raise ValueError("ple_only and no_ple are mutually exclusive") + self.ple_only = ple_only + self.no_ple = no_ple + self._has_ple = False self.dir_model = dir_model self.ftype = ftype self.fname_out = fname_out @@ -198,7 +204,8 @@ def __init__(self, dir_model: Path, ftype: gguf.LlamaFileType, fname_out: Path, # Configure GGUF Writer self.gguf_writer = gguf.GGUFWriter(path=None, arch=gguf.MODEL_ARCH_NAMES[self.model_arch], endianess=self.endianess, use_temp_file=self.use_temp_file, - split_max_tensors=split_max_tensors, split_max_size=split_max_size, dry_run=dry_run, small_first_shard=small_first_shard) + split_max_tensors=split_max_tensors, split_max_size=split_max_size, dry_run=dry_run, small_first_shard=small_first_shard, + tensor_filter=self._include_ple_tensor if ple_only or no_ple else None) # Mistral specific self.disable_mistral_community_chat_template = disable_mistral_community_chat_template @@ -1127,6 +1134,11 @@ def _flush_nvfp4_experts(self, key, expert_blocks, expert_scales, expert_input_s del experts, merged + def _include_ple_tensor(self, name: str) -> bool: + is_ple = name == gguf.TENSOR_NAMES[gguf.MODEL_TENSOR.PER_LAYER_TOKEN_EMBD] + ".weight" + self._has_ple |= is_ple + return is_ple if self.ple_only else not is_ple + def prepare_tensors(self): # A directory that has a config, an index and a tokenizer but no weight files - what a disk purge leaves # behind - would otherwise convert "successfully" into a metadata-only GGUF: index_tensors() picks the @@ -1249,6 +1261,8 @@ def load(): break for new_name, data_torch in (self.modify_tensors(data_torch, name, bid)): + if (self.ple_only or self.no_ple) and not self._include_ple_tensor(new_name): + continue # TODO: why do we squeeze here? # data = data_torch.squeeze().numpy() data = data_torch.numpy() @@ -1410,6 +1424,8 @@ def write_vocab(self): def write(self): self.prepare_tensors() + if (self.ple_only or self.no_ple) and not self._has_ple: + raise ValueError("model has no per-layer embedding table to export separately") self.prepare_metadata(vocab_only=False) self.gguf_writer.write_header_to_file(path=self.fname_out) self.gguf_writer.write_kv_data_to_file() @@ -1602,7 +1618,8 @@ def prepare_metadata(self, vocab_only: bool): fname_default: str = gguf.naming_convention(self.metadata.name, self.metadata.basename, self.metadata.finetune, self.metadata.version, size_label=None, output_type=None, model_type="vocab") # Use the default filename - self.fname_out = self.fname_out / f"{fname_default}.gguf" + prefix = "ple-" if self.ple_only else "no-ple-" if self.no_ple else "" + self.fname_out = self.fname_out / f"{prefix}{fname_default}.gguf" else: # Output path is a custom defined templated filename # Note: `not is_dir()` is used because `.is_file()` will not detect @@ -1611,8 +1628,9 @@ def prepare_metadata(self, vocab_only: bool): # Process templated file name with the output ftype, useful with the "auto" ftype self.fname_out = self.fname_out.parent / gguf.fill_templated_filename(self.fname_out.name, output_type) - logger.info("Set model tokenizer") - self.set_vocab() + if not self.ple_only: + logger.info("Set model tokenizer") + self.set_vocab() def set_gguf_parameters(self): self.gguf_writer.add_block_count(self.block_count) diff --git a/convert_hf_to_gguf.py b/convert_hf_to_gguf.py index 55aa97c953df..bf39a3f6ee01 100755 --- a/convert_hf_to_gguf.py +++ b/convert_hf_to_gguf.py @@ -125,6 +125,15 @@ def parse_args() -> argparse.Namespace: "--no-nextn", "--no-mtp", dest="no_mtp", action="store_true", help="Exclude NextN speculative draft tensors from the converted GGUF. Pair with --mtp or --dspark on a second run to publish target and draft as two files.", ) + ple_group = parser.add_mutually_exclusive_group() + ple_group.add_argument( + "--ple-only", action="store_true", + help="Export only the per-layer embedding table as a separate GGUF for use with --shard (alias: --ple).", + ) + ple_group.add_argument( + "--no-ple", action="store_true", + help="Exclude the per-layer embedding table while retaining its metadata. Load the table separately with --shard (alias: --ple).", + ) parser.add_argument( "--dspark", action="store_true", help="Export only the DeepSeek-V4 DSpark draft tensors as a separate GGUF.", @@ -177,6 +186,10 @@ def parse_args() -> argparse.Namespace: args = parser.parse_args() if not args.print_supported_models and args.model is None: parser.error("the following arguments are required: model") + if (args.ple_only or args.no_ple) and (args.vocab_only or args.mmproj or args.mtp or args.dspark): + parser.error("--ple-only / --no-ple cannot be combined with --vocab-only, --mmproj, --mtp, or --dspark") + if args.ple_only and args.no_tensor_first_split: + parser.error("--ple-only cannot be combined with --no-tensor-first-split") return args @@ -303,6 +316,7 @@ def main() -> None: fuse_gate_up_exps=args.fuse_gate_up_exps, fp8_as_q8=args.fp8_as_q8, fuse_qkv=args.fuse_qkv, + ple_only=args.ple_only, no_ple=args.no_ple, ) if args.vocab_only: diff --git a/docs/models.md b/docs/models.md index eee5952634be..f466d76e50f4 100644 --- a/docs/models.md +++ b/docs/models.md @@ -24,3 +24,23 @@ The Hugging Face platform provides a variety of online tools for converting, qua - Use the [GGUF-my-LoRA space](https://huggingface.co/spaces/ggml-org/gguf-my-lora) to convert LoRA adapters to GGUF format (more info: https://github.com/ggml-org/llama.cpp/discussions/10123) - Use the [GGUF-editor space](https://huggingface.co/spaces/CISCai/gguf-editor) to edit GGUF meta data in the browser (more info: https://github.com/ggml-org/llama.cpp/discussions/9268) - Use the [Inference Endpoints](https://ui.endpoints.huggingface.co/) to directly host `llama.cpp` in the cloud (more info: https://github.com/ggml-org/llama.cpp/discussions/9669) + +## Exporting per-layer embeddings separately + +For models with a per-layer embedding (PLE) table, export the backbone and table from the same checkpoint: + +```sh +python3 convert_hf_to_gguf.py /path/to/model --no-ple --outfile model-no-ple.gguf +python3 convert_hf_to_gguf.py /path/to/model --ple-only --outfile model-ple.gguf +llama-cli -m model-no-ple.gguf --shard model-ple.gguf +``` + +`--no-ple` excludes only `per_layer_token_embd.weight` and keeps the PLE metadata and projection weights. `--ple-only` exports only that table, without a tokenizer. Each export supports `--outtype`, so the table and backbone can use different precisions. Default output names start with `no-ple-` and `ple-` respectively. + +These modes are mutually exclusive and require a model with a PLE table, such as Qwen3.8 Flash Next or Gemma 4. They cannot be combined with vocabulary, multimodal projector, MTP-only, or DSpark-only export. `--no-ple` can be combined with `--no-mtp`. + +`--ple` is an alias for `--shard`. Extra shards can be comma separated or supplied with repeated options, for example `-m backbone.gguf --shard /fast/ple.gguf,/other/token-embd.gguf`. Paths may be on different drives. Tensor names must be unique across all files, except for the PLE case below. + +A numbered backbone is auto-discovered and checked as usual; explicitly supplied paths outside that numbered set are appended as extra shards. Their split numbering is ignored, and the loader uses the actual combined tensor count. Paths already in the discovered set are loaded once. If the main filename does not follow the numbered naming scheme, an explicit backbone list must still match its declared split count and order. + +If an extra shard holds only `per_layer_token_embd.weight`, it replaces the PLE table of the model. If the model already has the table, the extra shard overrides it. A backbone split that holds only that tensor is skipped, and it can be absent from disk. This lets one PLE file be shared across quants and releases, for example `-m model-Q4_K_M-00001-of-00003.gguf --shard /fast/ple-Q8_0.gguf`. diff --git a/gguf-py/gguf/gguf_writer.py b/gguf-py/gguf/gguf_writer.py index c32cceae7724..882cf4671311 100644 --- a/gguf-py/gguf/gguf_writer.py +++ b/gguf-py/gguf/gguf_writer.py @@ -11,7 +11,7 @@ from math import prod from pathlib import Path from io import BufferedWriter -from typing import IO, Any, Sequence, Mapping +from typing import IO, Any, Callable, Sequence, Mapping from string import ascii_letters, digits import numpy as np @@ -86,7 +86,8 @@ class GGUFWriter: def __init__( self, path: os.PathLike[str] | str | None, arch: str, use_temp_file: bool = False, endianess: GGUFEndian = GGUFEndian.LITTLE, - split_max_tensors: int = 0, split_max_size: int = 0, dry_run: bool = False, small_first_shard: bool = False + split_max_tensors: int = 0, split_max_size: int = 0, dry_run: bool = False, small_first_shard: bool = False, + tensor_filter: Callable[[str], bool] | None = None, ): self.fout = None self.path = Path(path) if path else None @@ -101,6 +102,7 @@ def __init__( self.split_max_size = split_max_size self.dry_run = dry_run self.small_first_shard = small_first_shard + self.tensor_filter = tensor_filter logger.info("gguf: This GGUF file is for {0} Endian only".format( "Big" if self.endianess == GGUFEndian.BIG else "Little", )) @@ -381,6 +383,9 @@ def add_tensor( self, name: str, tensor: np.ndarray[Any, Any], raw_shape: Sequence[int] | None = None, raw_dtype: GGMLQuantizationType | None = None, tensor_endianess: GGUFEndian | None = None ) -> None: + if self.tensor_filter is not None and not self.tensor_filter(name): + return + # if tensor endianness is not passed, assume it's native to system if tensor_endianess is None: tensor_endianess = GGUFEndian.BIG if sys.byteorder == 'big' else GGUFEndian.LITTLE diff --git a/gguf-py/tests/test_gguf_writer.py b/gguf-py/tests/test_gguf_writer.py new file mode 100644 index 000000000000..606b2c5f9bca --- /dev/null +++ b/gguf-py/tests/test_gguf_writer.py @@ -0,0 +1,23 @@ +import numpy as np +import pytest + +from gguf import GGUFReader, GGUFWriter + + +@pytest.mark.parametrize("use_temp_file", [False, True]) +def test_tensor_filter_preserves_data_offsets(tmp_path, use_temp_file): + path = tmp_path / "filtered.gguf" + data = np.arange(32, dtype=np.float32).reshape(1, 32) + writer = GGUFWriter(path, "qwen4exp", use_temp_file=use_temp_file, + tensor_filter=lambda name: name != "excluded") + writer.add_tensor("first", data) + writer.add_tensor("excluded", data + 100) + writer.add_tensor("last", data + 200) + writer.write_header_to_file() + writer.write_kv_data_to_file() + writer.write_tensors_to_file() + writer.close() + tensors = GGUFReader(path).tensors + assert [t.name for t in tensors] == ["first", "last"] + np.testing.assert_array_equal(tensors[0].data, data) + np.testing.assert_array_equal(tensors[1].data, data + 200) diff --git a/src/llama-model-loader.cpp b/src/llama-model-loader.cpp index e0d29ada6ec7..a7427b440ef5 100644 --- a/src/llama-model-loader.cpp +++ b/src/llama-model-loader.cpp @@ -598,85 +598,167 @@ llama_model_loader::llama_model_loader( n_bytes += ggml_nbytes(cur); weights_map.emplace(tensor_name, llama_tensor_weight(files.back().get(), 0, metadata, cur)); } + auto paths = splits; uint16_t n_split = 0; + size_t n_backbone = 1; + const std::string kv_split_no = llm_kv(LLM_KV_SPLIT_NO); get_key(llm_kv(LLM_KV_SPLIT_COUNT), n_split, false); - // Load additional GGML contexts if (n_split > 1) { - // make sure the main file is loaded first uint16_t idx = 0; - const std::string kv_split_no = llm_kv(LLM_KV_SPLIT_NO); get_key(kv_split_no, idx); if (idx != 0) { throw std::runtime_error(format("illegal split file idx: %d (file: %s), model must be loaded with the first split", idx, fname.c_str())); } - // generate list of splits if needed - if (splits.empty()) { - splits = llama_get_list_splits(fname, idx, n_split); + std::vector prefix(llama_path_max(), 0); + if (paths.empty() || llama_split_prefix(prefix.data(), prefix.size(), fname.c_str(), idx, n_split)) { + auto backbone = llama_get_list_splits(fname, idx, n_split); + auto file_name = [](const std::string & path) { + return path.substr(path.find_last_of("/\\") + 1); + }; + std::vector extras; + for (const auto & path : paths) { + auto it = std::find_if(backbone.begin(), backbone.end(), [&](const std::string & b) { + return file_name(b) == file_name(path); + }); + if (it != backbone.end()) { + *it = path; // given path wins, splits can be in different dirs + } else { + extras.push_back(path); + } + } + paths = std::move(backbone); + paths.insert(paths.end(), extras.begin(), extras.end()); + } else if (paths.size() != n_split) { + throw std::runtime_error(format("invalid split count, given: %zu splits, but expected %d", paths.size(), n_split)); } + n_backbone = n_split; + } else if (paths.empty()) { + paths.push_back(fname); + } - // in case user give a custom list of splits, check if it matches the expected number - if (n_split != (uint16_t)splits.size()) { - throw std::runtime_error(format("invalid split count, given: %zu splits, but expected %d", splits.size(), n_split)); - } + if (paths.size() > UINT16_MAX) { + throw std::runtime_error("too many model files"); + } + const bool has_extras = paths.size() > n_backbone; - if (trace > 0) { - LLAMA_LOG_INFO("%s: loading additional %d GGUFs\n", __func__, n_split); + // special case for PLE-only GGUFs: no need to follow the exact split numbering (allow flexibility) + // ref: https://github.com/ggml-org/llama.cpp/pull/31185 + auto is_ple_only = [](const gguf_context * ctx) { + return gguf_get_n_tensors(ctx) == 1 && std::strcmp(gguf_get_tensor_name(ctx, 0), "per_layer_token_embd.weight") == 0; + }; + bool skip_shard_order_validation = false; + for (size_t idx = n_backbone; idx < paths.size(); idx++) { + struct gguf_init_params probe_params = { + /*.no_alloc = */ true, + /*.ctx = */ nullptr, + }; + gguf_context_ptr probe { gguf_init_from_file(paths[idx].c_str(), probe_params) }; + skip_shard_order_validation |= probe && is_ple_only(probe.get()); + } + + size_t n_skipped = 0; + size_t n_backbone_files = 1; + int n_backbone_tensors = gguf_get_n_tensors(metadata); + for (size_t idx = 1; idx < paths.size(); idx++) { + const char * fname_split = paths[idx].c_str(); + if (skip_shard_order_validation && idx < n_backbone) { + FILE * f = ggml_fopen(fname_split, "rb"); + if (f == nullptr) { + LLAMA_LOG_INFO("%s: GGUF split %s not found, assuming it only holds the per-layer embedding table\n", __func__, fname_split); + n_skipped++; + continue; + } + fclose(f); + } + ctx = nullptr; // still owned by contexts, not set by gguf on failure + struct gguf_init_params split_params = { + /*.no_alloc = */ true, + /*.ctx = */ &ctx, + }; + gguf_context_ptr ctx_gguf { gguf_init_from_file(fname_split, split_params) }; + ggml_context_ptr ctx_ptr(ctx); + if (!ctx_gguf) { + throw std::runtime_error(format("%s: failed to load GGUF split from %s", __func__, fname_split)); } - // load other splits - for (idx = 1; idx < n_split; idx++) { - const char * fname_split = splits[idx].c_str(); - - struct gguf_init_params split_params = { - /*.no_alloc = */ true, - /*.ctx = */ &ctx, - }; - gguf_context_ptr ctx_gguf { gguf_init_from_file(fname_split, split_params) }; - if (!ctx_gguf) { - throw std::runtime_error(format("%s: failed to load GGUF split from %s", __func__, fname_split)); + bool allow_override_tensor = false; + if (idx < n_backbone) { + const int kid = gguf_find_key(ctx_gguf.get(), kv_split_no.c_str()); + if (kid < 0) { + throw std::runtime_error(format("missing key %s in GGUF split %s", kv_split_no.c_str(), fname_split)); } - - // check idx - { - const int kid = gguf_find_key(ctx_gguf.get(), kv_split_no.c_str()); - if (kid < 0) { - throw std::runtime_error(format("missing key %s in GGUF split %s", kv_split_no.c_str(), fname_split)); - } - int idx_gguf = gguf_get_val_u16(ctx_gguf.get(), kid); - if (idx_gguf != idx) { - throw std::runtime_error(format("invalid split file idx: %d (file: %s), expected %d", idx_gguf, fname_split, idx)); - } + if (gguf_get_kv_type(ctx_gguf.get(), kid) != GGUF_TYPE_UINT16) { + throw std::runtime_error(format("invalid type for %s in GGUF split %s", kv_split_no.c_str(), fname_split)); + } + const uint16_t idx_gguf = gguf_get_val_u16(ctx_gguf.get(), kid); + if (idx_gguf != idx) { + throw std::runtime_error(format("invalid split file idx: %d (file: %s), expected %zu", idx_gguf, fname_split, idx)); } + if (skip_shard_order_validation && is_ple_only(ctx_gguf.get())) { + LLAMA_LOG_INFO("%s: skipping GGUF split %s, per-layer embedding table given separately\n", __func__, fname_split); + n_skipped++; + continue; + } + n_backbone_tensors += gguf_get_n_tensors(ctx_gguf.get()); + } else { + // an external PLE table replaces the one in the model, if any + allow_override_tensor = is_ple_only(ctx_gguf.get()); + const bool numbered = gguf_find_key(ctx_gguf.get(), kv_split_no.c_str()) >= 0 || + gguf_find_key(ctx_gguf.get(), llm_kv(LLM_KV_SPLIT_COUNT).c_str()) >= 0 || + gguf_find_key(ctx_gguf.get(), llm_kv(LLM_KV_SPLIT_TENSORS_COUNT).c_str()) >= 0; + LLAMA_LOG_INFO("%s: appended extra GGUF shard %s%s\n", __func__, fname_split, + numbered ? " (declared split numbering ignored)" : ""); + } - files.emplace_back(new llama_file(fname_split, "rb", use_direct_io)); - contexts.emplace_back(ctx); + files.emplace_back(new llama_file(fname_split, "rb", use_direct_io)); + contexts.emplace_back(std::move(ctx_ptr)); + if (idx < n_backbone) { + n_backbone_files = files.size(); + } - // Save tensors data offset info of the shard. - for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) { - std::string tensor_name = std::string(cur->name); - // make sure there is no duplicated tensor names - if (weights_map.find(tensor_name) != weights_map.end()) { + // skipped splits are not in files, so idx != file index + const uint16_t file_idx = files.size() - 1; + for (ggml_tensor * cur = ggml_get_first_tensor(ctx); cur; cur = ggml_get_next_tensor(ctx, cur)) { + const std::string tensor_name = cur->name; + auto it = weights_map.find(tensor_name); + if (it != weights_map.end()) { + // only override tensors of the model, not of another extra shard + if (!allow_override_tensor || it->second.idx >= n_backbone_files) { throw std::runtime_error(format("invalid model: tensor '%s' is duplicated", ggml_get_name(cur))); } - n_elements += ggml_nelements(cur); - n_bytes += ggml_nbytes(cur); - weights_map.emplace(tensor_name, llama_tensor_weight(files.back().get(), idx, ctx_gguf.get(), cur)); + LLAMA_LOG_INFO("%s: tensor '%s' overridden by %s\n", __func__, tensor_name.c_str(), fname_split); + n_elements -= ggml_nelements(it->second.tensor); + n_bytes -= ggml_nbytes(it->second.tensor); + weights_map.erase(it); } + n_elements += ggml_nelements(cur); + n_bytes += ggml_nbytes(cur); + weights_map.emplace(tensor_name, llama_tensor_weight(files.back().get(), file_idx, ctx_gguf.get(), cur)); } + } - get_key(llm_kv(LLM_KV_SPLIT_TENSORS_COUNT), n_tensors); - - // sanity check - { - const int n_tensors_loaded = (int) weights_map.size(); - if (n_tensors != n_tensors_loaded) { - throw std::runtime_error(format("corrupted model: %d tensors expected but %d found", n_tensors, n_tensors_loaded)); + if (n_skipped > 0) { + // the external table can only replace one tensor of the numbered set + int declared_tensors = 0; + get_key(llm_kv(LLM_KV_SPLIT_TENSORS_COUNT), declared_tensors); + if (n_skipped != 1 || n_backbone_tensors + 1 != declared_tensors) { + throw std::runtime_error(format("incomplete model: %zu split(s) skipped, %d of %d tensors found in the split files", n_skipped, n_backbone_tensors, declared_tensors)); + } + } + if (n_split > 1 || has_extras) { + int declared_tensors = gguf_get_n_tensors(metadata); + get_key(llm_kv(LLM_KV_SPLIT_TENSORS_COUNT), declared_tensors, n_split > 1 && !has_extras); + if (declared_tensors != (int) weights_map.size()) { + if (n_split > 1 && !has_extras) { + throw std::runtime_error(format("corrupted model: %d tensors expected but %zu found", declared_tensors, weights_map.size())); } + LLAMA_LOG_INFO("%s: using %zu registered tensors instead of %d declared tensors\n", __func__, weights_map.size(), declared_tensors); } - - LLAMA_LOG_INFO("%s: additional %d GGUFs metadata loaded.\n", __func__, n_split - 1); + } + if (files.size() > 1) { + LLAMA_LOG_INFO("%s: additional %zu GGUFs metadata loaded.\n", __func__, files.size() - 1); } } else if (file != nullptr) { struct ggml_context * ctx = NULL; diff --git a/tests/test-arg-parser.cpp b/tests/test-arg-parser.cpp index 543da2aed5d0..537c333949b1 100644 --- a/tests/test-arg-parser.cpp +++ b/tests/test-arg-parser.cpp @@ -245,6 +245,26 @@ static void test(void) { assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON)); assert(params.model.path == "model_file.gguf"); + { + auto parse_shards = [&list_str_to_char](std::vector args) { + common_params shard_params; + args.insert(args.begin(), {"binary_name", "-m", "model_file.gguf"}); + assert(common_params_parse(args.size(), list_str_to_char(args).data(), shard_params, LLAMA_EXAMPLE_COMMON)); + auto paths = shard_params.model.get_paths(); + assert(paths.front() == "model_file.gguf"); + return std::vector(paths.begin() + 1, paths.end()); + }; + assert(parse_shards({"--shard", "a.gguf"}) == std::vector({"a.gguf"})); + const std::vector two = {"a.gguf", "b.gguf"}; + assert(parse_shards({"--shard", "a.gguf", "--shard", "b.gguf"}) == two); + assert(parse_shards({"--shard", "a.gguf,b.gguf"}) == two); + const std::vector three = {"a.gguf", "b.gguf", "c.gguf"}; + assert(parse_shards({"--shard", "a.gguf,b.gguf", "--shard", "c.gguf"}) == three); + assert(parse_shards({"--shard", "a.gguf", "--shard", "b.gguf", "--shard", "c.gguf"}) == three); + assert(parse_shards({"--ple", "p.gguf"}) == parse_shards({"--shard", "p.gguf"})); + assert(parse_shards({"--shard", "model_file.gguf,a.gguf,b.gguf,a.gguf"}) == two); + } + argv = {"binary_name", "-t", "1234"}; assert(true == common_params_parse(argv.size(), list_str_to_char(argv).data(), params, LLAMA_EXAMPLE_COMMON)); assert(params.cpuparams.n_threads == 1234); diff --git a/tools/fit-params/fit-params.cpp b/tools/fit-params/fit-params.cpp index 3e78c89290a7..3dc46069ca9a 100644 --- a/tools/fit-params/fit-params.cpp +++ b/tools/fit-params/fit-params.cpp @@ -31,7 +31,7 @@ int llama_fit_params(int argc, char ** argv) { auto cparams = common_context_params_to_llama(params); if (!params.fit_params_print) { - const common_params_fit_status status = common_fit_params(params.model.path.c_str(), &mparams, &cparams, + const common_params_fit_status status = common_fit_params(params.model.get_paths(), &mparams, &cparams, params.tensor_split, params.tensor_buft_overrides.data(), params.fit_params_target.data(), params.fit_params_min_ctx, nullptr, params.verbosity >= LOG_LEVEL_DEBUG ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR); @@ -71,7 +71,7 @@ int llama_fit_params(int argc, char ** argv) { LOG_INF("%s: printing estimated memory in MiB to stdout (device, model, context, compute) ...\n", __func__); common_log_flush(common_log_main()); - common_fit_print(params.model.path.c_str(), &mparams, &cparams); + common_fit_print(params.model.get_paths(), &mparams, &cparams); } return 0; diff --git a/tools/llama-bench/llama-bench.cpp b/tools/llama-bench/llama-bench.cpp index 13bdbb991214..4a63ad1f8d50 100644 --- a/tools/llama-bench/llama-bench.cpp +++ b/tools/llama-bench/llama-bench.cpp @@ -2368,7 +2368,7 @@ int llama_bench(int argc, char ** argv) { uint32_t n_ctx_needed = std::max(inst.n_prompt + inst.n_gen + inst.n_depth, inst.fit_min_ctx); cparams.n_ctx = std::max(cparams.n_ctx, n_ctx_needed); - common_fit_params(inst.model.c_str(), &mparams, &cparams, + common_fit_params({ inst.model }, &mparams, &cparams, fit_tensor_split.data(), fit_overrides.data(), margins.data(), diff --git a/tools/server/server-models.cpp b/tools/server/server-models.cpp index 9f583b738107..6f61167b65f4 100644 --- a/tools/server/server-models.cpp +++ b/tools/server/server-models.cpp @@ -279,6 +279,7 @@ static void unset_reserved_args(common_preset & preset, bool unset_model_args) { preset.unset_option("LLAMA_ARG_CONNECT_CODE"); if (unset_model_args) { preset.unset_option("LLAMA_ARG_MODEL"); + preset.unset_option("LLAMA_ARG_SHARD"); preset.unset_option("LLAMA_ARG_MMPROJ"); preset.unset_option("LLAMA_ARG_ALIAS"); preset.unset_option("LLAMA_ARG_HF_REPO");