Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
10 changes: 10 additions & 0 deletions common/arg.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -3091,6 +3091,16 @@ common_params_context common_params_parser_init(common_params & params, llama_ex
params.model.path = value;
}
).set_examples({LLAMA_EXAMPLE_COMMON, LLAMA_EXAMPLE_EXPORT_LORA, LLAMA_EXAMPLE_DOWNLOAD, LLAMA_EXAMPLE_TOKENIZE}).set_env("LLAMA_ARG_MODEL"));
add_opt(common_arg(
{"--ple", "--shard"}, "FNAME,...",
"extra GGUF shards for the model, comma separated; may be given more than once\n"
"paths may live anywhere, including on different drives; --ple is an alias for --shard",
[](common_params & params, const std::string & value) {
for (const auto & item : parse_csv_row(value)) {
params.model.shards.push_back(item);
}
}
).set_examples({LLAMA_EXAMPLE_COMMON}).set_env("LLAMA_ARG_SHARD"));
add_opt(common_arg(
{"-mu", "--model-url"}, "MODEL_URL",
"model download url (default: unused)",
Expand Down
19 changes: 16 additions & 3 deletions common/common.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -1278,13 +1278,13 @@ void common_fit_apply(common_params & params, llama_model_params & mparams, llam
cparams_dft.n_rs_seq = 0;

const common_fit_extra_model extra = {
/*.path_model =*/ params_dft.model.path.c_str(),
/*.paths_model =*/ params_dft.model.get_paths(),
/*.mparams =*/ &mparams_dft,
/*.cparams =*/ &cparams_dft,
/*.shares_model =*/ !has_draft, // an MTP context runs on the weights of the main model
};

common_fit_params(params.model.path.c_str(), &mparams, &cparams,
common_fit_params(params.model.get_paths(), &mparams, &cparams,
params.tensor_split,
params.tensor_buft_overrides.data(),
params.fit_params_target.data(),
Expand All @@ -1300,7 +1300,7 @@ common_init_result::common_init_result(common_params & params, bool model_only)

common_fit_apply(params, mparams, cparams);

llama_model * model = llama_model_load_from_file(params.model.path.c_str(), mparams);
llama_model * model = common_model_load(params.model.get_paths(), mparams);
if (model == NULL) {
return;
}
Expand Down Expand Up @@ -1736,6 +1736,19 @@ struct llama_model_params common_model_params_to_llama(common_params & params) {
return mparams;
}

struct llama_model * common_model_load(const std::vector<std::string> & paths, const struct llama_model_params & mparams) {
if (paths.size() == 1) {
return llama_model_load_from_file(paths[0].c_str(), mparams);
}

std::vector<const char *> c_paths;
c_paths.reserve(paths.size());
for (const auto & path : paths) {
c_paths.push_back(path.c_str());
}
return llama_model_load_from_splits(c_paths.data(), c_paths.size(), mparams);
}

struct llama_context_params common_context_params_to_llama(const common_params & params) {
auto cparams = llama_context_default_params();

Expand Down
14 changes: 14 additions & 0 deletions common/common.h
Original file line number Diff line number Diff line change
Expand Up @@ -311,6 +311,18 @@ struct common_params_model {
std::string hf_file = ""; // HF file
std::string docker_repo = ""; // Docker repo

std::vector<std::string> shards;

std::vector<std::string> get_paths() const {
std::vector<std::string> paths = { path };
for (const auto & shard : shards) {
if (std::find(paths.begin(), paths.end(), shard) == paths.end()) {
paths.push_back(shard);
}
}
return paths;
}

std::string get_name() const {
if (!hf_repo.empty()) {
return hf_repo;
Expand Down Expand Up @@ -1028,6 +1040,8 @@ struct llama_context_params common_context_params_to_llama(const common_params &
// apply --fit to mparams and cparams, no-op if params.fit_params is false
void common_fit_apply(common_params & params, struct llama_model_params & mparams, struct llama_context_params & cparams);

struct llama_model * common_model_load(const std::vector<std::string> & paths, const struct llama_model_params & mparams);

// clear LoRA adapters from context, then apply new list of adapters
void common_set_adapter_lora(struct llama_context * ctx, std::vector<common_adapter_lora_info> & lora);

Expand Down
40 changes: 20 additions & 20 deletions common/fit.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -56,7 +56,7 @@ struct common_fit_log_scope {
};

static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
const char * path_model,
const std::vector<std::string> & paths_model,
const llama_model_params * mparams,
const llama_context_params * cparams,
std::vector<ggml_backend_dev_t> & devs,
Expand All @@ -70,7 +70,7 @@ static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
mparams_copy.no_alloc = true;
mparams_copy.load_mode = LLAMA_LOAD_MODE_NONE;

llama_model * model = llama_model_load_from_file(path_model, mparams_copy);
llama_model * model = common_model_load(paths_model, mparams_copy);
if (model == nullptr) {
throw std::runtime_error("failed to load model");
}
Expand Down Expand Up @@ -164,7 +164,7 @@ static std::vector<llama_device_memory_data> common_get_device_memory_data_impl(
}

std::vector<common_memory_breakdown_row> common_fit_measure(
const char * path_model,
const std::vector<std::string> & paths_model,
const llama_model_params * mparams,
const llama_context_params * cparams,
uint32_t * n_ctx_train,
Expand All @@ -176,7 +176,7 @@ std::vector<common_memory_breakdown_row> common_fit_measure(
mparams_copy.no_alloc = true;
mparams_copy.load_mode = LLAMA_LOAD_MODE_NONE;

llama_model_ptr model(llama_model_load_from_file(path_model, mparams_copy));
llama_model_ptr model(common_model_load(paths_model, mparams_copy));
if (!model) {
throw std::runtime_error("failed to load model");
}
Expand All @@ -198,7 +198,7 @@ std::vector<common_memory_breakdown_row> common_fit_measure(

common_memory_reports common_measure_memory(common_params & params) {
const ggml_log_level log_level = params.verbosity >= LOG_LEVEL_DEBUG ? GGML_LOG_LEVEL_DEBUG : GGML_LOG_LEVEL_ERROR;
const char * path_model = params.model.path.c_str();
const std::vector<std::string> paths_model = params.model.get_paths();

common_memory_reports res;

Expand All @@ -210,7 +210,7 @@ common_memory_reports common_measure_memory(common_params & params) {
auto mparams_fit = mparams;
auto cparams_fit = cparams;
common_fit_apply(params, mparams_fit, cparams_fit);
res.info_max = common_fit_measure(path_model, &mparams_fit, &cparams_fit, &res.n_ctx_train, &res.n_ctx_max, log_level);
res.info_max = common_fit_measure(paths_model, &mparams_fit, &cparams_fit, &res.n_ctx_train, &res.n_ctx_max, log_level);
}

// min: the smallest context --fit may pick (--fit only changes n_ctx when it is 0)
Expand All @@ -228,14 +228,14 @@ common_memory_reports common_measure_memory(common_params & params) {
res.info_min = res.info_max;
} else {
cparams.n_ctx = n_ctx_min;
res.info_min = common_fit_measure(path_model, &mparams, &cparams, nullptr, &res.n_ctx_min, log_level);
res.info_min = common_fit_measure(paths_model, &mparams, &cparams, nullptr, &res.n_ctx_min, log_level);
}

return res;
}

common_device_memory_data_vec common_get_device_memory_data(
const char * path_model,
const std::vector<std::string> & paths_model,
const llama_model_params * mparams,
const llama_context_params * cparams,
std::vector<ggml_backend_dev_t> & devs,
Expand All @@ -244,7 +244,7 @@ common_device_memory_data_vec common_get_device_memory_data(
uint32_t & hp_n_expert,
ggml_log_level log_level) {
std::vector<llama_device_memory_data> impl = common_get_device_memory_data_impl(
path_model, mparams, cparams, devs, hp_ngl, hp_n_ctx_train, hp_n_expert, log_level);
paths_model, mparams, cparams, devs, hp_ngl, hp_n_ctx_train, hp_n_expert, log_level);

common_device_memory_data_vec ret(impl.size());
for (size_t i = 0; i < impl.size(); i++) {
Expand All @@ -258,7 +258,7 @@ common_device_memory_data_vec common_get_device_memory_data(
}

static void common_params_fit_impl(
const char * path_model, struct llama_model_params * mparams, struct llama_context_params * cparams,
const std::vector<std::string> & paths_model, struct llama_model_params * mparams, struct llama_context_params * cparams,
float * tensor_split, struct llama_model_tensor_buft_override * tensor_buft_overrides,
size_t * margins_s, uint32_t n_ctx_min, const common_fit_extra_model * extra, enum ggml_log_level log_level) {
if (mparams->split_mode == LLAMA_SPLIT_MODE_TENSOR) {
Expand Down Expand Up @@ -302,7 +302,7 @@ static void common_params_fit_impl(
dmds_t measured;
try {
measured = common_get_device_memory_data_impl(
extra->path_model, extra->mparams, extra->cparams, devs_extra, ngl_extra, nct_extra, nex_extra, log_level);
extra->paths_model, extra->mparams, extra->cparams, devs_extra, ngl_extra, nct_extra, nex_extra, log_level);
} catch (const std::runtime_error & e) {
// the extra model is optional, fit the main model alone rather than giving up
LOG_WRN("%s: failed to measure the memory of the extra model, fitting without it: %s\n", __func__, e.what());
Expand Down Expand Up @@ -342,7 +342,7 @@ static void common_params_fit_impl(
// step 1: get data for default parameters and check whether any changes are necessary in the first place

LOG_TRC("%s: getting device memory data for initial parameters:\n", __func__);
dmds_t dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
dmds_t dmds_full = common_get_device_memory_data_impl(paths_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);

// saturate instead of overflowing, this also preserves the UINT32_MAX sentinel of n_ctx_min:
const uint32_t n_ctx_max = (uint32_t) std::min<uint64_t>(uint64_t(hp_nct) * n_streams, UINT32_MAX);
Expand All @@ -354,7 +354,7 @@ static void common_params_fit_impl(
if (n_streams > 1) {
LOG_TRC("%s: context size unset and KV cache not unified -> using %" PRIu32 " for %" PRIu32 " sequences:\n",
__func__, n_ctx_max, n_streams);
dmds_full = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
dmds_full = common_get_device_memory_data_impl(paths_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
}
}
add_extra_memory(dmds_full);
Expand Down Expand Up @@ -493,7 +493,7 @@ static void common_params_fit_impl(

int64_t sum_projected_used_min_ctx = 0;
cparams->n_ctx = n_ctx_min_total;
dmds_t dmds_min_ctx = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
dmds_t dmds_min_ctx = common_get_device_memory_data_impl(paths_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
add_extra_memory(dmds_min_ctx);
if (nd == 0) {
sum_projected_used_min_ctx = dmds_min_ctx.back().mb.total();
Expand Down Expand Up @@ -675,7 +675,7 @@ static void common_params_fit_impl(
set_ngl_tensor_split_tbo(ngl_per_device, overflow_bufts, mparams_copy);

dmds_t dmd_nl = common_get_device_memory_data_impl(
path_model, &mparams_copy, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
paths_model, &mparams_copy, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
add_extra_memory(dmd_nl);

LOG_TRC("%s: memory for test allocation by device:\n", func_name);
Expand Down Expand Up @@ -704,7 +704,7 @@ static void common_params_fit_impl(

LOG_TRC("%s: getting device memory data with all MoE tensors moved to system memory:\n", __func__);
dmds_t dmds_cpu_moe = common_get_device_memory_data_impl(
path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
paths_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, log_level);
add_extra_memory(dmds_cpu_moe);

for (size_t id = 0; id < nd; id++) {
Expand Down Expand Up @@ -958,7 +958,7 @@ static void common_params_fit_impl(
}

enum common_params_fit_status common_fit_params(
const char * path_model,
const std::vector<std::string> & paths_model,
llama_model_params * mparams,
llama_context_params * cparams,
float * tensor_split,
Expand All @@ -970,7 +970,7 @@ enum common_params_fit_status common_fit_params(
const int64_t t0_us = llama_time_us();
common_params_fit_status status = COMMON_PARAMS_FIT_STATUS_SUCCESS;
try {
common_params_fit_impl(path_model, mparams, cparams, tensor_split, tensor_buft_overrides, margins, n_ctx_min, extra, log_level);
common_params_fit_impl(paths_model, mparams, cparams, tensor_split, tensor_buft_overrides, margins, n_ctx_min, extra, log_level);
LOG_TRC("%s: successfully fit params to free device memory\n", __func__);
} catch (const common_params_fit_exception & e) {
LOG_WRN("%s: failed to fit params to free device memory: %s\n", __func__, e.what());
Expand Down Expand Up @@ -1180,15 +1180,15 @@ void common_memory_breakdown_print(const struct llama_context * ctx) {
}

void common_fit_print(
const char * path_model,
const std::vector<std::string> & paths_model,
llama_model_params * mparams,
llama_context_params * cparams) {
std::vector<ggml_backend_dev_t> devs;
uint32_t hp_ngl = 0; // hparams.n_gpu_layers
uint32_t hp_nct = 0; // hparams.n_ctx_train
uint32_t hp_nex = 0; // hparams.n_expert

auto dmd = common_get_device_memory_data_impl(path_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, GGML_LOG_LEVEL_ERROR);
auto dmd = common_get_device_memory_data_impl(paths_model, mparams, cparams, devs, hp_ngl, hp_nct, hp_nex, GGML_LOG_LEVEL_ERROR);
GGML_ASSERT(dmd.size() == devs.size() + 1);

for (size_t id = 0; id < devs.size(); id++) {
Expand Down
12 changes: 6 additions & 6 deletions common/fit.h
Original file line number Diff line number Diff line change
Expand Up @@ -16,7 +16,7 @@ enum common_params_fit_status {
// - its context follows the context of the main model, so its memory is measured again whenever that context changes
// - shares_model tells the fit that the weights are already counted in the main model, as for an MTP context
struct common_fit_extra_model {
const char * path_model;
std::vector<std::string> paths_model;
llama_model_params * mparams;
llama_context_params * cparams;
bool shares_model;
Expand All @@ -28,19 +28,19 @@ struct common_fit_extra_model {
// - only parameters that have the same value as in llama_default_model_params are modified
// with the exception of the context size which is modified if and only if equal to 0
common_params_fit_status common_fit_params(
const char * path_model,
const std::vector<std::string> & paths_model,
llama_model_params * mparams,
llama_context_params * cparams,
float * tensor_split, // writable buffer for tensor split, needs at least llama_max_devices elements
llama_model_tensor_buft_override * tensor_buft_overrides, // writable buffer for overrides, needs at least llama_max_tensor_buft_overrides elements
size_t * margins, // margins of memory to leave per device in bytes
uint32_t n_ctx_min, // minimum context size to set when trying to reduce memory use
const common_fit_extra_model * extra, // model to fit alongside the main one, nullptr if there is none
const common_fit_extra_model * extra, // model to fit alongside the main one, nullptr if there is none
ggml_log_level log_level); // minimum log level to print during fitting, lower levels go to debug log

// print estimated memory to stdout
void common_fit_print(
const char * path_model,
const std::vector<std::string> & paths_model,
llama_model_params * mparams,
llama_context_params * cparams);

Expand Down Expand Up @@ -75,7 +75,7 @@ std::vector<common_memory_breakdown_row> common_memory_breakdown_rows(const llam
// load model + context with no_alloc and return the memory breakdown, params are used as-is
// n_ctx_train and n_ctx are optional outputs; throws on failure
std::vector<common_memory_breakdown_row> common_fit_measure(
const char * path_model,
const std::vector<std::string> & paths_model,
const llama_model_params * mparams,
const llama_context_params * cparams,
uint32_t * n_ctx_train,
Expand All @@ -99,7 +99,7 @@ using common_device_memory_data_vec = std::vector<common_device_memory_data>;

// Load a model + context with no_alloc and return the per-device memory breakdown.
common_device_memory_data_vec common_get_device_memory_data(
const char * path_model,
const std::vector<std::string> & paths_model,
const llama_model_params * mparams,
const llama_context_params * cparams,
std::vector<ggml_backend_dev_t> & devs,
Expand Down
2 changes: 1 addition & 1 deletion common/speculative.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -2815,7 +2815,7 @@ common_speculative_init_result::common_speculative_init_result(
model_path = params.speculative.draft.mparams.path;
LOG_INF("%s: loading draft model '%s'\n", __func__, model_path.c_str());

llama_model * model_dft = llama_model_load_from_file(model_path.c_str(), mparams);
llama_model * model_dft = common_model_load(params.speculative.draft.mparams.get_paths(), mparams);
if (model_dft == NULL) {
LOG_ERR("%s: failed to load draft model, '%s'\n", __func__, model_path.c_str());
return;
Expand Down
Loading