Carry ggml-org#25731 (TML Inkling) merged onto b10775

common/chat-peg-parser.cpp

@@ -630,7 +630,8 @@ common_peg_parser common_chat_peg_builder::build_json_tools_function_is_key(
     const std::string &  args_key,
     const std::string &  effective_args_key,
     const std::string &  call_id_key,
-    const std::string &  gen_call_id_key) {
+    const std::string &  gen_call_id_key,
+    bool                 require_object_args) {
 
     auto tool_choices = choice();
 
@@ -667,10 +668,10 @@ common_peg_parser common_chat_peg_builder::build_json_tools_function_is_key(
         // Arguments — either wrapped in args_key or parsed directly
         common_peg_parser args_parser = eps();
         if (args_key.empty()) {
-            args_parser = tool_args(schema(json(), "tool-" + name + "-schema", params));
+            args_parser = tool_args(schema(require_object_args ? json_object() : json(), "tool-" + name + "-schema", params));
         } else {
             args_parser = literal("\"" + effective_args_key + "\"") + space() + literal(":") + space() +
-                          tool_args(schema(json(), "tool-" + name + "-schema", params));
+                          tool_args(schema(require_object_args ? json_object() : json(), "tool-" + name + "-schema", params));
         }
         inner_fields.push_back(args_parser);
 
@@ -709,7 +710,8 @@ common_peg_parser common_chat_peg_builder::build_json_tools_nested_keys(
     const std::string &  effective_name_key,
     const std::string &  effective_args_key,
     const std::string &  call_id_key,
-    const std::string &  gen_call_id_key) {
+    const std::string &  gen_call_id_key,
+    bool                 require_object_args) {
 
     auto tool_choices = choice();
 
@@ -731,7 +733,7 @@ common_peg_parser common_chat_peg_builder::build_json_tools_nested_keys(
         auto nested_name = literal("\"" + nested_name_field + "\"") + space() + literal(":") + space() +
                           atomic(literal("\"") + tool_name(literal(name)) + literal("\""));
         auto nested_args = literal("\"" + nested_args_field + "\"") + space() + literal(":") + space() +
-                          tool_args(schema(json(), "tool-" + name + "-schema", params));
+                          tool_args(schema(require_object_args ? json_object() : json(), "tool-" + name + "-schema", params));
 
         auto nested_object = literal("{") + space() +
                             nested_name + space() + literal(",") + space() +
@@ -783,7 +785,8 @@ common_peg_parser common_chat_peg_builder::build_json_tools_flat_keys(
     const std::string &              call_id_key,
     const std::string &              gen_call_id_key,
     const std::vector<std::string> & parameters_order,
-    bool                             accept_openai_wrapper) {
+    bool                             accept_openai_wrapper,
+    bool                             require_object_args) {
 
     auto tool_choices    = choice();
     auto name_key_parser = literal("\"" + effective_name_key + "\"");
@@ -800,7 +803,7 @@ common_peg_parser common_chat_peg_builder::build_json_tools_flat_keys(
         auto tool_name_ = name_key_parser + space() + literal(":") + space() +
                          atomic(literal("\"") + tool_name(literal(name)) + literal("\""));
         auto tool_args_ = args_key_parser + space() + literal(":") + space() +
-                         tool_args(schema(json(), "tool-" + name + "-schema", params));
+                         tool_args(schema(require_object_args ? json_object() : json(), "tool-" + name + "-schema", params));
 
         // Build ID parsers if keys are provided
         common_peg_parser id_parser = eps();
@@ -915,7 +918,8 @@ common_peg_parser common_chat_peg_builder::standard_json_tools(
                                                        const std::string &              call_id_key,
                                                        const std::string &              gen_call_id_key,
                                                        const std::vector<std::string> & parameters_order,
-                                                       bool                             accept_openai_wrapper) {
+                                                       bool                             accept_openai_wrapper,
+                                                       bool                             require_object_args) {
     if (!tools.is_array() || tools.empty()) {
         return eps();
     }
@@ -926,14 +930,14 @@ common_peg_parser common_chat_peg_builder::standard_json_tools(
     // Dispatch to the appropriate builder based on the JSON layout mode
     common_peg_parser tool_choices = eps();
     if (function_is_key) {
-        tool_choices = build_json_tools_function_is_key(tools, args_key, effective_args_key, call_id_key, gen_call_id_key);
+        tool_choices = build_json_tools_function_is_key(tools, args_key, effective_args_key, call_id_key, gen_call_id_key, require_object_args);
     } else {
         auto name_spec = parse_key_spec(effective_name_key);
         auto args_spec = parse_key_spec(effective_args_key);
         if (!name_spec.first.empty() || !args_spec.first.empty()) {
-            tool_choices = build_json_tools_nested_keys(tools, effective_name_key, effective_args_key, call_id_key, gen_call_id_key);
+            tool_choices = build_json_tools_nested_keys(tools, effective_name_key, effective_args_key, call_id_key, gen_call_id_key, require_object_args);
         } else {
-            tool_choices = build_json_tools_flat_keys(tools, effective_name_key, effective_args_key, call_id_key, gen_call_id_key, parameters_order, accept_openai_wrapper);
+            tool_choices = build_json_tools_flat_keys(tools, effective_name_key, effective_args_key, call_id_key, gen_call_id_key, parameters_order, accept_openai_wrapper, require_object_args);
         }
     }
 

common/chat-peg-parser.h

@@ -138,7 +138,8 @@ class common_chat_peg_builder : public common_peg_parser_builder {
                                           const std::string &              call_id_key = "",
                                           const std::string &              gen_call_id_key = "",
                                           const std::vector<std::string> & parameters_order = {},
-                                          bool                             accept_openai_wrapper = false);
+                                          bool                             accept_openai_wrapper = false,
+                                          bool                             require_object_args = false);
 
     // Legacy-compatible helper for building XML/tagged style tool calls
     // Used by tests and manual parsers
@@ -162,21 +163,24 @@ class common_chat_peg_builder : public common_peg_parser_builder {
                                                        const std::string &            args_key,
                                                        const std::string &            effective_args_key,
                                                        const std::string &            call_id_key,
-                                                       const std::string &            gen_call_id_key);
+                                                       const std::string &            gen_call_id_key,
+                                                       bool                           require_object_args);
 
     common_peg_parser build_json_tools_nested_keys(const common_json & tools,
                                                    const std::string &            effective_name_key,
                                                    const std::string &            effective_args_key,
                                                    const std::string &            call_id_key,
-                                                   const std::string &            gen_call_id_key);
+                                                   const std::string &            gen_call_id_key,
+                                                   bool                           require_object_args);
 
     common_peg_parser build_json_tools_flat_keys(const common_json &   tools,
                                                  const std::string &              effective_name_key,
                                                  const std::string &              effective_args_key,
                                                  const std::string &              call_id_key,
                                                  const std::string &              gen_call_id_key,
                                                  const std::vector<std::string> & parameters_order,
-                                                 bool                             accept_openai_wrapper);
+                                                 bool                             accept_openai_wrapper,
+                                                 bool                             require_object_args);
 };
 
 inline common_peg_arena build_chat_peg_parser(

common/chat.cpp

@@ -2957,6 +2957,125 @@ static common_chat_params common_chat_params_init_minimax_m3(const common_chat_t
     return data;
 }
 
+// Inkling / TML typed-content-block parser: <|end_message|> separates blocks within a turn,
+// <|content_model_end_sampling|> is the sole end-of-generation token (mirrors sglang TmlDetector).
+static common_chat_params common_chat_params_init_inkling(const common_chat_template &          tmpl,
+                                                          const autoparser::generation_params & inputs) {
+    common_chat_params data;
+
+    const std::string MSG_MODEL    = "<|message_model|>";
+    const std::string MSG_USER     = "<|message_user|>";
+    const std::string MSG_SYSTEM   = "<|message_system|>";
+    const std::string MSG_TOOL     = "<|message_tool|>";
+    const std::string THINK        = "<|content_thinking|>";
+    const std::string TEXT         = "<|content_text|>";
+    const std::string END_MESSAGE  = "<|end_message|>";
+    const std::string END_SAMPLING = "<|content_model_end_sampling|>";
+    const std::string INVOKE_TOOL  = "<|content_invoke_tool_json|>";
+
+    data.prompt             = common_chat_template_direct_apply_impl(tmpl, inputs);
+    data.generation_prompt  = common_chat_template_generation_prompt_impl(tmpl, inputs);
+    data.format             = COMMON_CHAT_FORMAT_PEG_NATIVE;
+    data.supports_thinking  = true;
+    data.thinking_start_tag = THINK;
+    data.thinking_end_tags  = {END_MESSAGE};
+    data.preserved_tokens   = {
+        MSG_MODEL, MSG_USER, MSG_SYSTEM, MSG_TOOL,
+        THINK, TEXT, END_MESSAGE, END_SAMPLING, INVOKE_TOOL,
+    };
+
+    auto has_tools = inputs.tools.is_array() && !inputs.tools.empty();
+    data.message_delimiters = {
+        { COMMON_CHAT_ROLE_ASSISTANT, MSG_MODEL },
+        { COMMON_CHAT_ROLE_USER,      MSG_USER },
+        { COMMON_CHAT_ROLE_SYSTEM,    MSG_SYSTEM },
+        { COMMON_CHAT_ROLE_TOOL,      MSG_TOOL },
+    };
+
+    auto extract_reasoning = inputs.reasoning_format != COMMON_REASONING_FORMAT_NONE;
+
+    if (inputs.has_continuation()) {
+        const auto & msg = inputs.continue_msg;
+
+        data.generation_prompt = MSG_MODEL + THINK + msg.reasoning_content;
+        if (inputs.continue_final_message == COMMON_CHAT_CONTINUATION_CONTENT) {
+            data.generation_prompt += END_MESSAGE + TEXT + msg.render_content();
+        }
+
+        data.prompt += data.generation_prompt;
+    }
+
+    auto parser = build_chat_peg_parser([&](common_chat_peg_builder & p) {
+        auto generation_prompt = p.literal(MSG_MODEL);
+        auto end               = p.end();
+
+        // thinking block; may also reappear mid-turn (after content), so it is both an optional
+        // prefix and a choice inside the block loops. With reasoning_format=NONE keep it
+        // (markers included) inline as content
+        common_peg_parser reasoning_block = p.eps();
+        if (extract_reasoning) {
+            reasoning_block = p.literal(THINK) +
+                              p.reasoning(p.until_one_of({ END_MESSAGE, TEXT, END_SAMPLING })) +
+                              p.optional(p.literal(END_MESSAGE));
+        } else {
+            reasoning_block = p.content(p.literal(THINK) +
+                                        p.until_one_of({ END_MESSAGE, TEXT, END_SAMPLING }) +
+                                        p.optional(p.literal(END_MESSAGE)));
+        }
+        auto reasoning = p.optional(reasoning_block);
+
+        // TML re-emits <|message_model|> before each content block; a turn may contain several
+        // text blocks (one per content part), so the block repeats and bodies concatenate.
+        // THINK stops the content scan so a mid-turn thinking block is never leaked as text
+        auto text_block = p.optional(p.literal(MSG_MODEL)) +
+                          p.optional(p.literal(TEXT)) +
+                          p.content(p.until_one_of({ THINK, END_MESSAGE, END_SAMPLING })) +
+                          p.optional(p.literal(END_MESSAGE));
+        auto text_content = p.one_or_more(p.choice({ reasoning_block, text_block }));
+
+        if (!has_tools || inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_NONE) {
+            return generation_prompt + reasoning + text_content +
+                   p.optional(p.literal(END_SAMPLING)) + end;
+        }
+
+        // each call is its own block (role opener + bare name echo + JSON section);
+        // force_tool_calls=true makes the JSON section required so a pure-text answer fails the
+        // block cleanly; parallel calls are separate blocks, hence repeat + parallel=false
+        auto tool_section = p.standard_json_tools(
+            INVOKE_TOOL, END_MESSAGE, inputs.tools, /* parallel_tool_calls = */ false,
+            /* force_tool_calls = */ true,
+            /* name_key          = */ "name",
+            /* args_key          = */ "args",
+            /* array_wrapped     = */ false,
+            /* function_is_key   = */ false,
+            /* call_id_key       = */ "",
+            /* gen_call_id_key   = */ "",
+            /* parameters_order  = */ {},
+            /* accept_openai_wrapper = */ false,
+            /* require_object_args   = */ true);
+        // the name-echo scan must stop at any block marker: a greedy until(INVOKE_TOOL) returns
+        // NEED_MORE_INPUT mid-stream, which choice() treats as a match and shadows the text branch
+        auto tool_block = p.optional(p.literal(MSG_MODEL)) +
+                          p.until_one_of({ INVOKE_TOOL, TEXT, THINK, END_MESSAGE, END_SAMPLING }) +
+                          tool_section;
+        auto tool_calls = inputs.parallel_tool_calls ? p.one_or_more(tool_block) : tool_block;
+        // turns may interleave narration, thinking and calls; parse block-by-block (tool block
+        // first) since a whole-body choice would let the text branch swallow tool blocks into
+        // visible content
+        auto mixed_body = p.one_or_more(p.choice({ tool_block, reasoning_block, text_block }));
+        auto body       = inputs.tool_choice == COMMON_CHAT_TOOL_CHOICE_REQUIRED
+                              ? tool_calls
+                              : mixed_body;
+
+        return generation_prompt + reasoning + body +
+               p.optional(p.literal(END_SAMPLING)) + end;
+    });
+
+    data.parser = parser.save();
+
+    return data;
+}
+
 namespace workaround {
 
 static void map_developer_role_to_system(json & messages) {
@@ -3534,6 +3653,14 @@ std::optional<common_chat_params> common_chat_try_specialized_template(
         return common_chat_params_init_cohere2moe(tmpl, params);
     }
 
+    // Inkling / TML: this marker combination is unique to the template
+    if (src.find("<|content_thinking|>") != std::string::npos &&
+        src.find("<|content_text|>") != std::string::npos &&
+        src.find("<|message_model|>") != std::string::npos) {
+        LOG_DBG("Using specialized template: Inkling\n");
+        return common_chat_params_init_inkling(tmpl, params);
+    }
+
     if (is_lfm2_template(src)) {
         LOG_DBG("Using specialized template: LFM2\n");
         return common_chat_params_init_lfm2(tmpl, params, /* tool_list_tokens = */ true);

conversion/__init__.py

@@ -125,6 +125,7 @@
     "HunYuanVLForConditionalGeneration": "hunyuan",
     "HYV3ForCausalLM": "hunyuan",
     "IQuestCoderForCausalLM": "llama",
+    "InklingForConditionalGeneration": "inkling",
     "InternLM2ForCausalLM": "internlm",
     "InternLM3ForCausalLM": "internlm",
     "JAISLMHeadModel": "jais",
@@ -306,6 +307,7 @@
     "GraniteSpeechPlusForConditionalGeneration": "granite",
     "HunYuanVLForConditionalGeneration": "hunyuan",
     "Idefics3ForConditionalGeneration": "smolvlm",
+    "InklingForConditionalGeneration": "inkling",
     "InternVisionModel": "internvl",
     "JanusForConditionalGeneration": "januspro",
     "KimiK25ForConditionalGeneration": "kimivl",

conversion/base.py

@@ -274,8 +274,17 @@ def index_tensors(self, remote_hf_model_id: str | None = None) -> dict[str, Call
                             data_gen = lambda data=data_torch: LazyTorchTensor.from_eager(data)  # noqa: E731
                         else:
                             data_gen = lambda data=data_torch: data  # noqa: E731
+                    # the index maps each tensor to one shard; a duplicate would silently overwrite it
+                    if weight_map and name in weight_map and weight_map[name] != part_name:
+                        raise ValueError(
+                            f"tensor '{name}' found in '{part_name}' but the index assigns "
+                            f"it to '{weight_map[name]}'; refusing to load a wrong-shard copy")
                     if titem := self.filter_tensors((name, data_gen)):
                         tname, tgen = titem
+                        if tname in tensors:
+                            raise ValueError(
+                                f"duplicate tensor '{tname}' found in multiple model parts; "
+                                f"refusing to silently overwrite")
                         tensors[tname] = tgen
 
         # verify tensor name presence and identify potentially missing files
@@ -543,6 +552,20 @@ def dequant_packed(w: Tensor, scale: Tensor, shape_tensor: Tensor, zero_point: T
                 else:
                     raise NotImplementedError(f"Quant format {quant_format!r} for method {quant_method!r} is not yet supported")
             elif quant_method == "modelopt":
+                # Stacked-expert NVFP4 checkpoints (e.g. Inkling-NVFP4) store experts as
+                # w13_weight/w2_weight with .scale/.scale2/.input_amax/.original_shape
+                # auxiliaries; the main tensors do not end in .weight, so the NVFP4 path
+                # below would silently skip them. Reject them with a clear error.
+                stacked_expert_aux = [
+                    n for n in self.model_tensors
+                    if n.endswith((".scale2", ".input_amax", ".original_shape"))
+                ]
+                if stacked_expert_aux:
+                    raise NotImplementedError(
+                        "This checkpoint stores quantized experts in the stacked ModelOpt NVFP4 layout "
+                        f"({len(stacked_expert_aux)} auxiliary tensors like {stacked_expert_aux[0]!r}), "
+                        "which is not supported yet. Convert from the unquantized (BF16) checkpoint instead."
+                    )
                 # Mixed-precision ModelOpt models: NVFP4 tensors are handled by
                 # _generate_nvfp4_tensors; FP8 tensors have 1D weight_scale and
                 # are dequantized here. k/v scale tensors are unused.
@@ -1070,6 +1093,14 @@ def write_vocab(self):
 
     def write(self):
         self.prepare_tensors()
+        # zero tensors means the shards were never discovered, yet a metadata-only GGUF logs success
+        n_written = sum(len(shard) for shard in self.gguf_writer.tensors)
+        if n_written == 0:
+            raise ValueError(
+                "no tensors were written: the model shards could not be found. "
+                "Check that the safetensors filenames are discoverable (they must "
+                "start with 'model') or that model.safetensors exists."
+            )
         self.prepare_metadata(vocab_only=False)
         self.gguf_writer.write_header_to_file(path=self.fname_out)
         self.gguf_writer.write_kv_data_to_file()