From be3e2c6e58995ceaf3852ce75f33fd469e217a27 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yi=C4=9Fit=20ERDO=C4=9EAN?= Date: Wed, 12 Aug 2026 22:09:57 +0300 Subject: [PATCH 1/4] [#17572][fix] Emit the withheld buffer in DeepSeek streaming tool parsers MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit parse_streaming_increment accumulates deltas into self._buffer and withholds the buffer while it could still grow into a tool-call start token. Once the ambiguity cleared, the buffer was cleared but only the current delta was returned, so everything withheld by an earlier increment was dropped from the streamed response. Both start tokens begin with "<", so ordinary assistant text containing "<" lost characters. Emit the buffer instead of the delta. In the V3 and V3.1 parsers also test the buffer with the _ends_with_partial_token helper rather than testing the delta with startswith, which matches BaseToolParser.parse_streaming_increment and the gemma4, minimax_m3, poolside_v1, qwen3 and kimi_k3 parsers. V3.2 already gates on the buffer, so only the emitted text changes there, and DeepSeekV4Parser inherits the fix from DeepSeekV32Parser. Signed-off-by: Yiğit ERDOĞAN --- .../serve/tool_parser/deepseekv31_parser.py | 13 +++++--- .../serve/tool_parser/deepseekv32_parser.py | 10 ++++-- .../serve/tool_parser/deepseekv3_parser.py | 13 +++++--- .../unittest/llmapi/apps/test_tool_parsers.py | 32 +++++++++++++++++++ 4 files changed, 55 insertions(+), 13 deletions(-) diff --git a/tensorrt_llm/serve/tool_parser/deepseekv31_parser.py b/tensorrt_llm/serve/tool_parser/deepseekv31_parser.py index a0b306a1060e..b194c85db3d0 100644 --- a/tensorrt_llm/serve/tool_parser/deepseekv31_parser.py +++ b/tensorrt_llm/serve/tool_parser/deepseekv31_parser.py @@ -98,16 +98,19 @@ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> Streami has_tool_call = self.bot_token in current_text or "<｜tool▁call▁begin｜>" in current_text if not has_tool_call: + # The buffer, not just the latest delta, is what is being withheld, so + # the partial-token check and the emitted text must both be the buffer. if any( - e_token.startswith(new_text) - for e_token in [self.bot_token, "<｜tool▁call▁begin｜>"] + self._ends_with_partial_token(current_text, b_token) + for b_token in [self.bot_token, "<｜tool▁call▁begin｜>"] ): return StreamingParseResult() + normal_text = current_text self._buffer = "" for e_token in [self.eot_token, "<｜tool▁call▁end｜>"]: - if e_token in new_text: - new_text = new_text.replace(e_token, "") - return StreamingParseResult(normal_text=new_text) + if e_token in normal_text: + normal_text = normal_text.replace(e_token, "") + return StreamingParseResult(normal_text=normal_text) if not hasattr(self, "_tool_indices"): self._tool_indices = self._get_tool_indices(tools) diff --git a/tensorrt_llm/serve/tool_parser/deepseekv32_parser.py b/tensorrt_llm/serve/tool_parser/deepseekv32_parser.py index 3362ab7cb68a..9360261a2129 100644 --- a/tensorrt_llm/serve/tool_parser/deepseekv32_parser.py +++ b/tensorrt_llm/serve/tool_parser/deepseekv32_parser.py @@ -182,11 +182,15 @@ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> Streami ends_with_prefix = any(current_text.rstrip().endswith(prefix) for prefix in dsml_prefixes) if not has_tool_call and not potentially_dsml and not ends_with_prefix: + # The guards above withhold the whole buffer, so the buffer is what has + # to be emitted once they clear; returning only the latest delta would + # drop everything withheld by an earlier increment. + normal_text = current_text self._buffer = "" for e_token in [self.eot_token, self.invoke_end_token, self._eos_token]: - if e_token in new_text: - new_text = new_text.replace(e_token, "") - return StreamingParseResult(normal_text=new_text) + if e_token in normal_text: + normal_text = normal_text.replace(e_token, "") + return StreamingParseResult(normal_text=normal_text) if not hasattr(self, "_tool_indices"): self._tool_indices = self._get_tool_indices(tools) diff --git a/tensorrt_llm/serve/tool_parser/deepseekv3_parser.py b/tensorrt_llm/serve/tool_parser/deepseekv3_parser.py index 8eb49eb81c07..ebab617f7e66 100644 --- a/tensorrt_llm/serve/tool_parser/deepseekv3_parser.py +++ b/tensorrt_llm/serve/tool_parser/deepseekv3_parser.py @@ -101,16 +101,19 @@ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> Streami has_tool_call = self.bot_token in current_text or "<｜tool▁call▁begin｜>" in current_text if not has_tool_call: + # The buffer, not just the latest delta, is what is being withheld, so + # the partial-token check and the emitted text must both be the buffer. if any( - e_token.startswith(new_text) - for e_token in [self.bot_token, "<｜tool▁call▁begin｜>"] + self._ends_with_partial_token(current_text, b_token) + for b_token in [self.bot_token, "<｜tool▁call▁begin｜>"] ): return StreamingParseResult() + normal_text = current_text self._buffer = "" for e_token in [self.eot_token, "```", "<｜tool▁call▁end｜>"]: - if e_token in new_text: - new_text = new_text.replace(e_token, "") - return StreamingParseResult(normal_text=new_text) + if e_token in normal_text: + normal_text = normal_text.replace(e_token, "") + return StreamingParseResult(normal_text=normal_text) if not hasattr(self, "_tool_indices"): self._tool_indices = self._get_tool_indices(tools) diff --git a/tests/unittest/llmapi/apps/test_tool_parsers.py b/tests/unittest/llmapi/apps/test_tool_parsers.py index 7b24d4554337..a36050da170f 100644 --- a/tests/unittest/llmapi/apps/test_tool_parsers.py +++ b/tests/unittest/llmapi/apps/test_tool_parsers.py @@ -1778,6 +1778,38 @@ def make_tool_parser_test_cases(self): ) +# ============================================================================ +# DeepSeek streaming text preservation +# ============================================================================ + + +@pytest.mark.parametrize( + "parser_cls", + [DeepSeekV3Parser, DeepSeekV31Parser, DeepSeekV32Parser, DeepSeekV4Parser]) +@pytest.mark.parametrize( + "deltas", + [ + # A delta that is itself a prefix of a tool-call start token. + ["Use ", "<", "div> for a block element."], + # A delta that ends on such a prefix after other text. + ["The condition is a <", " b, so it holds."], + ], +) +def test_deepseek_streaming_preserves_withheld_text(sample_tools, parser_cls, + deltas): + """Withholding a delta may delay text but must never drop it.""" + parser = parser_cls() + + streamed = "".join( + parser.parse_streaming_increment(delta, sample_tools).normal_text + for delta in deltas) + + expected = "".join(deltas) + assert streamed == expected, f"Expected {expected!r}, got {streamed!r}" + assert parser_cls().detect_and_parse(expected, + sample_tools).normal_text == expected + + # ============================================================================ # Glm4ToolParser Tests # ============================================================================ From a4ac2f004d20a3c887432f69749720b4c59c0fb8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yi=C4=9Fit=20ERDO=C4=9EAN?= Date: Wed, 12 Aug 2026 22:25:27 +0300 Subject: [PATCH 2/4] [#17572][fix] Hold V3.2 text only while it can still complete a DSML tag MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The V3.2 guard withheld the buffer whenever a DSML marker appeared anywhere in it, so once ordinary text diverged from a delimiter the buffer was never released. "Use <｜DSML｜function" followed by "ality" is the smallest case: the text is neither a tool call nor emittable, and with no end-of-stream flush it is lost. The same applies to DeepSeekV4Parser, which inherits this path. Replace the marker-presence and rstrip-endswith heuristics with the _ends_with_partial_token helper, matching the V3 and V3.1 parsers. The check covers the closing tokens as well, so a delimiter split across deltas is still withheld and stripped rather than leaking into content. Also annotate the new test and add the diverged-prefix case, which fails on the V3.2 and V4 parsers without this change. Signed-off-by: Yiğit ERDOĞAN --- .../serve/tool_parser/deepseekv32_parser.py | 30 +++++++++++-------- .../unittest/llmapi/apps/test_tool_parsers.py | 7 +++-- 2 files changed, 23 insertions(+), 14 deletions(-) diff --git a/tensorrt_llm/serve/tool_parser/deepseekv32_parser.py b/tensorrt_llm/serve/tool_parser/deepseekv32_parser.py index 9360261a2129..9c2cb0b9321c 100644 --- a/tensorrt_llm/serve/tool_parser/deepseekv32_parser.py +++ b/tensorrt_llm/serve/tool_parser/deepseekv32_parser.py @@ -172,18 +172,24 @@ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> Streami # If we see these markers anywhere, we should keep buffering has_tool_call = self.bot_token in current_text or "<｜DSML｜invoke" in current_text - # Check if buffer contains any DSML markers or ends with potential tag prefix - # This handles partial/streaming DSML content - dsml_markers = ["｜DSML｜", "<｜", " for a block element."], # A delta that ends on such a prefix after other text. ["The condition is a <", " b, so it holds."], + # Text that starts like a start token and then diverges from it. + ["Use <｜DSML｜function", "ality"], ], ) -def test_deepseek_streaming_preserves_withheld_text(sample_tools, parser_cls, - deltas): +def test_deepseek_streaming_preserves_withheld_text( + sample_tools: list[ChatCompletionToolsParam], + parser_cls: type[BaseToolParser], deltas: list[str]) -> None: """Withholding a delta may delay text but must never drop it.""" parser = parser_cls() From 32209fde173cd0354f71c0404e875c55d2da3edc Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yi=C4=9Fit=20ERDO=C4=9EAN?= Date: Wed, 12 Aug 2026 22:37:37 +0300 Subject: [PATCH 3/4] [#17572][fix] Require a complete invoke header before buffering as a tool call MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The V3.2 streaming path treated the bare "<｜DSML｜invoke" prefix appearing anywhere in the buffer as a tool call, so ordinary text that merely starts like the token, such as "<｜DSML｜invoke" followed by "ality", was routed into the tool-call branch. No invoke matched there, the buffer was kept, and the text was never emitted. DeepSeekV4Parser inherits this path. Key off the header up to its opening quote instead. Everything after that point is the arbitrary function name, so this is the longest fixed prefix ordinary text cannot reproduce by accident, and using the same string in the partial-token list keeps a header that arrives split across deltas buffered. The regression case now diverges from both tokens the parser looks for. Signed-off-by: Yiğit ERDOĞAN --- .../serve/tool_parser/deepseekv32_parser.py | 15 ++++++++++----- tests/unittest/llmapi/apps/test_tool_parsers.py | 5 +++-- 2 files changed, 13 insertions(+), 7 deletions(-) diff --git a/tensorrt_llm/serve/tool_parser/deepseekv32_parser.py b/tensorrt_llm/serve/tool_parser/deepseekv32_parser.py index 9c2cb0b9321c..60f3fe520c50 100644 --- a/tensorrt_llm/serve/tool_parser/deepseekv32_parser.py +++ b/tensorrt_llm/serve/tool_parser/deepseekv32_parser.py @@ -65,6 +65,10 @@ class DeepSeekV32Parser(BaseToolParser): _eos_token = "<｜end▁of▁sentence｜>" # nosec B105 + # Shortest prefix of an invoke header that ordinary text cannot reproduce by + # accident; everything after it is the arbitrary function name. + _INVOKE_HEADER_PREFIX = '<｜DSML｜invoke name="' # nosec B105 + def __init__(self): super().__init__() self.bot_token = "<｜DSML｜function_calls>" # nosec B105 @@ -167,10 +171,11 @@ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> Streami self._buffer += new_text current_text = self._buffer - # Check if we have a tool call or any DSML-related content - # Key insight: DSML tags contain distinctive markers like "｜DSML｜" - # If we see these markers anywhere, we should keep buffering - has_tool_call = self.bot_token in current_text or "<｜DSML｜invoke" in current_text + # An invoke header counts as a tool call once its opening quote is in, since + # the function name that follows is arbitrary text. Matching the bare + # "<｜DSML｜invoke" prefix instead would also match ordinary text that merely + # starts like the token, such as "<｜DSML｜invokeality". + has_tool_call = self.bot_token in current_text or self._INVOKE_HEADER_PREFIX in current_text # Hold the buffer back only while its tail could still complete into one of # the DSML delimiters. Testing whether a marker appears anywhere instead @@ -178,7 +183,7 @@ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> Streami # for example "<｜DSML｜function" followed by "ality". partial_tokens = [ self.bot_token, - "<｜DSML｜invoke", + self._INVOKE_HEADER_PREFIX, self.eot_token, self.invoke_end_token, self._eos_token, diff --git a/tests/unittest/llmapi/apps/test_tool_parsers.py b/tests/unittest/llmapi/apps/test_tool_parsers.py index 22bcdc4d2614..d5dffa055a0e 100644 --- a/tests/unittest/llmapi/apps/test_tool_parsers.py +++ b/tests/unittest/llmapi/apps/test_tool_parsers.py @@ -1793,8 +1793,9 @@ def make_tool_parser_test_cases(self): ["Use ", "<", "div> for a block element."], # A delta that ends on such a prefix after other text. ["The condition is a <", " b, so it holds."], - # Text that starts like a start token and then diverges from it. - ["Use <｜DSML｜function", "ality"], + # Text that starts like a start token and then diverges from it, for + # both of the tokens the V3.2 and V4 parsers look for. + ["Use <｜DSML｜function", "ality and <｜DSML｜invoke", "ality"], ], ) def test_deepseek_streaming_preserves_withheld_text( From 8b382f6e79d0bc8f9f68d38f0a6068fb9db967e1 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Yi=C4=9Fit=20ERDO=C4=9EAN?= Date: Thu, 13 Aug 2026 08:49:07 +0300 Subject: [PATCH 4/4] [#17572][chore] Drop redundant membership checks and change-log comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `str.replace` already scans the string, so guarding it with `in` only repeats the scan. The comments that motivated the change described the previous behaviour rather than the invariant the code now holds, which is noise for anyone reading only the current version. Signed-off-by: Yiğit ERDOĞAN --- .../serve/tool_parser/deepseekv31_parser.py | 5 +---- .../serve/tool_parser/deepseekv32_parser.py | 17 +++-------------- .../serve/tool_parser/deepseekv3_parser.py | 5 +---- 3 files changed, 5 insertions(+), 22 deletions(-) diff --git a/tensorrt_llm/serve/tool_parser/deepseekv31_parser.py b/tensorrt_llm/serve/tool_parser/deepseekv31_parser.py index b194c85db3d0..78caac5139a7 100644 --- a/tensorrt_llm/serve/tool_parser/deepseekv31_parser.py +++ b/tensorrt_llm/serve/tool_parser/deepseekv31_parser.py @@ -98,8 +98,6 @@ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> Streami has_tool_call = self.bot_token in current_text or "<｜tool▁call▁begin｜>" in current_text if not has_tool_call: - # The buffer, not just the latest delta, is what is being withheld, so - # the partial-token check and the emitted text must both be the buffer. if any( self._ends_with_partial_token(current_text, b_token) for b_token in [self.bot_token, "<｜tool▁call▁begin｜>"] @@ -108,8 +106,7 @@ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> Streami normal_text = current_text self._buffer = "" for e_token in [self.eot_token, "<｜tool▁call▁end｜>"]: - if e_token in normal_text: - normal_text = normal_text.replace(e_token, "") + normal_text = normal_text.replace(e_token, "") return StreamingParseResult(normal_text=normal_text) if not hasattr(self, "_tool_indices"): diff --git a/tensorrt_llm/serve/tool_parser/deepseekv32_parser.py b/tensorrt_llm/serve/tool_parser/deepseekv32_parser.py index 60f3fe520c50..1692b4ca0e87 100644 --- a/tensorrt_llm/serve/tool_parser/deepseekv32_parser.py +++ b/tensorrt_llm/serve/tool_parser/deepseekv32_parser.py @@ -65,8 +65,7 @@ class DeepSeekV32Parser(BaseToolParser): _eos_token = "<｜end▁of▁sentence｜>" # nosec B105 - # Shortest prefix of an invoke header that ordinary text cannot reproduce by - # accident; everything after it is the arbitrary function name. + # Invoke header up to the function name, which is arbitrary text. _INVOKE_HEADER_PREFIX = '<｜DSML｜invoke name="' # nosec B105 def __init__(self): @@ -171,16 +170,10 @@ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> Streami self._buffer += new_text current_text = self._buffer - # An invoke header counts as a tool call once its opening quote is in, since - # the function name that follows is arbitrary text. Matching the bare - # "<｜DSML｜invoke" prefix instead would also match ordinary text that merely - # starts like the token, such as "<｜DSML｜invokeality". has_tool_call = self.bot_token in current_text or self._INVOKE_HEADER_PREFIX in current_text # Hold the buffer back only while its tail could still complete into one of - # the DSML delimiters. Testing whether a marker appears anywhere instead - # would keep the buffer forever once ordinary text diverges from a delimiter, - # for example "<｜DSML｜function" followed by "ality". + # the DSML delimiters. partial_tokens = [ self.bot_token, self._INVOKE_HEADER_PREFIX, @@ -193,14 +186,10 @@ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> Streami ) if not has_tool_call and not ends_with_partial_token: - # The guard above withholds the whole buffer, so the buffer is what has - # to be emitted once it clears; returning only the latest delta would - # drop everything withheld by an earlier increment. normal_text = current_text self._buffer = "" for e_token in [self.eot_token, self.invoke_end_token, self._eos_token]: - if e_token in normal_text: - normal_text = normal_text.replace(e_token, "") + normal_text = normal_text.replace(e_token, "") return StreamingParseResult(normal_text=normal_text) if not hasattr(self, "_tool_indices"): diff --git a/tensorrt_llm/serve/tool_parser/deepseekv3_parser.py b/tensorrt_llm/serve/tool_parser/deepseekv3_parser.py index ebab617f7e66..7cb8958de416 100644 --- a/tensorrt_llm/serve/tool_parser/deepseekv3_parser.py +++ b/tensorrt_llm/serve/tool_parser/deepseekv3_parser.py @@ -101,8 +101,6 @@ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> Streami has_tool_call = self.bot_token in current_text or "<｜tool▁call▁begin｜>" in current_text if not has_tool_call: - # The buffer, not just the latest delta, is what is being withheld, so - # the partial-token check and the emitted text must both be the buffer. if any( self._ends_with_partial_token(current_text, b_token) for b_token in [self.bot_token, "<｜tool▁call▁begin｜>"] @@ -111,8 +109,7 @@ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> Streami normal_text = current_text self._buffer = "" for e_token in [self.eot_token, "```", "<｜tool▁call▁end｜>"]: - if e_token in normal_text: - normal_text = normal_text.replace(e_token, "") + normal_text = normal_text.replace(e_token, "") return StreamingParseResult(normal_text=normal_text) if not hasattr(self, "_tool_indices"):