Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
27 changes: 11 additions & 16 deletions tensorrt_llm/serve/tool_parser/qwen3_coder_parser.py
Original file line number Diff line number Diff line change
Expand Up @@ -153,23 +153,18 @@ def parse_streaming_increment(self, new_text: str, tools: List[Tool]) -> Streami
if self.tool_call_end_token in self._buf:
end_pos = self._buf.find(self.tool_call_end_token)

# Add closing brace to complete the JSON object
# Streamed fragments never carry the outer closing brace, so the
# arguments are completed here: "{}" when nothing was streamed.
current_streamed = self.streamed_args_for_tool[self.current_tool_id]
if current_streamed:
# Count opening and closing braces to check if JSON is complete
open_braces = current_streamed.count("{")
close_braces = current_streamed.count("}")
if open_braces > close_braces:
calls.append(
ToolCallItem(
tool_index=self.current_tool_id,
name=None,
parameters="}",
)
)
self.streamed_args_for_tool[self.current_tool_id] = (
current_streamed + "}"
)
closing = "}" if current_streamed else "{}"
calls.append(
ToolCallItem(
tool_index=self.current_tool_id,
name=None,
parameters=closing,
)
)
self.streamed_args_for_tool[self.current_tool_id] = current_streamed + closing

# Complete the tool call
self._buf = self._buf[end_pos + len(self.tool_call_end_token) :]
Expand Down
56 changes: 56 additions & 0 deletions tests/unittest/llmapi/apps/test_tool_parsers.py
Original file line number Diff line number Diff line change
Expand Up @@ -1226,6 +1226,62 @@ def test_parse_streaming_increment_end_token_handling(
assert parser._buf == ""
assert parser._in_tool_call is False

@pytest.mark.parametrize("chunked", [False, True],
ids=["one_delta", "chunked"])
def test_streaming_zero_arg_tool(self, parser, chunked):
"""A call without parameter blocks streams "{}" as its arguments."""
tools = [
ChatCompletionToolsParam(
type="function",
function=FunctionDefinition(
name="get_time",
description="Get current time",
parameters={
"type": "object",
"properties": {},
},
),
)
]
deltas = [
"<tool_call>\n", "<function=get_time>\n", "</function>\n",
"</tool_call>"
]
if not chunked:
deltas = ["".join(deltas)]

calls = [
call for delta in deltas
for call in parser.parse_streaming_increment(delta, tools).calls
]

names = [c.name for c in calls if c.name]
assert names == ["get_time"]
params = "".join(c.parameters for c in calls)
assert params == "{}", f"Expected '{{}}', got {params!r}"
assert params == self.make_parser().detect_and_parse(
"".join(deltas), tools).calls[0].parameters

def test_streaming_closes_arguments_with_brace_in_value(
self, sample_tools, parser):
"""The closing brace is appended regardless of the streamed text."""
deltas = [
"<tool_call>\n<function=get_weather>\n",
"<parameter=location>a}b</parameter>\n",
"</function>\n</tool_call>",
]

calls = [
call
for delta in deltas for call in parser.parse_streaming_increment(
delta, sample_tools).calls
]

params = "".join(c.parameters for c in calls)
assert json.loads(params) == {"location": "a}b"}
assert params == self.make_parser().detect_and_parse(
"".join(deltas), sample_tools).calls[0].parameters

def test_parse_streaming_increment_multiple_tools_streaming(
self, sample_tools, parser):
"""Test streaming parser handles multiple tool calls."""
Expand Down
Loading