diff --git a/docs/source/developer-guide/telemetry.md b/docs/source/developer-guide/telemetry.md index c95b0674be6a..a8399f12a53b 100644 --- a/docs/source/developer-guide/telemetry.md +++ b/docs/source/developer-guide/telemetry.md @@ -192,7 +192,7 @@ unset or when the safety sanitizer rejects the runtime value. | `ray_placement_config.defer_workers_init` | `` | `value` | | | | `ray_placement_config.per_worker_gpu_share` | `Optional[float]` | `value` | | | | `ray_placement_config.placement_bundle_indices` | `Optional[List[List[int]]]` | `value` | | | -| `reasoning_parser` | `Optional[str]` | `categorical` | allowlist | `auto`, `deepseek-r1`, `laguna`, `qwen3`, `qwen3_5`, `minimax_m2`, `minimax_m2_append_think`, `nano-v3`, `gemma4`, `kimi_k2`, `kimi_k25` | +| `reasoning_parser` | `Optional[str]` | `categorical` | allowlist | `auto`, `deepseek-r1`, `poolside_v1`, `laguna`, `qwen3`, `qwen3_5`, `minimax_m2`, `minimax_m2_append_think`, `nano-v3`, `gemma4`, `kimi_k2`, `kimi_k25` | | `reorder_policy_config.policy_args.agent_inflight_seq_num` | `` | `value` | | | | `reorder_policy_config.policy_args.agent_percentage` | `` | `value` | | | | `reorder_policy_config.policy_name` | `Optional[Literal['AgentTree']]` | `categorical` | | `AgentTree` | diff --git a/tensorrt_llm/llmapi/llm_args.py b/tensorrt_llm/llmapi/llm_args.py index 1afb2cc25994..ca4c8af3dec8 100644 --- a/tensorrt_llm/llmapi/llm_args.py +++ b/tensorrt_llm/llmapi/llm_args.py @@ -4595,8 +4595,9 @@ class BaseLlmArgs(StrictBaseModel): default=None, description="The parser to separate reasoning content from output.", status="prototype", - telemetry=TelemetryField.categorical('auto', 'deepseek-r1', 'laguna', - 'qwen3', 'qwen3_5', 'minimax_m2', + telemetry=TelemetryField.categorical('auto', 'deepseek-r1', + 'poolside_v1', 'laguna', 'qwen3', + 'qwen3_5', 'minimax_m2', 'minimax_m2_append_think', 'nano-v3', 'gemma4', 'kimi_k2', 'kimi_k25')) diff --git a/tensorrt_llm/llmapi/reasoning_parser.py b/tensorrt_llm/llmapi/reasoning_parser.py index d6217e8e287d..4f847325748d 100644 --- a/tensorrt_llm/llmapi/reasoning_parser.py +++ b/tensorrt_llm/llmapi/reasoning_parser.py @@ -15,9 +15,10 @@ import json from abc import ABC, abstractmethod +from collections.abc import KeysView from dataclasses import dataclass from pathlib import Path -from typing import Any, Optional, Type +from typing import Any, ClassVar, Optional, Type from tensorrt_llm import logger @@ -28,6 +29,11 @@ class ReasoningParserResult: reasoning_content: str = "" +# Enough of the rendered prompt's tail to hold a prefilled marker and any +# trailing whitespace, without copying a prompt that may be very long. +_PROMPT_TAIL_CHARS = 64 + + def register_reasoning_parser(*keys: str, **default_kwargs): """Decorator that registers a BaseReasoningParser under one or more keys. @@ -42,6 +48,16 @@ class MyParser(BaseReasoningParser): """ def decorator(parser_cls: Type["BaseReasoningParser"]): + if parser_cls.resolves_thinking_from_prompt: + # Fail at import rather than per request: `resolve_prefilled_thinking` + # reads these off the class, and subclasses of parsers that only set + # them in `__init__` would otherwise raise inside the request path. + for attr in ("reasoning_start", "reasoning_end"): + if not isinstance(getattr(parser_cls, attr, None), str): + raise TypeError( + f"{parser_cls.__name__} sets " + f"resolves_thinking_from_prompt but does not define " + f"{attr} as a class attribute") for key in keys: ReasoningParserFactory._parsers[key] = (parser_cls, default_kwargs) return parser_cls @@ -69,7 +85,54 @@ def create_reasoning_parser( **default_kwargs) @classmethod - def keys(cls): + def resolves_thinking_from_prompt(cls, reasoning_parser: str) -> bool: + """Whether this parser selects its mode from the rendered prompt.""" + entry = cls._parsers.get(reasoning_parser.lower()) + return bool(entry) and entry[0].resolves_thinking_from_prompt + + @classmethod + def resolve_prefilled_thinking(cls, reasoning_parser: str, + prompt: str) -> Optional[bool]: + """Read the reasoning mode off the tail of a rendered prompt. + + These templates append the marker last, after the assistant header: + `...<|assistant|>` with thinking on, `...<|assistant|>` + with it off. The two are mutually exclusive and both land at the very + end, so whichever marker ends the prompt *is* the mode. + + Testing the suffix rather than the whole prompt is what makes this + correct, not merely cheap: prior assistant turns render with their own + `...` pairs, so a containment check would misfire on + every multi-turn request. `_PROMPT_TAIL_CHARS` is a copy bound, not a + semantic window; whitespace before the marker is unbounded, but more + trailing whitespace than that slice would push the marker out of view + and read as unresolved. + + Returns: + True - the template prefilled ``: reasoning is open. + False - the template prefilled ``: reasoning is already + closed, so all model output is content. + None - the mode cannot be determined from this prompt (unknown + parser, parser has not opted in, or neither marker is at + the tail). Callers must treat this as "ask elsewhere" + (e.g. the relayed disagg value), not as thinking-off. + """ + entry = cls._parsers.get(reasoning_parser.lower()) + if entry is None: + return None + parser_cls = entry[0] + if not parser_cls.resolves_thinking_from_prompt: + return None + # Only the tail matters, so avoid copying the whole prompt. + tail = prompt[-_PROMPT_TAIL_CHARS:].rstrip() + if tail.endswith(parser_cls.reasoning_end): + return False + if tail.endswith(parser_cls.reasoning_start): + return True + return None + + @classmethod + def keys(cls) -> KeysView[str]: return cls._parsers.keys() @classmethod @@ -92,6 +155,14 @@ class BaseReasoningParser(ABC): # when the request carries tools). needs_raw_special_tokens: bool = False + # Opt in on parsers whose template prefills the reasoning marker into the + # prompt and that select their mode from `enable_thinking`. Only those can + # have the mode resolved from the rendered prompt, and they must define + # both markers below. + resolves_thinking_from_prompt: ClassVar[bool] = False + reasoning_start: ClassVar[str] + reasoning_end: ClassVar[str] + def __init__(self, *, chat_template_kwargs: Optional[dict[str, Any]] = None) -> None: @@ -126,7 +197,6 @@ def parse_delta(self, delta_text: str) -> ReasoningParserResult: @register_reasoning_parser("deepseek-r1", reasoning_at_start=True) -@register_reasoning_parser("laguna") @register_reasoning_parser("qwen3") # Qwen3.5 (and forced-thinking Qwen3 variants) use a chat template that # pre-injects `\n` into the assistant prompt prefix, so the model @@ -289,6 +359,34 @@ def finish(self) -> ReasoningParserResult: return self._parser.finish() +@register_reasoning_parser("poolside_v1", "laguna") +class PoolsideV1ReasoningParser(DeepSeekV4ReasoningParser): + """Poolside Laguna models, which prefill the marker the same way. + + The family's templates disagree on the `enable_thinking` default, so the + mode is resolved from the rendered prompt rather than from a constant. + `laguna` stays as an alias of `poolside_v1` for existing deployments. + """ + + resolves_thinking_from_prompt = True + + def __init__( + self, + *, + chat_template_kwargs: Optional[dict[str, Any]] = None, + ) -> None: + super().__init__(chat_template_kwargs=chat_template_kwargs) + kwargs = chat_template_kwargs or {} + if kwargs.get("thinking") is None and kwargs.get( + "enable_thinking") is None: + # Mode unresolved (offline LLM API, disagg generation server, + # add_generation_prompt=false). Keep splitting on a `` the + # model emits itself, as these models do in multi-turn and tools. + self._parser = DeepSeekR1Parser( + reasoning_at_start=False, + chat_template_kwargs=chat_template_kwargs) + + @register_reasoning_parser("minimax_m3") class MiniMaxM3ReasoningParser(DeepSeekR1Parser): """Reasoning parser for MiniMax-M3. @@ -344,7 +442,7 @@ def parse(self, text: str) -> ReasoningParserResult: "qwen3_next": "qwen3", "deepseek_v3": "deepseek-r1", "deepseek_v32": "deepseek-r1", - "laguna": "laguna", + "laguna": "poolside_v1", "deepseek_v4": "deepseek_v4", "nemotron_h": "nemotron-v3", "nemotron_h_puzzle": "nemotron-v3", diff --git a/tensorrt_llm/serve/openai_protocol.py b/tensorrt_llm/serve/openai_protocol.py index 0cce73d0b5f8..c9405ccbfa39 100644 --- a/tensorrt_llm/serve/openai_protocol.py +++ b/tensorrt_llm/serve/openai_protocol.py @@ -212,6 +212,10 @@ class DisaggregatedParams(OpenAIBaseModel): # Orchestrator -> context-worker instruction: return prompt_token_ids as a # base64 int32 buffer (prompt_token_ids_b64) instead of a JSON int array. return_prompt_token_ids_b64: bool = False + # Context worker -> generation worker: the reasoning mode the context + # worker read off the prompt it rendered. The generation worker only sees + # prompt_token_ids, so it cannot resolve this for itself. + resolved_thinking: Optional[bool] = None class ConversationParams(OpenAIBaseModel): diff --git a/tensorrt_llm/serve/openai_server.py b/tensorrt_llm/serve/openai_server.py index 952a041f7011..1c38c3c90634 100644 --- a/tensorrt_llm/serve/openai_server.py +++ b/tensorrt_llm/serve/openai_server.py @@ -180,6 +180,30 @@ async def route_handler(request: Request): TIMEOUT_KEEP_ALIVE = 5 # seconds. +_warned_unresolvable_thinking = False + + +def _warn_unresolvable_thinking_once(reasoning_parser: str) -> None: + """Warn when a generation worker cannot learn the reasoning mode. + + Nothing was rendered here and the context worker relayed no value, so + every request on this deployment is parsed in a possibly wrong mode with + nothing in the response saying so. Reachable via a rolling upgrade where + the context worker predates `resolved_thinking`, a hand-crafted + `generation_only` request, or an orchestration path that does not go + through `_get_ctx_request`. + """ + global _warned_unresolvable_thinking + if _warned_unresolvable_thinking: + return + _warned_unresolvable_thinking = True + logger.warning( + f"Reasoning parser {reasoning_parser!r} resolves its mode from the " + "rendered prompt, but this request had neither a rendered prompt nor " + "a relayed mode from a context worker. Reasoning content will not be " + "separated correctly. Check that the context worker is running a " + "build that relays 'resolved_thinking'.") + def _configure_parser_special_token_decoding( sampling_params: SamplingParams, reasoning_parser_name: Optional[str], @@ -1533,6 +1557,9 @@ async def chat_stream_generator( gather_generation_logits, reasoning_parser=self.generator.args.reasoning_parser, backend=self.generator.args.backend) + # Pre-render, so the mode may be unresolved. Safe because the + # boundary lookup reads the parser class attributes, not the + # branch `__init__` picked. add_thinking_budget_logits_processor( sampling_params, reasoning_parser=self.generator.args.reasoning_parser, @@ -1584,6 +1611,7 @@ async def chat_stream_generator( base64.b64decode(request.prompt_token_ids_b64), dtype=np.int32).tolist() + rendered_prompt = None if request.prompt_token_ids is not None: prompt = request.prompt_token_ids else: @@ -1601,6 +1629,8 @@ async def chat_stream_generator( ) prompt, (mm_data, mm_embeddings) = await asyncio.gather( prompt_task, mm_coroutines) + if isinstance(prompt, str): + rendered_prompt = prompt prompt = prompt_inputs(prompt) if request.prompt_token_ids is not None: @@ -1620,6 +1650,31 @@ async def chat_stream_generator( prompt["mm_processor_kwargs"] = request.mm_processor_kwargs postproc_args.reasoning_parser = self.generator.args.reasoning_parser + # Templates that prefill / leave the marker in the + # prompt, so the request kwargs alone cannot tell the parser which + # mode was rendered. Take it from the prompt instead. + if postproc_args.reasoning_parser and ( + ReasoningParserFactory.resolves_thinking_from_prompt( + postproc_args.reasoning_parser)): + thinking = None + if rendered_prompt and request.add_generation_prompt: + thinking = ReasoningParserFactory.resolve_prefilled_thinking( + postproc_args.reasoning_parser, rendered_prompt) + if thinking is None and request.disaggregated_params is not None: + # Generation worker: it never rendered, so use the mode the + # context worker resolved and relayed. + thinking = request.disaggregated_params.resolved_thinking + if thinking is None: + _warn_unresolvable_thinking_once( + postproc_args.reasoning_parser) + if thinking is not None: + # Both keys, because the parser ORs them: leaving a stale + # `thinking` in place would override what we resolved. + postproc_args.chat_template_kwargs = { + **(request.chat_template_kwargs or {}), + "thinking": thinking, + "enable_thinking": thinking, + } postproc_args.tool_parser = self.tool_parser postproc_args.tool_call_id_type = self.tool_call_id_type if conversation and conversation[-1].get( diff --git a/tensorrt_llm/serve/postprocess_handlers.py b/tensorrt_llm/serve/postprocess_handlers.py index 9b9b27cd90ca..364f1cbf4e8f 100644 --- a/tensorrt_llm/serve/postprocess_handlers.py +++ b/tensorrt_llm/serve/postprocess_handlers.py @@ -416,6 +416,17 @@ def chat_response_post_processor( tool_calls=tool_calls) disaggregated_params = to_disaggregated_params( output.disaggregated_params) + if (disaggregated_params is not None and args.chat_template_kwargs + and args.reasoning_parser + and ReasoningParserFactory.resolves_thinking_from_prompt( + args.reasoning_parser)): + # Relay the mode we resolved from the rendered prompt; the + # generation worker never renders and so cannot resolve it. Gated + # on the parser opting in, so we never overwrite a mode another + # parser derives from the caller's own kwargs. + resolved = args.chat_template_kwargs.get("enable_thinking") + if resolved is not None: + disaggregated_params.resolved_thinking = resolved choice = ChatCompletionResponseChoice( index=output.index, message=message, diff --git a/tensorrt_llm/usage/llm_args_golden_manifest.json b/tensorrt_llm/usage/llm_args_golden_manifest.json index 38436f22514a..2146edc5a267 100644 --- a/tensorrt_llm/usage/llm_args_golden_manifest.json +++ b/tensorrt_llm/usage/llm_args_golden_manifest.json @@ -1367,6 +1367,7 @@ "allowed_values": [ "auto", "deepseek-r1", + "poolside_v1", "laguna", "qwen3", "qwen3_5", diff --git a/tests/integration/test_lists/test-db/l0_cpu.yml b/tests/integration/test_lists/test-db/l0_cpu.yml index 599698d4ffd7..fb7f55b2d31b 100644 --- a/tests/integration/test_lists/test-db/l0_cpu.yml +++ b/tests/integration/test_lists/test-db/l0_cpu.yml @@ -38,6 +38,7 @@ l0_cpu: - unittest/inputs - unittest/llmapi/apps/test_chat_utils.py - unittest/llmapi/apps/test_harmony_channel_validation.py + - unittest/llmapi/apps/test_reasoning_prompt_resolution.py - unittest/llmapi/apps/test_tool_parsers.py - unittest/llmapi/test_bench_async.py - unittest/llmapi/test_additional_model_outputs.py -m "gpu1" diff --git a/tests/unittest/api_stability/references/trtllm_serve_cli.yaml b/tests/unittest/api_stability/references/trtllm_serve_cli.yaml index ee3a2dd7cb40..f5a914bb0836 100644 --- a/tests/unittest/api_stability/references/trtllm_serve_cli.yaml +++ b/tests/unittest/api_stability/references/trtllm_serve_cli.yaml @@ -377,7 +377,7 @@ commands: flags: - "--post_processor_hook" reasoning_parser: - type: Choice(['auto', 'deepseek-r1', 'deepseek_v4', 'gemma4', 'kimi_k2', 'kimi_k25', 'kimi_k3', 'laguna', 'minimax_m2', 'minimax_m2_append_think', 'minimax_m3', 'nano-v3', 'nemotron-v3', 'qwen3', 'qwen3_5']) + type: Choice(['auto', 'deepseek-r1', 'deepseek_v4', 'gemma4', 'kimi_k2', 'kimi_k25', 'kimi_k3', 'laguna', 'minimax_m2', 'minimax_m2_append_think', 'minimax_m3', 'nano-v3', 'nemotron-v3', 'poolside_v1', 'qwen3', 'qwen3_5']) default: null status: prototype required: false diff --git a/tests/unittest/llmapi/apps/test_reasoning_prompt_resolution.py b/tests/unittest/llmapi/apps/test_reasoning_prompt_resolution.py new file mode 100644 index 000000000000..30a44aa29aa9 --- /dev/null +++ b/tests/unittest/llmapi/apps/test_reasoning_prompt_resolution.py @@ -0,0 +1,207 @@ +# SPDX-FileCopyrightText: Copyright (c) 2026 NVIDIA CORPORATION & AFFILIATES. All rights reserved. +# SPDX-License-Identifier: Apache-2.0 +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +"""End-to-end checks for resolving reasoning mode from the rendered prompt. + +`openai_server` reads the mode off the rendered prompt and writes it into +`ChatPostprocArgs.chat_template_kwargs`; `postprocess_handlers` builds the +parser from that field. The two halves live in different modules, so this +pins the contract between them against a template that prefills the marker. +""" + +import jinja2 +import pytest + +from tensorrt_llm.llmapi import DisaggregatedParams as LlmDisaggregatedParams +from tensorrt_llm.llmapi.reasoning_parser import ReasoningParserFactory +from tensorrt_llm.serve.postprocess_handlers import ( + ChatPostprocArgs, + apply_reasoning_parser, + chat_response_post_processor, +) + +pytestmark = pytest.mark.cpu_only + +# Mirrors the shape of the Laguna templates: the marker goes into the prompt, +# so it never appears in the output and the request kwargs cannot reveal it. +PREFILLING_TEMPLATE = ( + "{%- set enable_thinking = enable_thinking | default(true) -%}" + "{%- for message in messages -%}" + "{{- '<|' + message['role'] + '|>' + message['content'] -}}" + "{%- endfor -%}" + "{%- if add_generation_prompt -%}" + "{{- '<|assistant|>' -}}" + "{%- if enable_thinking -%}{{- '' -}}" + "{%- else -%}{{- '' -}}{%- endif -%}" + "{%- endif -%}" +) + +MESSAGES = [{"role": "user", "content": "hi"}] + + +def render(chat_template_kwargs: dict | None, add_generation_prompt: bool = True) -> str: + template = jinja2.Environment().from_string(PREFILLING_TEMPLATE) + return template.render( + messages=MESSAGES, + add_generation_prompt=add_generation_prompt, + **(chat_template_kwargs or {}), + ) + + +def build_args( + rendered_prompt: str | None, request_kwargs: dict | None, add_generation_prompt: bool = True +) -> ChatPostprocArgs: + """Mirror what `openai_server` does after rendering the prompt.""" + args = ChatPostprocArgs( + role="assistant", model="test-model", chat_template_kwargs=request_kwargs + ) + args.reasoning_parser = "poolside_v1" + args.num_prompt_tokens = 3 + if args.reasoning_parser and rendered_prompt and add_generation_prompt: + thinking = ReasoningParserFactory.resolve_prefilled_thinking( + args.reasoning_parser, rendered_prompt + ) + if thinking is not None: + args.chat_template_kwargs = { + **(request_kwargs or {}), + "thinking": thinking, + "enable_thinking": thinking, + } + return args + + +@pytest.mark.parametrize( + ("request_kwargs", "reasoning", "content"), + [ + (None, "hidden", "visible"), + ({"enable_thinking": True}, "hidden", "visible"), + ], +) +def test_thinking_template_splits_reasoning( + request_kwargs: dict[str, bool] | None, reasoning: str, content: str +) -> None: + """A bare request must still land in the mode the template rendered.""" + prompt = render(request_kwargs) + assert prompt.endswith("") + + args = build_args(prompt, request_kwargs) + got_content, got_reasoning = apply_reasoning_parser( + args, 0, "hiddenvisible", streaming=False + ) + + assert got_content == content + assert got_reasoning == reasoning + + +def test_non_thinking_template_keeps_everything_as_content() -> None: + prompt = render({"enable_thinking": False}) + assert prompt.endswith("") + + args = build_args(prompt, {"enable_thinking": False}) + content, reasoning = apply_reasoning_parser(args, 0, "visible", streaming=False) + + assert content == "visible" + assert not reasoning + + +def test_streaming_path_uses_the_same_resolved_mode() -> None: + prompt = render(None) + args = build_args(prompt, None) + + deltas = [ + apply_reasoning_parser(args, 0, chunk, streaming=True) + for chunk in ("hid", "denvis", "ible") + ] + + assert "".join(c for c, _ in deltas if c) == "visible" + assert "".join(r for _, r in deltas if r) == "hidden" + + +class _FakeOutput: + """Minimal stand-in for one entry of `GenerationResultBase.outputs`.""" + + def __init__(self, text: str) -> None: + self.index = 0 + self.text = text + self.token_ids = [] + self.length = 0 + self.finish_reason = "stop" + self.stop_reason = None + self.logprobs = None + self.disaggregated_params = LlmDisaggregatedParams(request_type="context_only") + + +class _FakeResult: + def __init__(self, text: str) -> None: + self.outputs = [_FakeOutput(text)] + self.prompt_token_ids = [1, 2, 3] + self.cached_tokens = 0 + + +@pytest.mark.parametrize( + ("template_kwargs", "expected"), + [(None, True), ({"enable_thinking": False}, False)], +) +def test_context_worker_stamps_the_resolved_mode( + template_kwargs: dict[str, bool] | None, expected: bool +) -> None: + """Calls the real handler, so dropping the stamp fails this test.""" + args = build_args(render(template_kwargs), template_kwargs) + + response = chat_response_post_processor(_FakeResult("hiddenvisible"), args) + + assert response.choices[0].disaggregated_params.resolved_thinking is expected + + +def test_context_worker_does_not_stamp_for_other_parsers() -> None: + """Only parsers that resolve from the prompt should relay a mode.""" + args = build_args(render(None), None) + args.reasoning_parser = "deepseek_v4" + + response = chat_response_post_processor(_FakeResult("visible"), args) + + assert response.choices[0].disaggregated_params.resolved_thinking is None + + +@pytest.mark.parametrize("relayed", [True, False]) +def test_relayed_mode_drives_the_parser(relayed: bool) -> None: + """Contract the generation worker relies on, given a relayed value. + + The generation-side block lives in `openai_server` and needs a live + server, so this pins what it feeds the parser rather than the wiring. + """ + args = ChatPostprocArgs( + role="assistant", + model="test-model", + chat_template_kwargs={"thinking": relayed, "enable_thinking": relayed}, + ) + args.reasoning_parser = "poolside_v1" + content, reasoning = apply_reasoning_parser(args, 0, "hiddenvisible", streaming=False) + + if relayed: + assert (content, reasoning) == ("visible", "hidden") + else: + assert content == "hiddenvisible" + assert not reasoning + + +def test_unrendered_prompt_falls_back_without_crashing() -> None: + """Disagg generation servers get `prompt_token_ids` and never render.""" + args = build_args(None, None) + + content, reasoning = apply_reasoning_parser(args, 0, "abc", streaming=False) + + # Unresolved, so the pre-existing split on an emitted `` applies. + assert content == "c" + assert reasoning == "b" diff --git a/tests/unittest/llmapi/test_reasoning_parser.py b/tests/unittest/llmapi/test_reasoning_parser.py index cf9da1d1eb58..320d20ef1f1d 100644 --- a/tests/unittest/llmapi/test_reasoning_parser.py +++ b/tests/unittest/llmapi/test_reasoning_parser.py @@ -18,9 +18,9 @@ import pytest -from tensorrt_llm.llmapi.reasoning_parser import (NemotronV3ReasoningParser, - ReasoningParserFactory, - resolve_auto_reasoning_parser) +from tensorrt_llm.llmapi.reasoning_parser import ( + MODEL_TYPE_TO_REASONING_PARSER, NemotronV3ReasoningParser, + ReasoningParserFactory, resolve_auto_reasoning_parser) pytestmark = pytest.mark.cpu_only @@ -353,38 +353,225 @@ def test_qwen3_reasoning_parser_stream(delta_texts: list, content: list, @pytest.mark.parametrize(("text", "content", "reasoning_context"), [ - ("abc", "c", "b"), - ("ab", "b", "a"), - ("a", "", "a"), - ("a", "a", ""), - ("", "", ""), + (f"hidden{R1_END}visible", "visible", "hidden"), + (f"{R1_END}visible", "visible", ""), + (R1_END, "", ""), + ("unterminated", "", "unterminated"), ]) -def test_laguna_reasoning_parser(text: str, content: str, - reasoning_context: str): - reasoning_parser = ReasoningParserFactory.create_reasoning_parser("laguna") +def test_poolside_v1_reasoning_parser_when_thinking( + text: str, content: str, reasoning_context: str) -> None: + reasoning_parser = ReasoningParserFactory.create_reasoning_parser( + "poolside_v1", {"enable_thinking": True}) result = reasoning_parser.parse(text) assert result.content == content assert result.reasoning_content == reasoning_context +@pytest.mark.parametrize("chat_template_kwargs", [ + { + "enable_thinking": False + }, + { + "thinking": False + }, +]) +def test_poolside_v1_reasoning_parser_when_not_thinking( + chat_template_kwargs: dict[str, bool]) -> None: + """Output is all visible content when thinking is off. + + The template emits `` into the prompt in that mode, so the model + output carries no markers at all. + """ + reasoning_parser = ReasoningParserFactory.create_reasoning_parser( + "poolside_v1", chat_template_kwargs) + result = reasoning_parser.parse("visible") + assert result.content == "visible" + assert result.reasoning_content == "" + + @pytest.mark.parametrize(("delta_texts", "content", "reasoning_context"), [ - (["a", "lr", "b"], ["", "r", "b"], ["a", "l", ""]), - (["ab"], ["", "b"], ["", "a"]), - (["ab"], ["", "b"], ["a", ""]), - (["", "ab"], ["", "b"], ["", "a"]), - (["a", "b"], ["", "b"], ["a", ""]), - (["ab"], ["", "", "b" - ], ["a", "", ""]), + (["a", f"l{R1_END}r", "b"], ["", "r", "b"], ["a", "l", ""]), + (["ab"], ["", "b"], ["a", ""]), + (["", f"a{R1_END}b"], ["", "b"], ["", "a"]), + ([f"a{R1_END}", "b"], ["", "b"], ["a", ""]), + (["ab"], ["", "", "b"], ["a", "", ""]), ]) -def test_laguna_reasoning_parser_stream(delta_texts: list, content: list, - reasoning_context: list): - reasoning_parser = ReasoningParserFactory.create_reasoning_parser("laguna") +def test_poolside_v1_reasoning_parser_stream( + delta_texts: list[str], content: list[str], + reasoning_context: list[str]) -> None: + reasoning_parser = ReasoningParserFactory.create_reasoning_parser( + "poolside_v1", {"enable_thinking": True}) for i, delta_text in enumerate(delta_texts): result = reasoning_parser.parse_delta(delta_text) assert result.content == content[i] assert result.reasoning_content == reasoning_context[i] +def test_laguna_alias_of_poolside_v1() -> None: + """`laguna` is kept as an alias so existing deployments keep working.""" + laguna = ReasoningParserFactory.create_reasoning_parser("laguna") + poolside = ReasoningParserFactory.create_reasoning_parser("poolside_v1") + assert type(laguna) is type(poolside) + assert MODEL_TYPE_TO_REASONING_PARSER["laguna"] == "poolside_v1" + + +@pytest.mark.parametrize("parser", ["poolside_v1", "laguna"]) +@pytest.mark.parametrize(("tail", "expected"), [(R1_START, True), + (R1_END, False)]) +def test_alias_resolves_identically(parser: str, tail: str, + expected: bool) -> None: + assert ReasoningParserFactory.resolve_prefilled_thinking( + parser, f"{tail}") is expected + + +@pytest.mark.parametrize(("prompt", "expected"), [ + (f"hi\n{R1_START}", True), + (f"hi\n{R1_END}", False), + (f"hi\n\n{R1_END}", False), + (f"hi\n{R1_START}\n", True), + ("hi\n", None), + (f"a prompt quoting {R1_END} mid-turn\n", None), +]) +def test_resolve_prefilled_thinking(prompt: str, expected: bool | None) -> None: + assert ReasoningParserFactory.resolve_prefilled_thinking( + "poolside_v1", prompt) is expected + + +def test_resolve_prefilled_thinking_unknown_parser() -> None: + assert ReasoningParserFactory.resolve_prefilled_thinking( + "not-a-parser", R1_START) is None + + +@pytest.mark.parametrize(("tail", "expected"), [(R1_START, True), + (R1_END, False)]) +def test_resolve_prefilled_thinking_opted_in(tail: str, expected: bool) -> None: + assert ReasoningParserFactory.resolve_prefilled_thinking( + "poolside_v1", f"{tail}") is expected + + +@pytest.mark.parametrize("parser", ["poolside_v1", "laguna"]) +@pytest.mark.parametrize(("text", "content", "reasoning_context"), [ + ("abc", "c", "b"), + ("ab", "b", "a"), + ("a", "", "a"), + ("a", "a", ""), +]) +def test_unresolved_mode_keeps_previous_behaviour( + parser: str, text: str, content: str, reasoning_context: str) -> None: + """With no mode supplied, split on a `` the model emitted itself. + + Nothing resolves the mode for the offline LLM API, the disaggregated + generation server, or `add_generation_prompt=false`. Falling back to + `IdentityReasoningParser` there would drop the split these models still + need, since they emit the tags in multi-turn and tool-calling flows. + """ + reasoning_parser = ReasoningParserFactory.create_reasoning_parser(parser) + + result = reasoning_parser.parse(text) + + assert result.content == content + assert result.reasoning_content == reasoning_context + + +@pytest.mark.parametrize("kwargs", [{ + "enable_thinking": False +}, { + "thinking": False +}]) +def test_explicit_no_thinking_still_uses_identity( + kwargs: dict[str, bool]) -> None: + """An explicit off is a resolved mode, so the template closed reasoning.""" + reasoning_parser = ReasoningParserFactory.create_reasoning_parser( + "poolside_v1", kwargs) + + result = reasoning_parser.parse("abc") + + assert result.content == "abc" + assert result.reasoning_content == "" + + +def test_clearing_only_enable_thinking_leaves_reasoning_on() -> None: + """Why the server writes both keys: the parser ORs them. + + If it cleared only `enable_thinking`, a `thinking=True` the caller sent + would still force reasoning mode and the answer would land in the wrong + field. This pins the OR so that shortcut cannot be taken silently. + """ + reasoning_parser = ReasoningParserFactory.create_reasoning_parser( + "poolside_v1", { + "thinking": True, + "enable_thinking": False + }) + + result = reasoning_parser.parse(f"hidden{R1_END}visible") + + assert result.reasoning_content == "hidden" + assert result.content == "visible" + + +@pytest.mark.parametrize("sent", [{ + "thinking": True +}, { + "enable_thinking": True +}, { + "thinking": True, + "enable_thinking": True +}]) +def test_resolved_mode_overrides_whatever_the_caller_sent( + sent: dict[str, bool]) -> None: + """Both keys written, as the server does, so the resolved mode wins.""" + resolved = {**sent, "thinking": False, "enable_thinking": False} + reasoning_parser = ReasoningParserFactory.create_reasoning_parser( + "poolside_v1", resolved) + + result = reasoning_parser.parse("visible") + + assert result.content == "visible" + assert result.reasoning_content == "" + + +@pytest.mark.parametrize("parser", [ + "deepseek-r1", "deepseek_v4", "qwen3", "qwen3_5", "minimax_m2", + "minimax_m3", "nemotron-v3", "nano-v3", "gemma4", "kimi_k2", "kimi_k25" +]) +def test_resolve_prefilled_thinking_requires_opt_in(parser: str) -> None: + """Parsers that have not opted in must never be resolved from the prompt. + + `deepseek_v4` shares the base class and `nemotron-v3` / `nano-v3` also read + `enable_thinking`, so without the flag they would silently pick up a mode + the server inferred. + """ + # Otherwise a typo or a dropped registration passes vacuously, since an + # unknown name also resolves to None. + assert parser in ReasoningParserFactory.keys() + for tail in (R1_START, R1_END, ""): + assert ReasoningParserFactory.resolve_prefilled_thinking( + parser, f"{tail}") is None + + +@pytest.mark.parametrize( + ("prompt_tail", "model_output", "content", "reasoning_content"), [ + (R1_START, f"hidden{R1_END}visible", "visible", "hidden"), + (R1_END, "visible", "visible", ""), + ]) +def test_poolside_v1_mode_resolved_from_prompt(prompt_tail: str, + model_output: str, content: str, + reasoning_content: str) -> None: + """Mirror the server path: resolve from the prompt, then parse. + + A request that sends no chat template kwargs must still land in the mode + the template actually rendered. + """ + prompt = f"hi\n{prompt_tail}" + thinking = ReasoningParserFactory.resolve_prefilled_thinking( + "poolside_v1", prompt) + reasoning_parser = ReasoningParserFactory.create_reasoning_parser( + "poolside_v1", {"enable_thinking": thinking}) + result = reasoning_parser.parse(model_output) + assert result.content == content + assert result.reasoning_content == reasoning_content + + TOOL_CALL = "" TOOL_CALL_END = "" @@ -750,13 +937,13 @@ def test_auto_detect_gemma4(tmp_path): def test_auto_detect_laguna(tmp_path): - """Laguna model → 'laguna' parser.""" + """Laguna model → 'poolside_v1' parser.""" model_dir = str(tmp_path / "Laguna") os.makedirs(model_dir) _write_config(model_dir, "laguna") result = resolve_auto_reasoning_parser(model_dir) - assert result == "laguna" + assert result == "poolside_v1" @pytest.mark.parametrize("model_type", ["nemotron_h", "nemotron_h_puzzle"])