From 2caa9cff979034b13f047a8b7e0ff89a06730507 Mon Sep 17 00:00:00 2001 From: Sihan Wang Date: Mon, 3 Aug 2026 20:56:15 +0000 Subject: [PATCH 1/2] trtllm: forward routing.priority to the engine waiting queue generate_locally read `priority` off the top level of the request, but the Rust frontend puts it at `routing.priority` (PreprocessedRequest.routing is not flattened) -- the sibling `routing.dp_rank` two lines up is read correctly. So every real request fell back to DEFAULT_REQUEST_PRIORITY and per-request priority never reached TRT-LLM, even with scheduler_config.waiting_queue_policy=priority set on the worker. The top-level key stays the health-check path, which pins 1.0. deepapi scales the [0.0, 1.0] engine priority to an integer before putting it on x-dynamo-request-priority, because nvext.agent_hints.priority is typed as an i32 while TRT-LLM rejects anything outside [0.0, 1.0]. Divide it back and clamp, so a malformed value degrades instead of erroring the request. NOT YET RUN: tensorrt_llm is not importable on the host these were written on, so the new tests need a run in the build container before merge. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QqzQrZR2qrvxM6DsxqGciu --- .../trtllm/request_handlers/handler_base.py | 18 ++++- .../trtllm/tests/test_trtllm_handler_base.py | 76 ++++++++++++++++++- 2 files changed, 90 insertions(+), 4 deletions(-) diff --git a/components/src/dynamo/trtllm/request_handlers/handler_base.py b/components/src/dynamo/trtllm/request_handlers/handler_base.py index a7fca7289ada..d0c1dc29b941 100644 --- a/components/src/dynamo/trtllm/request_handlers/handler_base.py +++ b/components/src/dynamo/trtllm/request_handlers/handler_base.py @@ -64,6 +64,12 @@ logger = logging.getLogger(__name__) +# deepapi scales the engine's [0.0, 1.0] priority to an integer before putting +# it on the x-dynamo-request-priority header, because the frontend types +# nvext.agent_hints.priority as an i32. Divide it back before TRT-LLM, which +# rejects anything outside [0.0, 1.0]. +DEEPINFRA_PRIORITY_SCALE = 10 + class TRTLLMEnginePauseController: """Adapts TRT-LLM sleep/wake to the standard pause controller interface. @@ -1115,7 +1121,17 @@ async def _generate_locally_impl( ) # Priority is a float in [0.0, 1.0]; health checks use 1.0. Default is 0.5. - priority = request.get("priority", DEFAULT_REQUEST_PRIORITY) + # Real requests carry it in the routing hints, scaled to an integer by + # DEEPINFRA_PRIORITY_SCALE because agent_hints.priority (and the + # x-dynamo-request-priority header feeding it) is typed as an i32. + priority = request.get("priority") + if priority is None: + scaled = routing.get("priority") if routing else None + priority = ( + DEFAULT_REQUEST_PRIORITY + if scaled is None + else min(1.0, max(0.0, scaled / DEEPINFRA_PRIORITY_SCALE)) + ) cache_salt = request_cache_salt(request) try: diff --git a/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py b/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py index f14e407b2bf4..e67acbdbcabe 100644 --- a/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py +++ b/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py @@ -716,9 +716,9 @@ class TestHealthCheckPriority: """Verify generate_locally forwards the correct priority to generate_async. Health check requests (built by TrtllmHealthCheckPayload) must reach - the TRT-LLM engine at priority=1.0. Regular inference requests - (built by the Rust frontend as PreprocessedRequest, which has no - priority field) must fall back to DEFAULT_REQUEST_PRIORITY (0.5). + the TRT-LLM engine at priority=1.0. Regular inference requests carry it + in routing hints, scaled by DEEPINFRA_PRIORITY_SCALE; with no routing + priority at all they fall back to DEFAULT_REQUEST_PRIORITY (0.5). """ def _make_handler(self) -> HandlerBase: @@ -803,6 +803,76 @@ async def test_regular_request_gets_default_priority(self): _, kwargs = handler.engine.llm.generate_async.call_args assert kwargs["priority"] == DEFAULT_REQUEST_PRIORITY + @pytest.mark.asyncio + @pytest.mark.parametrize( + "scaled,expected", [(6, 0.6), (10, 1.0), (5, 0.5), (0, 0.0)] + ) + async def test_routing_priority_is_unscaled(self, scaled, expected): + """routing.priority is an integer 0-10; TRT-LLM wants [0.0, 1.0].""" + handler = self._make_handler() + generation_result = self._make_mock_generation_result() + handler.engine.llm.generate_async = MagicMock(return_value=generation_result) + + request = { + "token_ids": [1, 2, 3], + "stop_conditions": {"max_tokens": 10}, + "sampling_options": {"temperature": 0.7}, + "routing": {"priority": scaled}, + } + + chunks = [ + c async for c in handler.generate_locally(request, self._make_context()) + ] + assert len(chunks) > 0 + + handler.engine.llm.generate_async.assert_called_once() + _, kwargs = handler.engine.llm.generate_async.call_args + assert kwargs["priority"] == pytest.approx(expected) + + @pytest.mark.asyncio + async def test_routing_priority_clamped_into_range(self): + """A value out of range must not reach TRT-LLM, which rejects it.""" + handler = self._make_handler() + generation_result = self._make_mock_generation_result() + handler.engine.llm.generate_async = MagicMock(return_value=generation_result) + + request = { + "token_ids": [1, 2, 3], + "stop_conditions": {"max_tokens": 10}, + "sampling_options": {"temperature": 0.7}, + "routing": {"priority": 99}, + } + + chunks = [ + c async for c in handler.generate_locally(request, self._make_context()) + ] + assert len(chunks) > 0 + + handler.engine.llm.generate_async.assert_called_once() + _, kwargs = handler.engine.llm.generate_async.call_args + assert kwargs["priority"] == 1.0 + + @pytest.mark.asyncio + async def test_health_check_priority_wins_over_routing(self): + """The top-level health-check priority is not overridden by routing.""" + handler = self._make_handler() + generation_result = self._make_mock_generation_result() + handler.engine.llm.generate_async = MagicMock(return_value=generation_result) + + request = TrtllmHealthCheckPayload( + disaggregation_mode=DisaggregationMode.AGGREGATED, + ).to_dict() + request["routing"] = {"priority": 0} + + chunks = [ + c async for c in handler.generate_locally(request, self._make_context()) + ] + assert len(chunks) > 0 + + handler.engine.llm.generate_async.assert_called_once() + _, kwargs = handler.engine.llm.generate_async.call_args + assert kwargs["priority"] == 1.0 + @pytest.mark.asyncio async def test_routing_cache_salt_forwarded_to_generate_async(self): handler = self._make_handler() From 70b251e8b3ae08ea937bdee5e49bc178a82651b5 Mon Sep 17 00:00:00 2001 From: Sihan Wang Date: Mon, 3 Aug 2026 21:06:40 +0000 Subject: [PATCH 2/2] trtllm: pass routing.priority through unmodified Drops the DEEPINFRA_PRIORITY_SCALE divide added in the previous commit. deepapi now sends TRT-LLM's own rails (1.0 urgent, 0.0 reserved for flex, default omitted) rather than a scaled integer, because the integer-typed header cannot express the graded values the scale existed to carry. That makes this a plain pass-through with a clamp, and removes a deepinfra-local encoding this file would otherwise have to keep explaining across rebases. The clamp stays: TRT-LLM rejects anything outside [0.0, 1.0] outright, and degrading is better than failing the request. Co-Authored-By: Claude Opus 5 (1M context) Claude-Session: https://claude.ai/code/session_01QqzQrZR2qrvxM6DsxqGciu --- .../trtllm/request_handlers/handler_base.py | 19 ++++++------------- .../trtllm/tests/test_trtllm_handler_base.py | 14 ++++++-------- 2 files changed, 12 insertions(+), 21 deletions(-) diff --git a/components/src/dynamo/trtllm/request_handlers/handler_base.py b/components/src/dynamo/trtllm/request_handlers/handler_base.py index d0c1dc29b941..1ee615f6e508 100644 --- a/components/src/dynamo/trtllm/request_handlers/handler_base.py +++ b/components/src/dynamo/trtllm/request_handlers/handler_base.py @@ -64,12 +64,6 @@ logger = logging.getLogger(__name__) -# deepapi scales the engine's [0.0, 1.0] priority to an integer before putting -# it on the x-dynamo-request-priority header, because the frontend types -# nvext.agent_hints.priority as an i32. Divide it back before TRT-LLM, which -# rejects anything outside [0.0, 1.0]. -DEEPINFRA_PRIORITY_SCALE = 10 - class TRTLLMEnginePauseController: """Adapts TRT-LLM sleep/wake to the standard pause controller interface. @@ -1120,17 +1114,16 @@ async def _generate_locally_impl( f"Using dynamo router dp_rank={dp_rank} for TRTLLM attention DP scheduling" ) - # Priority is a float in [0.0, 1.0]; health checks use 1.0. Default is 0.5. - # Real requests carry it in the routing hints, scaled to an integer by - # DEEPINFRA_PRIORITY_SCALE because agent_hints.priority (and the - # x-dynamo-request-priority header feeding it) is typed as an i32. + # Priority is a float in [0.0, 1.0]; health checks use 1.0. Default is + # 0.5. Real requests carry it in the routing hints. Clamp rather than + # let TRT-LLM reject the request outright on an out-of-range value. priority = request.get("priority") if priority is None: - scaled = routing.get("priority") if routing else None + routed = routing.get("priority") if routing else None priority = ( DEFAULT_REQUEST_PRIORITY - if scaled is None - else min(1.0, max(0.0, scaled / DEEPINFRA_PRIORITY_SCALE)) + if routed is None + else min(1.0, max(0.0, float(routed))) ) cache_salt = request_cache_salt(request) diff --git a/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py b/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py index e67acbdbcabe..34c95bbbe4d0 100644 --- a/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py +++ b/components/src/dynamo/trtllm/tests/test_trtllm_handler_base.py @@ -717,8 +717,8 @@ class TestHealthCheckPriority: Health check requests (built by TrtllmHealthCheckPayload) must reach the TRT-LLM engine at priority=1.0. Regular inference requests carry it - in routing hints, scaled by DEEPINFRA_PRIORITY_SCALE; with no routing - priority at all they fall back to DEFAULT_REQUEST_PRIORITY (0.5). + in routing hints; with no routing priority at all they fall back to + DEFAULT_REQUEST_PRIORITY (0.5). """ def _make_handler(self) -> HandlerBase: @@ -804,11 +804,9 @@ async def test_regular_request_gets_default_priority(self): assert kwargs["priority"] == DEFAULT_REQUEST_PRIORITY @pytest.mark.asyncio - @pytest.mark.parametrize( - "scaled,expected", [(6, 0.6), (10, 1.0), (5, 0.5), (0, 0.0)] - ) - async def test_routing_priority_is_unscaled(self, scaled, expected): - """routing.priority is an integer 0-10; TRT-LLM wants [0.0, 1.0].""" + @pytest.mark.parametrize("routed,expected", [(1, 1.0), (0, 0.0)]) + async def test_routing_priority_forwarded(self, routed, expected): + """The header is integer-typed, so only the [0, 1] rails arrive.""" handler = self._make_handler() generation_result = self._make_mock_generation_result() handler.engine.llm.generate_async = MagicMock(return_value=generation_result) @@ -817,7 +815,7 @@ async def test_routing_priority_is_unscaled(self, scaled, expected): "token_ids": [1, 2, 3], "stop_conditions": {"max_tokens": 10}, "sampling_options": {"temperature": 0.7}, - "routing": {"priority": scaled}, + "routing": {"priority": routed}, } chunks = [