From e71e92e569187fc3bf7f50fd09edc312bc344fa5 Mon Sep 17 00:00:00 2001 From: trtllm-agent <296075020+trtllm-agent@users.noreply.github.com> Date: Wed, 8 Jul 2026 11:55:54 -0700 Subject: [PATCH 1/2] [nvbugs/6428092][fix] Propagate use_host_stop_criteria across PP ranks test_tinyllama_logits_processor_2gpu[1-2] (pp_size=2) crashed with IndexError in finish_if_reason because the last PP rank produced a SampleStateTorch with use_host_stop_criteria=True (and host.finish_reasons=None), while only sample_state.host was sent through _ring_broadcast_sample_state. Non-last ranks kept the constructor default False and tried to index into an empty finish_reasons list from process_draft_tokens. Include the flag in the PP send/recv payload and restore it on the receiver. Use getattr/hasattr to stay compatible with sampler flavors that lack this field. Signed-off-by: trtllm-agent <296075020+trtllm-agent@users.noreply.github.com> --- tensorrt_llm/_torch/pyexecutor/py_executor.py | 15 ++++++++++----- 1 file changed, 10 insertions(+), 5 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor.py b/tensorrt_llm/_torch/pyexecutor/py_executor.py index 8ffe970ab5a9..45fcf109575b 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor.py @@ -3330,10 +3330,14 @@ def _ring_broadcast_sample_state( if not self.dist.is_last_pp_rank: # Receive tokens from previous pp rank (w.r.t model forward direction) with nvtx_range("recv_sample_state"): - sample_state.host, py_result_diffs = self.dist.recv_object( - src=self.dist.prev_pp_rank, - tag=tag, - ) + sample_state.host, py_result_diffs, use_host_stop_criteria = \ + self.dist.recv_object(src=self.dist.prev_pp_rank, tag=tag) + + # Propagate use_host_stop_criteria: the last rank's fast host-stop + # path leaves host.finish_reasons=None, so non-last ranks must + # know to skip the finish_reasons indexing in update_requests. + if hasattr(sample_state, "use_host_stop_criteria"): + sample_state.use_host_stop_criteria = use_host_stop_criteria for request, py_result_diff in zip(requests, py_result_diffs): request.py_result.apply_diff(py_result_diff) @@ -3351,7 +3355,8 @@ def _ring_broadcast_sample_state( self.wait_on_pp_send_handles(self.send_handles, microbatch_id) with nvtx_range("send_sample_state"): self.send_handles[microbatch_id] = self.dist.isend_object( - (sample_state.host, py_result_diffs), + (sample_state.host, py_result_diffs, + getattr(sample_state, "use_host_stop_criteria", False)), dest=self.dist.next_pp_rank, tag=tag, ) From 2f7a01c860c4a7e4da4486225b1cd0c1a5f5d527 Mon Sep 17 00:00:00 2001 From: trtllm-agent <296075020+trtllm-agent@users.noreply.github.com> Date: Thu, 17 Sep 2026 13:19:46 -0700 Subject: [PATCH 2/2] [nvbugs/6428092][chore] Remove stale waiver after fix Signed-off-by: trtllm-agent <296075020+trtllm-agent@users.noreply.github.com> --- tests/integration/test_lists/waives.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 2abacdb610eb..995678eb0a88 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -283,7 +283,6 @@ unittest/bindings/test_transfer_agent_bindings.py::TestMooncakeFunctionalTransfe unittest/executor/test_rpc.py::TestRpcCorrectness::test_incremental_task_async SKIP (https://nvbugs/5741476) unittest/executor/test_rpc_proxy.py SKIP (https://nvbugs/5605741) unittest/executor/test_rpc_worker.py SKIP (https://nvbugs/5605741) -unittest/llmapi/test_llm_multi_gpu_pytorch.py -m "gpu2" SKIP (https://nvbugs/6428092) unittest/llmapi/test_llm_multi_gpu_pytorch.py::test_llm_get_stats_pp2[False-False-True] SKIP (https://nvbugs/6432826) unittest/llmapi/test_llm_pytorch.py::test_gqa_nemo_lora[None] SKIP (https://nvbugs/6162504) unittest/llmapi/test_llm_pytorch.py::test_gqa_nemo_lora[cuda_graph_config0] SKIP (https://nvbugs/6162504)