From 22213af5cc456d9d7ac7b24a09a3cc38ffa13713 Mon Sep 17 00:00:00 2001 From: trtllm-agent <296075020+trtllm-agent@users.noreply.github.com> Date: Wed, 8 Jul 2026 11:49:11 -0700 Subject: [PATCH 1/2] [nvbugs/6428124][fix] Forward use_host_stop_criteria over PP ring Nemotron-H LoRA PP=2 crashed with IndexError in finish_if_reason because the last PP rank produced a SampleStateTorch with use_host_stop_criteria=True (and host.finish_reasons=None), but only sample_state.host was sent through _ring_broadcast_sample_state. Non-last ranks kept the constructor default False and tried to index into an empty finish_reasons list from process_draft_tokens. Include the flag in the PP send/recv payload and restore it on the receiver. Use getattr/hasattr to stay compatible with the TRT-LLM sampler flavor which lacks this field. Remove the corresponding waiver. Signed-off-by: trtllm-agent <296075020+trtllm-agent@users.noreply.github.com> --- tensorrt_llm/_torch/pyexecutor/py_executor.py | 19 ++++++++++++++----- 1 file changed, 14 insertions(+), 5 deletions(-) diff --git a/tensorrt_llm/_torch/pyexecutor/py_executor.py b/tensorrt_llm/_torch/pyexecutor/py_executor.py index 8ffe970ab5a9..66163567bd27 100644 --- a/tensorrt_llm/_torch/pyexecutor/py_executor.py +++ b/tensorrt_llm/_torch/pyexecutor/py_executor.py @@ -3330,10 +3330,18 @@ def _ring_broadcast_sample_state( if not self.dist.is_last_pp_rank: # Receive tokens from previous pp rank (w.r.t model forward direction) with nvtx_range("recv_sample_state"): - sample_state.host, py_result_diffs = self.dist.recv_object( - src=self.dist.prev_pp_rank, - tag=tag, - ) + (sample_state.host, py_result_diffs, + use_host_stop_criteria) = self.dist.recv_object( + src=self.dist.prev_pp_rank, + tag=tag, + ) + + # The last PP rank owns the sampling shape. Its fast host stop + # criteria path leaves host.finish_reasons=None, so the flag + # governing update_requests's branching must ride the ring with + # the host state or non-last ranks index into an empty list. + if hasattr(sample_state, "use_host_stop_criteria"): + sample_state.use_host_stop_criteria = use_host_stop_criteria for request, py_result_diff in zip(requests, py_result_diffs): request.py_result.apply_diff(py_result_diff) @@ -3351,7 +3359,8 @@ def _ring_broadcast_sample_state( self.wait_on_pp_send_handles(self.send_handles, microbatch_id) with nvtx_range("send_sample_state"): self.send_handles[microbatch_id] = self.dist.isend_object( - (sample_state.host, py_result_diffs), + (sample_state.host, py_result_diffs, + getattr(sample_state, "use_host_stop_criteria", False)), dest=self.dist.next_pp_rank, tag=tag, ) From 369014d1070425505f604b0fe96eb16ebed3cdb7 Mon Sep 17 00:00:00 2001 From: handongl Date: Fri, 24 Jul 2026 00:28:50 -0700 Subject: [PATCH 2/2] [nvbugs/6428124][chore] Remove stale waiver after fix Signed-off-by: handongl --- tests/integration/test_lists/waives.txt | 1 - 1 file changed, 1 deletion(-) diff --git a/tests/integration/test_lists/waives.txt b/tests/integration/test_lists/waives.txt index 2abacdb610eb..b585906e2357 100644 --- a/tests/integration/test_lists/waives.txt +++ b/tests/integration/test_lists/waives.txt @@ -263,7 +263,6 @@ unittest/_torch/modeling/test_modeling_nemotron_h.py::test_nemotron_h_breakable_ unittest/_torch/modeling/test_modeling_nemotron_h.py::test_nemotron_h_breakable_prefill_cuda_graph[tp1] SKIP (https://nvbugs/6777501) unittest/_torch/modeling/test_modeling_nemotron_h_multimodal.py::test_nemotron_nano_v2_vl_image_batch_equivalence SKIP (https://nvbugs/6758946) unittest/_torch/modeling/test_modeling_nemotron_h_multimodal.py::test_nemotron_nano_v2_vl_video_batch_equivalence SKIP (https://nvbugs/6625695) -unittest/_torch/modules/tests_lora_modules/test_nemotron_h_lora_sanity.py::TestNemotronHLoRA::test_lora_pp2_sanity SKIP (https://nvbugs/6428124) unittest/_torch/moe/test_moe_backend.py::test_moe_backend[act=Relu2-e60_k4_h2048_i1408-seq=8-dtype=torch.bfloat16-backend=TRTLLM-quant=NVFP4-routing=Renormalize] SKIP (https://nvbugs/5989912) unittest/_torch/multi_gpu/test_linear.py::test_row_linear_norm_fusion[2-hidden:16-seqlen:2] SKIP (https://nvbugs/6501404) unittest/_torch/ray_orchestrator/multi_gpu/test_llm_update_weights_multi_gpu.py::test_llm_update_weights_nemotron_h SKIP (https://nvbugs/6729495)