From 653e07b3662dc10bb6873abf40d02f2b555d1963 Mon Sep 17 00:00:00 2001 From: Brian Nguyen Date: Thu, 30 Jul 2026 12:23:52 -0700 Subject: [PATCH 1/2] [TRTLLM-14810][chore] Sync sa_worker.py with main (#16759) Main PR #16759 upstreamed this branch's SA in-worker hybrid-state promotion verbatim; syncing the file back removes the catch-up-merge conflict. One intentional residual delta: the method stays named `forward` (not `_forward_impl`) because this branch's SpecWorkerBase does not yet carry main's #16382 exception-safe forward wrapper -- workers here are invoked via nn.Module.__call__ -> forward. The merge will finish the rename. Signed-off-by: Brian Nguyen --- tensorrt_llm/_torch/speculative/sa_worker.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/tensorrt_llm/_torch/speculative/sa_worker.py b/tensorrt_llm/_torch/speculative/sa_worker.py index 550f239d0967..41c342e2d2b1 100644 --- a/tensorrt_llm/_torch/speculative/sa_worker.py +++ b/tensorrt_llm/_torch/speculative/sa_worker.py @@ -188,9 +188,7 @@ def forward( # one-engine workers (dflash/eagle3); no-op for pure-attention # models via the isinstance gate. num_gens = batch_size - num_contexts - if num_gens > 0 and isinstance( - attn_metadata.kv_cache_manager, MambaHybridCacheManager - ): + if num_gens > 0 and isinstance(attn_metadata.kv_cache_manager, MambaHybridCacheManager): attn_metadata.kv_cache_manager.update_mamba_states( attn_metadata=attn_metadata, num_accepted_tokens=num_accepted_tokens, From 37ba252893ca4dae5c8440179d2aa9e0de393e74 Mon Sep 17 00:00:00 2001 From: Brian Nguyen Date: Thu, 30 Jul 2026 12:25:05 -0700 Subject: [PATCH 2/2] [TRTLLM-14810][chore] Require packaging>=24.2; drop manual install workaround from Kimi K3 docs Mirrors main PR #17075: FlashInfer source builds with --no-build-isolation need packaging>=24.2 for setuptools license-expression validation, so declare it in requirements.txt and remove the manual 'pip install -U packaging' step from the Kimi K3 README and deployment guide. Signed-off-by: Brian Nguyen --- .../deployment-guide-for-kimi-k3-on-trtllm.md | 3 +-- examples/kimi_k3/README.md | 5 +++-- requirements.txt | 2 ++ 3 files changed, 6 insertions(+), 4 deletions(-) diff --git a/docs/source/deployment-guide/deployment-guide-for-kimi-k3-on-trtllm.md b/docs/source/deployment-guide/deployment-guide-for-kimi-k3-on-trtllm.md index b16dcc21a359..86869bf99779 100644 --- a/docs/source/deployment-guide/deployment-guide-for-kimi-k3-on-trtllm.md +++ b/docs/source/deployment-guide/deployment-guide-for-kimi-k3-on-trtllm.md @@ -56,13 +56,12 @@ Kimi K3 additionally depends on `fla` and `einops`, installed into the same in-p To use the optimized CuTeDSL MLA kernel, install the FlashInfer revision used by the Kimi K3 example into the same in-place environment: ```bash -.venv-3.12/bin/python -m pip install -U 'packaging>=24.2' # required by the FlashInfer source build .venv-3.12/bin/python -u -m pip install --force-reinstall --no-deps \ --no-build-isolation \ "flashinfer-python[cu13] @ git+https://github.com/PerkzZheng/flashinfer-k3.git@b6cc594918baf76c40c3a6236fd53f0f8fb9d2dc" ``` -The TensorRT LLM environment already provides FlashInfer's runtime dependencies. The `--no-deps` option prevents `pip` from replacing the pinned PyTorch, Triton, CUDA, and CuTeDSL packages. Install FlashInfer after TensorRT LLM because a later dependency-resolving TensorRT LLM installation can replace this source revision with the currently pinned `flashinfer-python==0.6.14`. +The `packaging>=24.2` requirement of this source build is already satisfied by `requirements.txt`. The TensorRT LLM environment already provides FlashInfer's runtime dependencies. The `--no-deps` option prevents `pip` from replacing the pinned PyTorch, Triton, CUDA, and CuTeDSL packages. Install FlashInfer after TensorRT LLM because a later dependency-resolving TensorRT LLM installation can replace this source revision with the currently pinned `flashinfer-python==0.6.14`. For general build-from-source instructions see [https://nvidia.github.io/TensorRT-LLM/latest/installation/build-from-source.html](https://nvidia.github.io/TensorRT-LLM/latest/installation/build-from-source.html). diff --git a/examples/kimi_k3/README.md b/examples/kimi_k3/README.md index feae47c36162..e4824af039a7 100644 --- a/examples/kimi_k3/README.md +++ b/examples/kimi_k3/README.md @@ -41,13 +41,14 @@ other GPU architectures may be added in a future release. revision into the same in-place environment after installing TensorRT-LLM: ```bash - .venv-3.12/bin/python -m pip install -U 'packaging>=24.2' # required by the FlashInfer source build .venv-3.12/bin/python -u -m pip install --force-reinstall --no-deps \ --no-build-isolation \ "flashinfer-python[cu13] @ git+https://github.com/PerkzZheng/flashinfer-k3.git@b6cc594918baf76c40c3a6236fd53f0f8fb9d2dc" ``` - The TensorRT-LLM environment already provides FlashInfer's runtime + The `packaging>=24.2` requirement of this source build is already + satisfied by `requirements.txt`. The TensorRT-LLM environment already + provides FlashInfer's runtime dependencies; `--no-deps` prevents pip from replacing its pinned PyTorch, Triton, CUDA, and CuTeDSL packages. Install FlashInfer last: TensorRT-LLM currently pins `flashinfer-python==0.6.14`, so a later diff --git a/requirements.txt b/requirements.txt index dab577614f57..9555bf2ac80c 100644 --- a/requirements.txt +++ b/requirements.txt @@ -52,6 +52,8 @@ fastapi>=0.120.1,<=0.121.3 starlette>=0.49.1 uvicorn setuptools<80 +# FlashInfer source builds (--no-build-isolation) need packaging>=24.2 for setuptools license-expression validation +packaging>=24.2 ordered-set peft>=0.18.1,<0.19.0 patchelf