[pull] main from NVIDIA:main

cpp/include/tensorrt_llm/batch_manager/llmRequest.h

@@ -2468,11 +2468,6 @@ class LlmRequest : public GenericLlmRequest<runtime::ITensor::SharedPtr>
     LlmRequest(LlmRequest&& request) = default;
     LlmRequest(LlmRequest const& request) = default;
 
-    /// @brief  Create a Response from the current state of the request
-    /// @details Note that there is some dependency on the order of operations in this method. Modify with care!
-    /// @return An optional Response
-    std::optional<executor::Response> createResponse(bool useFastLogits = false, int32_t mpiWorldRank = 0);
-
     std::optional<executor::Result> createResult(bool useFastLogits = false, int32_t mpiWorldRank = 0);
 
     void createSerializedResult(
@@ -2488,10 +2483,6 @@ class LlmRequest : public GenericLlmRequest<runtime::ITensor::SharedPtr>
 
     std::shared_ptr<LlmRequest> createChildRequest(RequestIdType requestId);
 
-    void movePromptEmbeddingTableToGpu(runtime::BufferManager const& manager);
-
-    void moveLoraWeightsToGpu(runtime::BufferManager const& manager);
-
     // Remove LoRA weights and LoRA config tensors
     void removeLoraTensors();
 };

cpp/tensorrt_llm/batch_manager/llmRequest.cpp

@@ -52,17 +52,6 @@ runtime::SizeType32 GenericLlmRequest<TTensor, TStream>::getBeamWidthByIter(bool
 
 template class GenericLlmRequest<runtime::ITensor::SharedPtr>;
 
-std::optional<executor::Response> LlmRequest::createResponse(bool useFastLogits, int32_t mpiWorldRank)
-{
-    auto requestId = isChild() ? mParentRequestId : mRequestId;
-    auto result = createResult(useFastLogits, mpiWorldRank);
-    if (result.has_value())
-    {
-        return executor::Response(requestId, result.value(), mClientId);
-    }
-    return std::nullopt;
-}
-
 void LlmRequest::createSerializedResult(
     std::vector<char>& serializedResult, bool& isFinal, bool useFastLogits, int32_t mpiWorldRank)
 {
@@ -355,29 +344,6 @@ std::shared_ptr<LlmRequest> LlmRequest::createChildRequest(RequestIdType request
     return childReq;
 }
 
-void LlmRequest::movePromptEmbeddingTableToGpu(runtime::BufferManager const& manager)
-{
-    if (!mPromptEmbeddingTable.has_value()
-        || mPromptEmbeddingTable.value()->getMemoryType() == runtime::MemoryType::kGPU)
-    {
-        return;
-    }
-
-    TensorPtr gpuPromptEmbeddingTable = manager.copyFrom(*mPromptEmbeddingTable.value(), runtime::MemoryType::kGPU);
-    mPromptEmbeddingTable = gpuPromptEmbeddingTable;
-}
-
-void LlmRequest::moveLoraWeightsToGpu(runtime::BufferManager const& manager)
-{
-    if (!mLoraWeights.has_value() || mLoraWeights.value()->getMemoryType() == runtime::MemoryType::kGPU)
-    {
-        return;
-    }
-    // TODO for tp / pp models we only need to move the bit that belong on the local device
-    TensorPtr gpuLoraWeights = manager.copyFrom(*mLoraWeights.value(), runtime::MemoryType::kGPU);
-    mLoraWeights = gpuLoraWeights;
-}
-
 void LlmRequest::removeLoraTensors()
 {
     mLoraWeights.reset();

cpp/tensorrt_llm/nanobind/batch_manager/bindings.cpp

@@ -484,8 +484,6 @@ void initBindings(nb::module_& m)
         .def("validate", &tb::LlmRequest::validate, nb::arg("max_input_len"), nb::arg("max_seq_len"),
             nb::arg("max_draft_len"), nb::arg("vocab_size_padded"), nb::arg("max_endocer_input_len") = std::nullopt,
             nb::arg("enable_kv_cache_reuse") = false)
-        .def("create_response", &tb::LlmRequest::createResponse, nb::arg("use_fast_logits") = false,
-            nb::arg("mpi_world_rank") = 0)
         .def("create_child_request", &tb::LlmRequest::createChildRequest, nb::arg("child_id"))
         .def("create_result", &tb::LlmRequest::createResult, nb::arg("use_fast_logits") = false,
             nb::arg("mpi_world_rank") = 0)
@@ -497,8 +495,6 @@ void initBindings(nb::module_& m)
                 self.createSerializedResult(serialized_result, is_final, use_fast_logits, mpi_world_rank);
                 return std::make_tuple(nb::bytes(serialized_result.data(), serialized_result.size()), is_final);
             })
-        .def("move_prompt_embedding_table_to_gpu", &tb::LlmRequest::movePromptEmbeddingTableToGpu, nb::arg("manager"))
-        .def("move_lora_weights_to_gpu", &tb::LlmRequest::moveLoraWeightsToGpu, nb::arg("manager"))
         .def("finish_by_reason", &tb::LlmRequest::finishByReason, nb::arg("finish_reason"))
         .def("set_first_scheduled_time", &tb::LlmRequest::setFirstScheduledTime)
         .def("update_perf_metrics", &tb::LlmRequest::updatePerfMetrics, nb::arg("iter_counter"))

cpp/tests/unit_tests/batch_manager/llmRequestTest.cpp

@@ -463,7 +463,7 @@ class ParamTest : public LlmRequestTest, public ::testing::WithParamInterface<Pa
 {
 };
 
-TEST_P(ParamTest, createResponse)
+TEST_P(ParamTest, createResult)
 {
     bool const streaming{std::get<0>(GetParam())};
     bool const excludeInputFromOutput{std::get<1>(GetParam())};
@@ -517,8 +517,8 @@ TEST_P(ParamTest, createResponse)
 
     for (auto& llmReq : llmRequests)
     {
-        auto response = llmReq->createResponse();
-        EXPECT_FALSE(response);
+        auto resultOpt = llmReq->createResult();
+        EXPECT_FALSE(resultOpt);
     }
 
     SizeType32 constexpr numIterations{5};
@@ -536,15 +536,14 @@ TEST_P(ParamTest, createResponse)
             }
 
             llmReq->setState(tb::LlmRequestState::kGENERATION_IN_PROGRESS);
-            auto response = llmReq->createResponse();
-            EXPECT_TRUE(streaming == response.has_value());
+            auto resultOpt = llmReq->createResult();
+            EXPECT_TRUE(streaming == resultOpt.has_value());
 
             for (int beamIdx = 0; beamIdx < numReturnBeams; ++beamIdx)
             {
                 if (streaming)
                 {
-                    EXPECT_EQ(response.value().getRequestId(), requestId);
-                    auto result = response.value().getResult();
+                    auto const& result = resultOpt.value();
                     EXPECT_EQ(result.outputTokenIds.size(), numReturnBeams);
                     auto const& beamTokens = result.outputTokenIds.at(beamIdx);
                     if (returnAllGeneratedTokens)
@@ -564,8 +563,8 @@ TEST_P(ParamTest, createResponse)
                 }
             }
 
-            response = llmReq->createResponse();
-            EXPECT_FALSE(response);
+            resultOpt = llmReq->createResult();
+            EXPECT_FALSE(resultOpt);
         }
     }
 
@@ -584,21 +583,17 @@ TEST_P(ParamTest, createResponse)
     for (auto seqIdx = 0; seqIdx < numSequences; seqIdx++)
     {
         auto llmReq = llmRequests.at(seqIdx);
-        auto response = llmReq->createResponse();
+        auto resultOpt = llmReq->createResult();
 
         if (!streaming && llmRequests.at(seqIdx)->getState() != tb::LlmRequestState::kGENERATION_COMPLETE)
         {
-            EXPECT_FALSE(response);
+            EXPECT_FALSE(resultOpt);
             continue;
         }
 
-        EXPECT_TRUE(response) << "seqIdx " << seqIdx;
-        EXPECT_FALSE(response.value().hasError()) << "seqIdx " << seqIdx;
+        EXPECT_TRUE(resultOpt) << "seqIdx " << seqIdx;
 
-        // All response should have the same request id of the original request.
-        EXPECT_EQ(response.value().getRequestId(), requestId);
-
-        auto result = response.value().getResult();
+        auto const& result = resultOpt.value();
         EXPECT_EQ(result.outputTokenIds.size(), numReturnBeams);
 
         // Only the first sequence has finished.
@@ -661,11 +656,10 @@ TEST_P(ParamTest, createResponse)
 
         for (auto seqIdx = 1; seqIdx < numSequences; seqIdx++)
         {
-            auto response = llmRequests.at(seqIdx)->createResponse();
-            EXPECT_TRUE(response) << "seqIdx " << seqIdx;
-            EXPECT_FALSE(response.value().hasError()) << "seqIdx " << seqIdx;
+            auto resultOpt = llmRequests.at(seqIdx)->createResult();
+            EXPECT_TRUE(resultOpt) << "seqIdx " << seqIdx;
 
-            auto result = response.value().getResult();
+            auto const& result = resultOpt.value();
             // All sequences have finished.
             EXPECT_TRUE(result.isSequenceFinal) << "seqIdx " << seqIdx;
             EXPECT_TRUE(result.isFinal) << "seqIdx " << seqIdx;
@@ -703,12 +697,12 @@ TEST_F(LlmRequestTest, createResultDisaggContextComplete)
     // This is the regression case — without the fix, createResult() returns nullopt
     // because DISAGG_CONTEXT_COMPLETE was not handled by createResult's early guard
     // or its context-phase branch.
-    auto response = llmReq.createResult(/*useFastLogits=*/false, /*mpiWorldRank=*/0);
-    ASSERT_TRUE(response.has_value()) << "createResult() must not return nullopt for DISAGG_CONTEXT_COMPLETE";
-    EXPECT_TRUE(response->contextPhaseParams.has_value())
+    auto resultOpt = llmReq.createResult(/*useFastLogits=*/false, /*mpiWorldRank=*/0);
+    ASSERT_TRUE(resultOpt.has_value()) << "createResult() must not return nullopt for DISAGG_CONTEXT_COMPLETE";
+    EXPECT_TRUE(resultOpt->contextPhaseParams.has_value())
         << "contextPhaseParams must be populated for context-only DISAGG_CONTEXT_COMPLETE requests";
-    EXPECT_EQ(response->contextPhaseParams->getReqId(), requestId);
-    EXPECT_TRUE(response->isSequenceFinal);
+    EXPECT_EQ(resultOpt->contextPhaseParams->getReqId(), requestId);
+    EXPECT_TRUE(resultOpt->isSequenceFinal);
 }
 
 TEST_F(LlmRequestTest, generationOnlyRequestAdoptsContextPhaseDraftTokens)

docs/source/features/kvcache.md

@@ -89,6 +89,7 @@ Models that select the V2 manager by default:
 | DeepSeek-V4 | Sparse attention attaches auxiliary per-layer buffers |
 | GPT-OSS | Sliding window on every other layer (VSWA), so the sliding-window and full-attention pools are sized independently |
 | Gemma3 / Gemma4 (text and multimodal) | Alternating sliding-window and full-attention layers (VSWA); same independent pool sizing |
+| Llama / Llama4 | Uniform KV pool layout; chunked attention does not partition the pools |
 
 Separately, Gemma4 hybrid attention and sparse-attention models are routed to
 V2 unconditionally: their per-layer buffer layouts cannot be represented by V1's

jenkins/BoltProfileGen.groovy

@@ -84,6 +84,15 @@ PROMOTE       = (params.promote ?: "false").toString()
 // bundle -> bolted tarball on the cluster ("consume immediately after generating").
 // The merge job (slurm_merge.sh) runs apply_bolt.py when BOLT_APPLY=1.
 APPLY_PROFILES = (params.applyProfiles ?: "true").toString()
+// publishBoltedCanonical: after merge (and, if applicable, promote), push the
+// just-BOLTed tarball back to the input artifactPath under the CANONICAL name so
+// downstream consumers of that build get BOLT transparently, preserving the
+// original as unbolted-<tarball>. Requires APPLY_PROFILES=true (the bolted tarball
+// is produced by the merge job's BOLT_APPLY=1 step) and PROMOTE=true (only the
+// postmerge producer repushes a canonical). Default OFF -- the rollout is flipped
+// on in a follow-up change -- so this is inert until then. Resolution mirrors the
+// other toggles: param, then env.
+PUBLISH_BOLTED_CANONICAL = (params.boltPublishCanonical ?: env.boltPublishCanonical ?: "false").toString() == "true"
 // Multiply each workload's client `iterations` (num_requests = concurrency *
 // iterations) to lengthen the measured serving window without editing the shared
 // perf-sanity configs. Default 64: the point where the aggregated workloads'
@@ -372,8 +381,18 @@ def submitProfileGen(pipeline)
                 return 0
             }
         '''.stripIndent()
+        // If a prior run against THIS artifactPath already published the BOLTed build
+        // as canonical, publishBoltedCanonical preserved the original as
+        // unbolted-<tarball>. Prefer it so we profile/BOLT the un-BOLTed input rather
+        // than double-BOLTing. Inert for a fresh artifactPath (no unbolted- object),
+        // where canonical IS the un-BOLTed build.
+        def unboltedTarUrl = "${URM_ARTIFACTORY_BASE}/${ARTIFACT_PATH}/unbolted-${BOLT_TARNAME}"
         def tarStage = """
             URL='${llmTarfile}'
+            if curl -fsI '${unboltedTarUrl}' >/dev/null 2>&1; then
+                echo '[INFO] found unbolted-${BOLT_TARNAME}; using it as the un-BOLTed input (avoids double-BOLT on reuse)'
+                URL='${unboltedTarUrl}'
+            fi
             DEST='${ws}/builds/${BOLT_TARNAME}'
             PARTS=16
         """.stripIndent() + boltFetchLib + '''
@@ -509,6 +528,26 @@ def submitProfileGen(pipeline)
             pipeline.echo("PROMOTE=false: skipping Artifactory promote of ${bundle}")
         }
 
+        // Publish the freshly BOLTed build back to the input artifactPath under the
+        // CANONICAL name (unbolted-<tarball> preserved), so downstream consumers of
+        // that build get BOLT transparently. The merge job (APPLY_PROFILES=true)
+        // already produced ${outDir}/bolt-${BOLT_TARNAME} natively on the aarch64
+        // cluster node; without this it is reclaimed by the retention sweep below.
+        // Runs BEFORE that sweep. Coupled to PROMOTE so only the postmerge producer
+        // (which promotes the bundle) repushes a canonical -- a premerge
+        // generate-and-consume run (promote=false) never does, even with the toggle
+        // on. Gated off by default (PUBLISH_BOLTED_CANONICAL).
+        if (PUBLISH_BOLTED_CANONICAL && APPLY_PROFILES == "true" && PROMOTE == "true") {
+            stage("Publish BOLTed build as canonical") {
+                publishBoltedCanonical(pipeline, remote,
+                    "${outDir}/bolt-${BOLT_TARNAME}",       // bolted build from the merge job
+                    "${ws}/builds/${BOLT_TARNAME}",         // un-BOLTed input on the cluster
+                    "${URM_ARTIFACTORY_BASE}/${ARTIFACT_PATH}", BOLT_TARNAME)
+            }
+        } else if (!PUBLISH_BOLTED_CANONICAL) {
+            pipeline.echo("PUBLISH_BOLTED_CANONICAL=false: not repushing a BOLTed canonical tarball.")
+        }
+
         // Retention: best-effort purge of workspaces older than 7 days so scratch
         // doesn't grow unbounded across runs. Depth 4 under the bolt-ci root maps
         // to <BRANCH>/<TRIPLE>/<instance>/<BUILD_TAG>, i.e. one per-run workspace.
@@ -732,6 +771,67 @@ def promoteBundle(pipeline, remote, String bundle)
     pipeline.echo("Promoted. latest = ${base}/latest.tar.gz")
 }
 
+// ---------------------------------------------------------------------------
+// Publish the just-BOLTed build under the CANONICAL name at the input artifactPath,
+// mirroring Build.groovy's premerge apply/consume convention: BOLT is the DEFAULT,
+// so canonical <tarName> becomes the BOLTed build and the original is preserved as
+// unbolted-<tarName>. The merge job (BOLT_APPLY=1) produced the bolted tarball
+// natively on the aarch64 cluster node into ${outDir}/bolt-<tarName>.
+//
+// Runs CLUSTER-SIDE (the frontend reaches Artifactory and the tarballs live on
+// scratch). Fails loudly if the bolted tarball is missing, and is idempotent on a
+// reused artifactPath: the unbolted- snapshot is skipped if it already exists, so a
+// re-run never overwrites the real unbolted with an already-BOLTed canonical.
+//
+// Credentials use the same stdin-fed netrc pattern as promoteBundle(): the secret
+// travels over ssh stdin (read by `cat > netrc`), never embedded in the base64'd
+// script, so it can't leak via the decoded remote command or `ps`.
+// ---------------------------------------------------------------------------
+def publishBoltedCanonical(pipeline, remote, String boltedLocal, String unboltedLocal, String artifactBase, String tarName)
+{
+    def host = URM_ARTIFACTORY_BASE.replaceFirst(/^https?:\/\//, "").tokenize('/').first()
+    def netrc = "${boltedLocal}.netrc"
+    def canonicalUrl = "${artifactBase}/${tarName}"
+    def unboltedUrl  = "${artifactBase}/unbolted-${tarName}"
+    pipeline.echo("Publishing BOLTed build as canonical ${canonicalUrl} (un-BOLTed preserved at ${unboltedUrl})")
+    // Remote side carries no secret: it reads the netrc from stdin. Clean up via
+    // trap, not a trailing rm, so a failed curl under `set -e` still removes the
+    // plaintext netrc; umask covers the window before chmod.
+    def publish = """
+        set -e
+        umask 077
+        trap 'rm -f "${netrc}"' EXIT
+        test -f "${boltedLocal}"   || { echo "[ERROR] bolted tarball missing: ${boltedLocal} (did the merge run with BOLT_APPLY=1?)" >&2; exit 1; }
+        test -f "${unboltedLocal}" || { echo "[ERROR] un-BOLTed tarball missing: ${unboltedLocal}" >&2; exit 1; }
+        cat > "${netrc}"
+        chmod 600 "${netrc}"
+        # 1) Preserve the un-BOLTed build as unbolted-${tarName}, unless already there.
+        #    A PUT-upload, NOT /api/copy: the target repo is virtual, so a server-side
+        #    copy returns 409. Idempotent via the existence check: on a re-run the
+        #    canonical is ALREADY the BOLTed build, and re-publishing would otherwise
+        #    overwrite the real unbolted with a bolted copy.
+        if curl -fsI --netrc-file "${netrc}" "${unboltedUrl}" >/dev/null 2>&1; then
+            echo "[INFO] unbolted-${tarName} already exists; skipping upload"
+        else
+            echo "[INFO] uploading un-BOLTed tarball -> unbolted-${tarName}"
+            curl -fS --netrc-file "${netrc}" --retry 5 --retry-all-errors \\
+                 --connect-timeout 30 --speed-time 300 --speed-limit 1024 \\
+                 -T "${unboltedLocal}" "${unboltedUrl}"
+        fi
+        # 2) Overwrite canonical ${tarName} with the BOLTed build (the release name).
+        curl -fS --netrc-file "${netrc}" --retry 5 --retry-all-errors \\
+             --connect-timeout 30 --speed-time 300 --speed-limit 1024 \\
+             -T "${boltedLocal}" "${canonicalUrl}"
+    """.stripIndent()
+    pipeline.withCredentials([pipeline.usernamePassword(credentialsId: 'urm-artifactory-creds',
+            usernameVariable: 'ART_USER', passwordVariable: 'ART_PASS')]) {
+        def feed = "printf 'machine ${host} login %s password %s\\n' \"\$ART_USER\" \"\$ART_PASS\" | "
+        Utils.exec(pipeline, timeout: false, numRetries: 2, noNVDFEvent: true,
+            script: feed + Utils.sshUserCmd(remote, b64BashRemoteCmdStdin(publish, "${boltedLocal}.publish.sh")))
+    }
+    pipeline.echo("Published. canonical = ${canonicalUrl}")
+}
+
 
 pipeline {
     agent {
@@ -798,6 +898,11 @@ pipeline {
             choices: ["true", "false"],
             description: "Re-BOLT the input tarball with the just-generated bundle in the merge job, as a same-commit check that the profiles apply. No extra GPU allocation."
         )
+        choice(
+            name: "boltPublishCanonical",
+            choices: ["false", "true"],
+            description: "After merge (requires applyProfiles=true), push the BOLTed build back to the input artifactPath under the canonical name (original preserved as unbolted-<tarball>). Default false; the rollout is turned on in a follow-up change."
+        )
         string(
             name: "slurmPlatform",
             defaultValue: "",

tensorrt_llm/_torch/attention/backends/fmha/flashinfer_trtllm_gen.py

@@ -602,11 +602,15 @@ def _is_supported_with_reason(
             and 0 < meta.num_contexts <= 4
             and attn.head_dim != 512
             and not has_q_only
+            and meta.num_generations == 0
         ):
             # NVBug 6579626: the per-layer host overhead of the FlashInfer
             # TRTLLM-Gen context path regresses TTFT for small BF16 batches.
             # Let the FMHA selector choose the fallback implementation. H512
             # and Q-only cached-KV requests cannot use that fallback.
+            # Keep mixed batches on FlashInfer: the monolithic fallback also
+            # switches decoding to C++ kernels that generation-only warmup
+            # does not cover, causing first-use JIT stalls (NVBug 6716104).
             return False, (
                 "small-batch BF16 context attention uses the fallback FMHA for "
                 "performance because the FlashInfer TRTLLM-Gen context path "