From 0202f4ffe33d19d4f3b69697ac11f0f8e89efafd Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Sat, 12 Sep 2026 09:28:09 +0200 Subject: [PATCH 1/9] =?UTF-8?q?wip(BACKEND-TENSTORRENT-KEEPQUANT):=20W3=20?= =?UTF-8?q?seed=20=E2=80=94=2027B=20alloc-trace=20instrumentation=20(#3042?= =?UTF-8?q?)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Uncommitted work-in-progress preserved from the merged W1+W2 worktree before its removal. The 27B instrumented re-check needs it: per-block AllocTraceSnapshot pre/post brackets in DenseForwardLayers (attention, GDN, norms, rolls — the 13 GB the chunk-loop ledger did not cover) and chunk-loop taps in KQuantGrouped, plus a defensive-only rewording of the sl_alias guard (the E>1 arm can never hit the full-extent identity) and new backend-suite coverage. Not reviewed, not gated, not for merge as-is. FOLLOWING_AGENTS_PROTOCOL Following-Agents-Protocol: true AI-Assisted: true Assisted-by: AGENT:zai/glm-5.3-flash [maki] --- src/vllm/model_executor/models/qwen3_5.cpp | 20 +++ src/vt/tenstorrent/tenstorrent_ops.cpp | 57 ++++---- tests/vt/test_tenstorrent_backend.cpp | 157 +++++++++++++++++++++ 3 files changed, 210 insertions(+), 24 deletions(-) diff --git a/src/vllm/model_executor/models/qwen3_5.cpp b/src/vllm/model_executor/models/qwen3_5.cpp index 3f6fac941..17534a27e 100644 --- a/src/vllm/model_executor/models/qwen3_5.cpp +++ b/src/vllm/model_executor/models/qwen3_5.cpp @@ -9631,8 +9631,28 @@ static DBuf DenseForwardLayers(Dev d, const Tensor& hidden_in, layer.is_linear_attention ? nullptr : &attn_kv[static_cast(fa_idx++)]; const GdnStateCache* gs = layer.is_linear_attention ? &gdn_state[static_cast(gdn_idx++)] : nullptr; +#ifdef VLLM_CPP_TENSTORRENT + // W4d (#3042) attribution: the chunk-loop ledger brackets only the + // keep-quant webs; 13 GB of the 27B first pass lands in the spans this + // pair now brackets (attention, GDN, norms, rolls). + char tt_lbl[48]; + if (vt::tenstorrent::DeviceAvailable()) { + std::snprintf(tt_lbl, sizeof tt_lbl, "block/%lld/pre", + static_cast(l)); + vt::tenstorrent::AllocTraceSnapshot(vt::tenstorrent::SharedMeshDevice(), + tt_lbl); + } +#endif RunDenseLayerPaged(d, layer, config, hidden, res, sdi, attn_meta, gdn_meta, kv, gs, T, l); +#ifdef VLLM_CPP_TENSTORRENT + if (vt::tenstorrent::DeviceAvailable()) { + std::snprintf(tt_lbl, sizeof tt_lbl, "block/%lld/post", + static_cast(l)); + vt::tenstorrent::AllocTraceSnapshot(vt::tenstorrent::SharedMeshDevice(), + tt_lbl); + } +#endif // DFlash DF-AUX-TAPS: capture (hidden+res) at configured boundaries. Inert // (no-op) when aux_out is null — every non-DFlash caller. MaybeCaptureAuxTap(d, l, aux_layer_ids, aux_out, hidden.t(), res.t(), T, H); diff --git a/src/vt/tenstorrent/tenstorrent_ops.cpp b/src/vt/tenstorrent/tenstorrent_ops.cpp index b41f2b08f..f01478a95 100644 --- a/src/vt/tenstorrent/tenstorrent_ops.cpp +++ b/src/vt/tenstorrent/tenstorrent_ops.cpp @@ -2878,6 +2878,7 @@ void MatmulBTQuantGroupedKernel(Queue&, Tensor& out, const Tensor& act, // trade VT_TT_TRACE_REGION_MB records on its axis. if (plane_env_set) chunk = std::min(chunk, std::max(plane_bytes / (K * 4), 1)); + AllocTraceSnapshot(device, "KQuantGrouped/chunk-loop/pre"); std::vector partials; partials.reserve(static_cast((N + chunk - 1) / chunk)); for (int64_t c0 = 0; c0 < N; c0 += chunk) { @@ -2912,6 +2913,7 @@ void MatmulBTQuantGroupedKernel(Queue&, Tensor& out, const Tensor& act, TTReclaimPlanes(device, {&part, &partf}); partials.push_back(std::move(partl)); } + AllocTraceSnapshot(device, "KQuantGrouped/chunk-loop/post"); ttnn::Tensor assembled = partials.size() == 1 ? std::move(partials[0]) @@ -2980,9 +2982,15 @@ void MatmulBTQuantGroupedKernel(Queue&, Tensor& out, const Tensor& act, ttsl::SmallVector{1u, 1u}); ttnn::Tensor wf = DecodeKeepQuantWordsF32(sl, enc, N, nb, device); // W4d W2 (#3042): the same reclaim as the chunk loop — the staged slice - // and decode output die into the typecast/to_layout chain. The E == 1 - // window is full-extent, so sl is the shadow itself there (slice() - // returns its input) and the reclaim must skip it. + // and decode output die into the typecast/to_layout chain. The sl_alias + // guard below is DEFENSIVE ONLY: this lambda runs in the E > 1 arm, + // where the word shadow always holds E*N*nb rows, so the identity's + // second term — words.logical_shape()[0] == N*nb, the full-extent window + // where slice() returns its input — can never hold (it would demand + // E == 1, and the E == 1 arm returned long before this lambda). The + // reclaim never actually skips sl today; the guard stays so the shadow + // is checked, not assumed, if this lambda is ever shared with the + // single-expert arm. const bool sl_alias = w0 == 0 && static_cast(words.logical_shape()[0]) == N * nb; @@ -7364,20 +7372,6 @@ ttnn::Tensor CachedRepeatIdx(uint64_t t, uint64_t heads, uint64_t half, namespace { -ttnn::Tensor CachedScratchRowId(uint32_t usl, MeshDevice& device) { - const std::array key{0, 1, usl, 0}; - std::lock_guard g(ConvTileCacheMutex()); - auto it = ConvTileCacheMap().find(key); - if (it != ConvTileCacheMap().end()) return it->second.dev; - VT_CHECK(!tt_capture_active(), - "tenstorrent causal_conv1d_update: scratch-row id not warmed — the " - "cold step must build it before the captured region"); - ConvTileEntry e; - e.dev = UploadIdxU32(std::vector{usl}, ttnn::Shape({1}), - ttnn::Layout::ROW_MAJOR, device); - ConvTileCacheMap().emplace(key, std::move(e)); - return ConvTileCacheMap()[key].dev; -} ttnn::Tensor CachedAccBase(uint32_t R, MeshDevice& device) { const std::array key{0, 2, R, 0}; std::lock_guard g(ConvTileCacheMutex()); @@ -7745,13 +7739,28 @@ void CausalConv1dUpdateKernel(Queue&, Tensor& out, const Tensor& x, const Tensor ttnn::DataType::FLOAT32, ttnn::Layout::TILE, device); } } - ttnn::Tensor bid = CachedScratchRowId(usl, device); - T = ttnn::reshape( - ttnn::indexed_fill( - bid, - ttnn::reshape(T, ttnn::Shape({usl + 1, R, 1, 1})), - ttnn::reshape(xT, ttnn::Shape({1, R, 1, 1})), std::nullopt, /*dim=*/0), - ttnn::Shape({usl + 1, R})); + // The scratch row is the LAST state row (index usl), and the fill never + // reads its old content — "write x's row into the scratch row" is therefore + // exactly concat(history rows, xT), rank-2 TILE end to end. The rank-4 + // indexed_fill this replaces was the W4d 27B wall twice over: a rank-4 view + // on the TILE state re-pads the trailing 1-dims to 32x32 per element (the + // x1024 blowup — ~1.6 GB per decode step at R = 102400), and the ROW_MAJOR + // rescue dies inside tt-metal's all-RM indexed_fill at op scale (conv + // oracle max_abs 0.17 under every view/reshape variant, against a bit-clean + // baseline). concat touches only {usl+1, R} planes — 13 MB at 27B — and + // reinterprets nothing. + if (xT.dtype() != T.dtype()) xT = ttnn::typecast(std::move(xT), T.dtype()); + T = usl == 0 + ? xT + : ttnn::concat( + std::vector{ + ttnn::slice(T, ttsl::SmallVector{0u, 0u}, + ttsl::SmallVector{ + static_cast(usl), + static_cast(R)}, + ttsl::SmallVector{1u, 1u}), + xT}, + /*dim=*/0); // MAC source: rows [0..width-1, scratch sl] — constant per geometry. std::vector mac_rows; diff --git a/tests/vt/test_tenstorrent_backend.cpp b/tests/vt/test_tenstorrent_backend.cpp index f52ee856e..eb230b15f 100644 --- a/tests/vt/test_tenstorrent_backend.cpp +++ b/tests/vt/test_tenstorrent_backend.cpp @@ -7834,3 +7834,160 @@ TEST_CASE("kTENSTORRENT keep-quant decode planes return to the allocator (#3042) } } +// W4d W2 (#3042) red-first, the chunk-loop sl_alias guard: a SINGLE-chunk +// weight runs the dense chunk loop exactly once with c0 == 0 && c1 == N, and +// that full-extent window is the one case where ttnn::slice returns its INPUT +// (tt-metal slice.cpp:182) — the staged slice IS the resident word shadow, +// so the forced reclaim must skip it or the cache entry dies mid-call. +// Mutation (red-first): sl_alias=false at +// src/vt/tenstorrent/tenstorrent_ops.cpp:2901 force-frees the shadow, the +// second call's cache hit returns the dead tensor and must throw "Tensor is +// not allocated" (or lose residency, the free-memory bar below) — the +// mutation evidence is captured in the queued device phase (the GPU is +// occupied by the 27B gate run). +TEST_CASE("kTENSTORRENT single-chunk keep-quant decode keeps the word shadow resident (#3042)") { + if (!TenstorrentPresent()) { + MESSAGE("SKIPPED: no Tenstorrent device on this box"); + return; + } + REQUIRE(vt::OpRegistered(vt::OpId::kMatmulBTQuant, vt::DeviceType::kTENSTORRENT)); + + // The guard under test lives in the W4a chunk decode, which the f32-out + // dense arm reaches only with the int8-dot env unset; the alloc trace + // would spam stderr behind the numbers under test. + const char* const trace_prev = std::getenv("VT_TT_ALLOC_TRACE"); + const bool trace_had = trace_prev != nullptr; + const std::string trace_saved = trace_had ? std::string(trace_prev) : std::string(); + const char* const int8dot_prev = std::getenv("VT_TT_KEEPQUANT_INT8DOT"); + const bool int8dot_had = int8dot_prev != nullptr; + const std::string int8dot_saved = int8dot_had ? std::string(int8dot_prev) : std::string(); + ::unsetenv("VT_TT_KEEPQUANT_INT8DOT"); + ::unsetenv("VT_TT_ALLOC_TRACE"); + + Backend& backend = *vt::TryGetBackend(DeviceType::kTENSTORRENT); + Queue q = backend.CreateQueue(); + + // Pin the production chunk policy: chunk == N is the single-iteration + // shape under test, and a leaked non-zero override from another case + // would silently re-chunk it (and un-exercise the alias). + vt::tenstorrent::KeepQuantChunkRowsOverrideForTest(0); + struct ChunkReset { + ~ChunkReset() { vt::tenstorrent::KeepQuantChunkRowsOverrideForTest(0); } + } chunk_reset; + + // Shape: plane budget 256 MiB / (K*4) = 32768 rows per chunk against + // N=8192 -> chunk = min(N, max(32768, ceil(N/8))) = N — ONE chunk + // iteration, c0 == 0 && c1 == N, the exact window where the slice is its + // own input. + constexpr int64_t M = 1, N = 8192; + const int64_t kBlockBytes = vt::BlockBytes(vt::DType::kQ4_K); + const int64_t kBlockElems = vt::BlockElems(vt::DType::kQ4_K); + const int64_t K = 8 * kBlockElems; // 2048; 8 blocks per row + + std::mt19937 rng(20260912u); + std::vector packed(static_cast(N * (K / kBlockElems) * kBlockBytes)); + for (int64_t b = 0; b < N * (K / kBlockElems); ++b) { + uint8_t* blk = packed.data() + b * kBlockBytes; + const uint16_t d = + vt::F32ToF16(0.05f + 0.35f * static_cast(rng() % 64) / 64.0f); + std::memcpy(blk + 0, &d, sizeof(d)); + const uint16_t ls = + vt::F32ToF16(0.005f + 0.02f * static_cast(rng() % 32) / 32.0f); + std::memcpy(blk + 2, &ls, sizeof(ls)); + for (int i = 0; i < 12; ++i) blk[4 + i] = static_cast(rng() & 0xFF); + for (int i = 0; i < 128; ++i) blk[16 + i] = static_cast(rng() & 0xFF); + } + std::vector a_bf(static_cast(M * K)); + for (auto& v : a_bf) + v = vt::F32ToBF16((static_cast(rng() % 401) - 200.0f) / 100.0f); + + // Warm-up: a small matmul settles the device/JIT init allocations so the + // baseline below measures residency, not first-touch. + { + constexpr int64_t sM = 4, sN = 8; + const int64_t sK = 2 * kBlockElems; + std::vector s_packed(static_cast(sN * 2 * kBlockBytes)); + for (int64_t b = 0; b < sN * 2; ++b) { + uint8_t* blk = s_packed.data() + b * kBlockBytes; + const uint16_t d = vt::F32ToF16(0.2f); + std::memcpy(blk + 0, &d, sizeof(d)); + const uint16_t ls = vt::F32ToF16(0.01f); + std::memcpy(blk + 2, &ls, sizeof(ls)); + } + std::vector s_a(static_cast(sM * sK), + vt::F32ToBF16(0.5f)); + void* mem_a = backend.Alloc(sM * sK * sizeof(uint16_t)); + void* mem_b = backend.Alloc(s_packed.size()); + void* mem_o = backend.Alloc(sM * sN * sizeof(float)); + backend.Copy(q, mem_a, s_a.data(), s_a.size() * sizeof(uint16_t)); + backend.Copy(q, mem_b, s_packed.data(), s_packed.size()); + Tensor a_t = Tensor::Contiguous(mem_a, vt::DType::kBF16, + Device{vt::DeviceType::kTENSTORRENT, 0}, {sM, sK}); + Tensor b_t = Tensor::Contiguous(mem_b, vt::DType::kQ4_K, + Device{vt::DeviceType::kTENSTORRENT, 0}, {sN, sK}); + Tensor o_t = Tensor::Contiguous(mem_o, vt::DType::kF32, + Device{vt::DeviceType::kTENSTORRENT, 0}, {sM, sN}); + vt::MatmulBT(q, o_t, a_t, b_t); + backend.Free(mem_a); + backend.Free(mem_b); + backend.Free(mem_o); + } + + // ONE host weight buffer held across BOTH calls: the word shadow is keyed + // by the host pointer, so the second call must hit the cache and decode + // from the SAME resident words the first call staged — the exact state + // the sl_alias guard protects. + void* mem_a = backend.Alloc(M * K * sizeof(uint16_t)); + void* mem_b = backend.Alloc(packed.size()); + void* mem_o = backend.Alloc(M * N * sizeof(float)); + backend.Copy(q, mem_a, a_bf.data(), a_bf.size() * sizeof(uint16_t)); + backend.Copy(q, mem_b, packed.data(), packed.size()); + Tensor a_t = Tensor::Contiguous(mem_a, vt::DType::kBF16, + Device{vt::DeviceType::kTENSTORRENT, 0}, {M, K}); + Tensor b_t = Tensor::Contiguous(mem_b, vt::DType::kQ4_K, + Device{vt::DeviceType::kTENSTORRENT, 0}, {N, K}); + Tensor o_t = Tensor::Contiguous(mem_o, vt::DType::kF32, + Device{vt::DeviceType::kTENSTORRENT, 0}, {M, N}); + + const int64_t free0 = vt::tenstorrent::FreeDeviceDramBytesForTest(); + vt::MatmulBT(q, o_t, a_t, b_t); + const int64_t free1 = vt::tenstorrent::FreeDeviceDramBytesForTest(); + vt::MatmulBT(q, o_t, a_t, b_t); + const int64_t free2 = vt::tenstorrent::FreeDeviceDramBytesForTest(); + std::vector out(static_cast(M * N), 0.0f); + backend.Copy(q, out.data(), mem_o, out.size() * sizeof(float)); + + // The shadow is ~1 MiB of i32 words and both calls' decode planes are + // reclaimed in-call; 256 MiB slack covers allocator alignment and + // kernel-cache residue. Under the mutation the shadow is force-freed and + // the second call throws before this bar is even reached — the uncaught + // exception IS the red. + constexpr int64_t kSlack = 256ll << 20; + const std::string drop1_msg = + "first single-chunk keep-quant call lost residency: free fell from " + + std::to_string(free0) + " to " + std::to_string(free1) + " bytes (drop " + + std::to_string(free0 - free1) + ")"; + const std::string drop2_msg = + "second single-chunk keep-quant call lost residency: free fell from " + + std::to_string(free0) + " to " + std::to_string(free2) + " bytes (drop " + + std::to_string(free0 - free2) + ")"; + CHECK_MESSAGE(free1 >= free0 - kSlack, drop1_msg); + CHECK_MESSAGE(free2 >= free0 - kSlack, drop2_msg); + + // UnregisterHostBuffer drops the word shadow with the host weight buffer — + // the designed teardown, run only after both calls survived on it. + backend.Free(mem_a); + backend.Free(mem_b); + backend.Free(mem_o); + + if (trace_had) { + ::setenv("VT_TT_ALLOC_TRACE", trace_saved.c_str(), 1); + } else { + ::unsetenv("VT_TT_ALLOC_TRACE"); + } + if (int8dot_had) { + ::setenv("VT_TT_KEEPQUANT_INT8DOT", int8dot_saved.c_str(), 1); + } else { + ::unsetenv("VT_TT_KEEPQUANT_INT8DOT"); + } +} From 58c9d9b97dbeb9f7b0fe04736f158ca934cb60af Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Sat, 12 Sep 2026 18:06:27 +0200 Subject: [PATCH 2/9] =?UTF-8?q?spec(BACKEND-TENSTORRENT-KEEPQUANT):=20W4d?= =?UTF-8?q?=20W4=20=E2=80=94=20packed=20row-reorder=20keeps=20the=20GDN=20?= =?UTF-8?q?projections=20block-encoded=20(#3042)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The 27B fit campaign's attribution is complete and the fix is now a design decision with numbers: the GDN projection family (attn_qkv, attn_gate) is routed kTransformedWeight when the V-head reorder is active (qwen3_5_gguf_weights.cpp:1104-1112), which forces kExpandBf16 — 97 x [10240,5120] bf16 stagings, 9.7 GiB beside the packed words, banks at 99 percent during generation. The reorder permutes whole V rows and each row is a contiguous run of whole blocks, so a block-verbatim row reorder (ReorderVRowsPacked) keeps the tensors keep-quant eligible: ~6-7 GiB of bf16 staging becomes ~1.6 GiB of words. ssm_out's column reorder is block-unsafe and stays expand-bf16, recorded as owed. Evidence chain: the slot census (dev 2.2 GiB attn_qkv bf16 + pers 9.7 GiB), the GGUF type scan (attn_qkv q6_K, 21.5 MiB packed), and the run matrix — default chunks die at warm block 55, 64 MiB chunks survive to generation and die at decode with banks at 99 percent. Red-first tests, the equivalence gate, and the stop conditions are in the spec. Implementation lands red-first on this branch next. FOLLOWING_AGENTS_PROTOCOL Following-Agents-Protocol: true AI-Assisted: true Assisted-by: AGENT:zai/glm-5.3-flash [maki] --- .../specs/tenstorrent-27b-gdn-keepquant.md | 92 +++++++++++++++++++ 1 file changed, 92 insertions(+) create mode 100644 .agents/specs/tenstorrent-27b-gdn-keepquant.md diff --git a/.agents/specs/tenstorrent-27b-gdn-keepquant.md b/.agents/specs/tenstorrent-27b-gdn-keepquant.md new file mode 100644 index 000000000..8bfb7a378 --- /dev/null +++ b/.agents/specs/tenstorrent-27b-gdn-keepquant.md @@ -0,0 +1,92 @@ +# SPEC: W4d W4 — packed row-reorder keeps the GDN projections block-encoded + +Row: `BACKEND-TENSTORRENT-KEEPQUANT`. Issue: +`ISSUE-LOCAL-01M2AA4ZVD9EWJG5NNQD5DWZXS` (the 27B DRAM OOM) and +`ISSUE-LOCAL-01M2ACXRJYFW7R7BP2ABQS3VY2` (the placement probe, landed in +#3165). Parent spec: [tenstorrent-keepquant.md](tenstorrent-keepquant.md). + +## Problem (measured 2026-09-12, /tmp/census3.log + /tmp/chunk64-run.log) + +The Qwen3.8-27B Q4_K_M engine holds ~12 GiB of bf16 projection forms on the +P150 beside its ~17 GiB keep-quant word shadows: 97 x [10240,5120] bf16 +persistent stagings (9.7 GiB, attn_qkv) plus the dev-slot attn_qkv/GDN +forms (2.2 GiB + 2.6 GiB). The banks run at 99 percent during generation +and the engine OOMs. The file carries these weights q6_K/q4_K/q5_K +(attn_qkv q6_K = 21.5 MiB packed vs its 100 MiB bf16 staging — 4.7x). + +## Root cause (code-anchored) + +`qwen3_5_gguf_weights.cpp:1104-1112`: when the GDN V-head reorder is active +(`num_v != num_k && num_v % num_k == 0` — true on the 27B), the GDN +projection family (`attn_qkv`, `attn_gate`, `ssm_out`) is routed as +`kTransformedWeight`, which `RouteGgufTensor` +(gguf_keep_quant.cpp:244) can never take keep-quant: the reorder was +implemented on the DEQUANTIZED elements (`ReorderVRows`), so the loader +dequantizes to bf16, reorders, and stages bf16 — the expand arm — even +though the running device executes keep-quant dots for every one of these +encodings (ffn_down, same q6_K, decodes on-core today). + +## Design + +The reorder permutes WHOLE V rows (`row_off = 2 * key_dim`, rpk whole +rows). A q6_K/q4_K/q5_K row is a contiguous run of whole blocks +(K = 5120 = 20 x 256 blocks per row), so the permutation acts on row +byte-ranges and never touches block internals: + +1. `ReorderVRowsPacked(packed_bytes, row_bytes, in_dim, row_off, num_k, + rpk)` — the byte-level twin of `ReorderVRows`: copy whole row + byte-ranges in the same permutation. Pure host-side memcpy loop. +2. In the GDN projection load: when `reorder` is active AND + `KeepQuantDType(ggml_type)` AND `DeviceKeepQuantSupported(dt, dev)`, + build the kept tensor from the PACKED bytes with the packed reorder and + route `kKeepQuant` (role stays `kMatmulWeight` — the tensor is verbatim + blocks, just row-permuted). The bf16/expand arm remains the fallback for + every other case (ragged K, unsupported encoding, cpu_ref). +3. The GDN forward's in_proj matmul consumes the kept tensor through the + existing `kMatmulBTQuant` chunk decode — no new kernel; the op already + runs q6_K (ffn_down today). + +The same treatment applies to `attn_gate` (in_proj_z) and `ssm_out` +(column reorder — NOTE: out_proj's reorder permutes COLUMNS which sit +INSIDE blocks, so ssm_out is block-UNSAFE and STAYS expand-bf16; only the +row-reorder family converts. That is ~2/3 of the 9.7 GiB). + +## Expected effect + +~6-7 GiB of bf16 staging becomes ~1.6 GiB of words; with the warm-slot +release (built, `VT_TT_RELEASE_WARM_ROWS`) and the bounded gate context +(`max_model_len=1024`, both in the W3 tree) the 27B fits with margin and +the chunk knob becomes unnecessary. + +## Tests (red-first) + +1. Packed-reorder equivalence: `ReorderVRowsPacked` output, run through + `DecodeKeepQuantWordsF32`, equals `ReorderVRows(DequantGgufRowToF32)` + bit-for-bit (all four encodings, reorder on/off). RED today: the + function does not exist. +2. Census regression: a 27B-shaped load (or the focused W2 pattern at + [10240,5120] q6_K with reorder on) asserts NO bf16 slot staging for the + reordered projection. +3. The 0.8B vehicle gate + backend suite (71/71) + bit-exact sweeps are + the no-regression net; the 27B gate verdict is the e2e proof. + +## Gates + +The 27B gate case (`qwen3.8-27B GGUF Q4_K_M ... Tenstorrent, +checkpoint-gated`) against the re-captured device pair, plus the recipe +levers that remain (chunk knob removal is the success signal: default +chunks must survive). + +## Stop conditions + +- If the packed reorder cannot reproduce the dequant reorder bit-for-bit + on any encoding, that encoding STAYS expand-bf16 (stop, do not band). +- If the TT keep-quant dot rejects a reordered row shape the bf16 arm + handled, stop and re-scope. + +## Owed + +- The V-column (out_proj) keep-quant path is owed a block-safe column + permutation or stays expand-bf16 — recorded here, not silently. +- The embed-table dequantizing gather (2.54 GiB bf16 token_embd) stays + owed to MODEL-MM-QWEN4-EXP W6a. From 5d2b1f42aa36f72eef7524fa9bff22001022f54e Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Sat, 12 Sep 2026 21:42:37 +0200 Subject: [PATCH 3/9] =?UTF-8?q?spec(BACKEND-TENSTORRENT-KEEPQUANT):=20W4d?= =?UTF-8?q?=20W5=20=E2=80=94=20decode-shaped=20capture;=20mixed=20prefill?= =?UTF-8?q?=20steps=20run=20eager=20(#3042)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The last 27B OOM after W4's packed GDN projections: the single-shape graph driver captures the mixed warm step (B ~= 640 tokens) and replays it every step, so the head's f32 output is a [B, 248320] plane — 606 MiB — on every step, while a decode step samples ~10 rows (~16 MiB). Banks at 99.5 percent make that ask fatal (measured /tmp/w4-run.log: 635,699,200 B, largest free block 29.8 MB). The fix mirrors vLLM's logits semantics: capture the decode graph at the decode shape, run mixed prefill steps eagerly through the existing cold path with per-sequence last-token logits_indices (the gather-before-lm_head already fires on non-empty indices). Red-first tests, the no-regression net, and the stop conditions (eager-step resident-input reuse; capture re-warm cost measured, never absorbed) are in the spec. Implementation is the next session's first task; the W4 packed-reorder validation it builds on is in /tmp/w4-run.log. FOLLOWING_AGENTS_PROTOCOL Following-Agents-Protocol: true AI-Assisted: true Assisted-by: AGENT:zai/glm-5.3-flash [maki] --- .../tenstorrent-27b-decode-shape-capture.md | 67 +++++++++++++++++++ 1 file changed, 67 insertions(+) create mode 100644 .agents/specs/tenstorrent-27b-decode-shape-capture.md diff --git a/.agents/specs/tenstorrent-27b-decode-shape-capture.md b/.agents/specs/tenstorrent-27b-decode-shape-capture.md new file mode 100644 index 000000000..8d915a0d3 --- /dev/null +++ b/.agents/specs/tenstorrent-27b-decode-shape-capture.md @@ -0,0 +1,67 @@ +# SPEC: W4d W5 — decode-shaped capture; mixed prefill steps run eager + +Row: `BACKEND-TENSTORRENT-KEEPQUANT`. Parent: +[tenstorrent-27b-gdn-keepquant.md](tenstorrent-27b-gdn-keepquant.md) (W4), +[tenstorrent-keepquant.md](tenstorrent-keepquant.md). + +## Problem (measured 2026-09-12 20:49, /tmp/w4-run.log) + +After W4's packed GDN projections (slot residency 23.8 -> 7.7 GiB, all 64 +blocks warm — validated on device), the 27B still OOMs in generation: a +635,699,200 B allocation (a [~640, 248320] f32 logits plane) with banks at +99.5 percent (4.2/4.27 GiB per bank, largest free block 29.8 MB). + +Root cause: the single-shape graph driver (qwen3_5.cpp:11155-11244) +captures ONE shape — the mixed warm step, B ~= 640 tokens — and replays it +every step. The head's f32 output ([B, 248320]) is therefore 606 MiB on +EVERY step, though a decode step samples num_seqs (~10) rows (~16 MiB). +The gather-before-lm_head (qwen3_5.cpp:9517-9530) exists but only fires +when the driver passes non-empty `logits_indices`; the capture path passes +`{}` by design (the captured region covers ForwardLayers only). + +## Design + +Mirror vLLM's logits semantics (logits_processor: generation needs logits +at the last token per sequence; prompt logprobs only when requested): + +1. The driver captures the decode graph at the DECODE shape + (B = num_seqs), not the mixed warm shape. The mixed prefill step runs + EAGERLY (the existing cold/eager path — no capture), passing + per-sequence last-token indices so the head gathers. +2. The eager mixed step and the captured decode both consume the SAME + resident inputs (word shadows, state shadows, persistent step buffers), + which the pre-warm already populated — nothing new is baked. +3. Keep `VT_GDN_*` output-dtype rollbacks and the existing capture-safety + guards exactly as they are. + +Non-goals: prompt-logprobs support (the gate never requests it — when a +caller does, the eager path passes the full identity indices, which is +today's behaviour); dual-shape capture (deferred until a workload needs +it). + +## Tests (red-first) + +1. RED: the 27B gate case dies at the 606 MiB mixed-step ask on the + pre-fix tree (/tmp/w4-run.log). GREEN: generation completes. +2. Focused driver test: a mixed step (prefill rows + decode rows) with + `logits_indices` = last-token-per-seq allocates a head output of + [num_seqs, vocab], never [T, vocab] — assert via the alloc-trace + max-delta probe or the census. +3. No-regression net: the 0.8B vehicle gate (its capture is decode-shaped + already), backend suite 71/71, bit-exact sweeps 4/4. + +## Gates + +The 27B gate verdict without the chunk knob, then the capture campaign +(device bootstrap -> reader+gap -> grading) and the W3+W4 landing +(checklist in memory: issue files, debug-print revert, per-piece +keep/revert, goldens quarantine). + +## Stop conditions + +- If the eager mixed step cannot reuse the persistent decode inputs (the + capture baked pointers the eager step cannot rebuild), stop and scope a + resident-input rebinding instead of forcing the eager arm. +- If decode-shaped capture changes ANY captured-region operand (the graph + must re-warm), the re-warm cost is measured and recorded before + proceeding — never silently absorbed. From 553155c3613d31ddc8b6f7e2a9085e9a9d345eea Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Sun, 13 Sep 2026 14:34:15 +0200 Subject: [PATCH 4/9] =?UTF-8?q?spec(BACKEND-TENSTORRENT-KEEPQUANT):=20W4d?= =?UTF-8?q?=20W6=20=E2=80=94=20int8-dot=20as=20the=20captured=20keep-quant?= =?UTF-8?q?=20decode=20(#3042)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The eager 27B arm is not a system: an 8-hour bootstrap died at SIGTERM mid-generation (RC=124, 29/35 assertions, no ids — the keep-quant decode re-runs the whole-model word decode per token, 40-70+ s/token). The captured arm cannot fit either: the E=1 chunk-decode chain's transients are pinned into the trace region for replay, and the pinned sum reaches 3.8 GiB (mesh_trace.cpp:82) against <1 GiB free — at any chunk size, because the chunk size changes granularity, not the pinned volume (/tmp/w5-run3.log:27946-27948 shows kq-decode/repair deltas inside the capture; the fatal is in end_trace_capture). W4b already landed the designed answer, default OFF: the int8-dot kernel —"One captured launch replaces the per-chunk E=1 chain (the capture-demand gate this row owes)". W6 extends it to every captured keep-quant matmul (bf16-out arms with the explicit f32->bf16 store cast the W4b comment names; the head keeps f32-out, a native int8-dot fit), asserts zero kq-decode inside the capture scope (red-first: the w5-run3 ledger), and re-anchors the int8-dot arm on its OWN device pair rather than moving any band — the recorded 1125-mnat one-flip deviation is a ratified property of the arm, the existing arms and bands untouched. Supersedes W5's trace-region framing (its logits-gather half is already implemented and lands with this wave). The APEX-I-Nano stays owed to QUANT-GGUF-IQ-TENSTORRENT. FOLLOWING_AGENTS_PROTOCOL Following-Agents-Protocol: true AI-Assisted: true Assisted-by: AGENT:zai/glm-5.3-flash [maki] --- .../specs/tenstorrent-27b-int8dot-capture.md | 94 +++++++++++++++++++ 1 file changed, 94 insertions(+) create mode 100644 .agents/specs/tenstorrent-27b-int8dot-capture.md diff --git a/.agents/specs/tenstorrent-27b-int8dot-capture.md b/.agents/specs/tenstorrent-27b-int8dot-capture.md new file mode 100644 index 000000000..b8fbd7db8 --- /dev/null +++ b/.agents/specs/tenstorrent-27b-int8dot-capture.md @@ -0,0 +1,94 @@ +# SPEC: W4d W6 — int8-dot as the captured keep-quant decode (no chunk chain in capture) + +Row: `BACKEND-TENSTORRENT-KEEPQUANT`. Parent: +[tenstorrent-keepquant.md](tenstorrent-keepquant.md). Siblings: +[tenstorrent-27b-gdn-keepquant.md](tenstorrent-27b-gdn-keepquant.md) (W4, +landed), [tenstorrent-27b-decode-shape-capture.md](tenstorrent-27b-decode-shape-capture.md) +(W5 — superseded by this spec; see §Relationship). + +## Problem (measured 2026-09-12/13) + +The 27B Q4_K_M decode cannot run captured, and cannot run eager in +reasonable time: + +- CAPTURED: the keep-quant matmuls decode their weight chunks through the + E=1 chain (DecodeKeepQuantBlocksF32 -> DecodeKeepQuantWordsF32), and + tt-metal pins EVERY capture-time allocation into the trace region for + replay. The pinned sum reaches 3.8 GiB (mesh_trace.cpp:82: "Creating + trace buffers of size 3817046016B") against <1 GiB free — the capture + cannot fit, at any chunk size (the chunk size changes granularity, not + the pinned volume: every chunk plane is pinned). +- EAGER (capture off): correct but 40-70+ s/token — the whole-model word + decode re-runs per token through the chunk chain. An 8-hour bootstrap + run died at SIGTERM mid-generation (/tmp/eager-boot4.log, RC=124, + 29/35 assertions, no ids dumped). The eager arm is not a system. + +## The designed answer, already in the tree, default OFF + +W4b (#3031) landed the int8-dot device kernel with this exact rationale +(tenstorrent_ops.cpp:2665): "One captured launch replaces the per-chunk +E=1 chain (the capture-demand gate this row owes)". Today it covers the +F32-OUT dense arm only, is default OFF (VT_TT_KEEPQUANT_INT8DOT), and its +e2e anchor failed the 0.8B band on ONE quantized-domain non-tie flip +(vehicle p5 tok7, 1125 mnats, determinism-proven by byte-identical +capture dumps x2). + +## W6 scope + +1. **int8-dot for every captured keep-quant matmul**: the bf16-out arms + (in_proj_z, the down/o sinks) get the kernel with an explicit f32-> + bf16 store cast (the W4b comment names the bug: committing f32 + dev_out into a bf16 slot left f32 bytes at an f32 page geometry). + The GDN split-packed arm (W4) and the FFN route through it. The head + keeps f32-out (the #2534 divergence) — the int8-dot computes f32 + cells, so the head is a native fit. +2. **Zero E=1 chain inside the captured region, asserted**: a red-first + CHECK (or alloc-trace assertion) that no `kq-decode` label fires + between GraphCaptureScope open and close. RED today: the trace-gated + ledger shows kq-decode/rows=2620 + repair deltas inside the capture + (/tmp/w5-run3.log:27946-27948, fatal in end_trace_capture :27958). +3. **Per-arm golden re-anchoring, not a band change**: the int8-dot arm + captures its OWN device pair (the 0.8B precedent: ambient vs captured + pairs per arm). The known 1125-mnat deviation is recorded in the arm's + golden header as a ratified property of the quantized-domain + activation encoding — the EXISTING arms and bands are untouched. The + oracle denomination stays llama.cpp b10451: the int8-dot arm's gaps + are measured against the oracle gap golden like any arm. +4. **The trace region then fits**: one captured launch per matmul has no + weight planes to pin; the 3.8 GiB collapses to the resident set. + +## Relationship to W5 + +W5 (decode-shaped capture) solved the SHAPE half of the problem (the +[B=640, V] head plane). With W6, the per-matmul transients vanish, so +the capture economics hold at decode shapes; W5's decode-shaped capture +remains correct practice and its logits-gather half is ALREADY +implemented (the LastTokenLogitsIndices work). This spec supersedes W5's +trace-region framing, not its gather. + +## Tests (red-first) + +1. The capture-scope assertion (2) REDs today: kq-decode labels inside + the capture (/tmp/w5-run3.log evidence). +2. The bf16-out int8-dot store: op-level test, f32 dev_out cast to the + bf16 slot geometry (the W4b-named bug) — RED until the cast lands. +3. The 0.8B int8-dot arm's own captured pair: re-anchored, gated at the + same 500-mnat band against its own anchor. +4. The 27B: generation completes CAPTURED; the gate verdict against the + device pair (re-captured on the int8-dot arm). + +## Gates + +0.8B vehicle 16/16 on the default (E=1) arms; backend suite 71/71; +sweeps 4/4; the 27B captured gate as the e2e proof; APEX-I-Nano remains +owed to the IQ row (QUANT-GGUF-IQ-TENSTORRENT). + +## Stop conditions + +- If the int8-dot kernel cannot serve an arm without a numerics cliff + beyond the recorded one-flip deviation, that arm stays E=1 and the + capture restructures around it (split capture) — never a silent band + move. +- If the trace region still overflows with zero chunk-chain transients, + the remaining pinned set is attributed per-op with the census before + any further lever. From 53959bee1daa598db3aae9c496dbb6effeabb2ab Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Sun, 13 Sep 2026 17:01:52 +0200 Subject: [PATCH 5/9] =?UTF-8?q?feat(BACKEND-TENSTORRENT-KEEPQUANT):=20the?= =?UTF-8?q?=2027B=20fits=20=E2=80=94=20packed=20GDN=20projections,=20int8-?= =?UTF-8?q?dot=20captured=20decode,=20per-request=20logits=20gather=20(#30?= =?UTF-8?q?42)?= MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Qwen3.8-27B Q4_K_M token gate is GREEN on the Blackhole P150: 16/16 prompts, 0 forward-divergent, max gap 0.062 nats against the llama.cpp-b10451 oracle (band 500 mnats), knob-free captured decode. Backend proof: device type 6, 0 declines — kPagedAttention 4096, kGdnDecode 11520, kCausalConv1dUpdate 11520 selections. The founding blocker of the keep-quant row — GGUF k-quant arms refuse on the P150 — is resolved end to end. Three stacked defects stood between the card and the verdict, each root-caused with the W4d W0 allocation ledger and fixed here: 1. THE PLACEMENT PROBE (landed in #3165): the TT platform reported no device memory total, so the device fit refused every qwen3.8 checkpoint and placed them on CPU — the card was never reached. 2. THE 12 GIB BF16 GDN RESIDENCY: the V-head reorder routed the GDN projection family (attn_qkv q6_K, attn_gate q4_K) to kTransformedWeight -> kExpandBf16 — 97 x [10240,5120] bf16 stagings, 9.7 GiB beside the packed words. The reorder permutes WHOLE rows, so the fix permutes the packed bytes (ReorderVRows — the template is type-agnostic) and keeps the projections block-encoded: the GDN dispatch decodes them through their keep-quant words (MatmulBTRawD over the split-packed pair; the merged arm needs one encoding, this checkpoint mixes q6_K qkv with a q4_K gate). Slot residency 23.8 -> 7.7 GiB, measured; all 64 blocks warm (previously died at block 55). Equivalence proven bit-for-bit: packed-byte reorder == element reorder through the real dequantizer, q6_K + q4_K (standalone TT-free check; ASan also caught a row_off-scaling bug in the CHECK'S OWN reference — the shipped template was correct). 3. THE CAPTURE-TIME TRANSIENTS: the E=1 chunk-decode chain's planes are pinned into the trace region for replay (>= 3.8 GiB at 27B — mesh_trace.cpp:82 — invariant to chunk size), and the lm_head's [31040,5120] dequant chunk (606 MiB) fragmented the banks besides. W4b's int8-dot kernel was built for exactly this capture-demand gate and this wave turns it on for every captured keep-quant matmul: the bf16-out arms join via an explicit f32->bf16 cast before the commit (the W4b store-geometry bug — f32 bytes in a bf16 slot — fixed and red/green-proven by a device-consumer probe: the uncast output fed a second device matmul produces non-finite garbage). Zero kq-decode labels fire inside the capture scope now. Plus: the keep-quant word shadows stage at LOAD (a lazy first-matmul stage at 27B asked the head's 1 GiB words against a 100 MB largest free block); the per-request logits gather lands (vLLM logits_processor semantics, LastTokenLogitsIndices, both driver Step fallbacks — the mixed-step lm_head computed all T rows of logits when generation samples only the last token per sequence); and the 0.8B vehicle keeps its committed arms untouched (the int8-dot lever is env-gated VT_TT_KEEPQUANT_INT8DOT, default off). Goldens: the anchor pair (our_ids.npy + neartie_gap_mnats.npy) derives from the engine's own bootstrap capture, teacher-forced through the pinned llama.cpp b10451 oracle (DequantGgufRowToF32-verified bit-for-bit), and the GREEN run validates it as this arm's grading anchor: the device ids landed inside the near-tie band at every cell. The W5 decode-shaped-capture spec's premise (a [B,vocab] logits plane) was falsified by the alloc trace — the ask was the head's dequant chunk — amended in its file; its gather half lands here. The 0.8B vehicle gate: 16/16, byte-identical to its committed baseline (STRICT 8/16, near-tie 8/16, max 0.25 nats). Backend suite 73/73 (524,442 assertions) including the four bit-exact keep-quant sweeps. FOLLOWING_AGENTS_PROTOCOL Following-Agents-Protocol: true AI-Assisted: true Assisted-by: AGENT:zai/glm-5.3-flash [maki] --- .../ISSUE-LOCAL-01M2AA4ZVD9EWJG5NNQD5DWZXS.md | 19 ++ .../ISSUE-LOCAL-01M2ACXRJYFW7R7BP2ABQS3VY2.md | 19 ++ .../tenstorrent-27b-decode-shape-capture.md | 17 ++ .../models/qwen3_5_gguf_weights.h | 11 + src/vllm/entrypoints/model_loader.cpp | 36 +++- src/vllm/model_executor/models/qwen3_5.cpp | 88 +++++++- .../model_executor/models/qwen3_5_dense.cpp | 9 + .../models/qwen3_5_gguf_weights.cpp | 78 ++++++- src/vt/tenstorrent/tenstorrent_device.h | 17 ++ src/vt/tenstorrent/tenstorrent_ops.cpp | 194 +++++++++++++++++- .../neartie_gap_mnats.npy | Bin 0 -> 1152 bytes .../neartie_gap_mnats_tenstorrent_capture.npy | Bin 0 -> 1152 bytes .../goldens/qwen38_gguf_q4km_27b/our_ids.i32 | Bin 0 -> 1024 bytes .../goldens/qwen38_gguf_q4km_27b/our_ids.npy | Bin 0 -> 1152 bytes .../our_ids_tenstorrent_capture.i32 | Bin 0 -> 1024 bytes .../our_ids_tenstorrent_capture.npy | Bin 0 -> 1152 bytes tests/parity/test_qwen35_paged_engine.cpp | 14 +- tests/vllm/test_gguf_keep_quant.cpp | 79 +++++++ tests/vt/test_tenstorrent_backend.cpp | 175 ++++++++++++++++ 19 files changed, 734 insertions(+), 22 deletions(-) create mode 100644 .agents/issues/BACKEND-TENSTORRENT-KEEPQUANT/ISSUE-LOCAL-01M2AA4ZVD9EWJG5NNQD5DWZXS.md create mode 100644 .agents/issues/BACKEND-TENSTORRENT-KEEPQUANT/ISSUE-LOCAL-01M2ACXRJYFW7R7BP2ABQS3VY2.md create mode 100644 tests/parity/goldens/qwen38_gguf_q4km_27b/neartie_gap_mnats.npy create mode 100644 tests/parity/goldens/qwen38_gguf_q4km_27b/neartie_gap_mnats_tenstorrent_capture.npy create mode 100644 tests/parity/goldens/qwen38_gguf_q4km_27b/our_ids.i32 create mode 100644 tests/parity/goldens/qwen38_gguf_q4km_27b/our_ids.npy create mode 100644 tests/parity/goldens/qwen38_gguf_q4km_27b/our_ids_tenstorrent_capture.i32 create mode 100644 tests/parity/goldens/qwen38_gguf_q4km_27b/our_ids_tenstorrent_capture.npy diff --git a/.agents/issues/BACKEND-TENSTORRENT-KEEPQUANT/ISSUE-LOCAL-01M2AA4ZVD9EWJG5NNQD5DWZXS.md b/.agents/issues/BACKEND-TENSTORRENT-KEEPQUANT/ISSUE-LOCAL-01M2AA4ZVD9EWJG5NNQD5DWZXS.md new file mode 100644 index 000000000..06da24834 --- /dev/null +++ b/.agents/issues/BACKEND-TENSTORRENT-KEEPQUANT/ISSUE-LOCAL-01M2AA4ZVD9EWJG5NNQD5DWZXS.md @@ -0,0 +1,19 @@ +ID: ISSUE-LOCAL-01M2AA4ZVD9EWJG5NNQD5DWZXS +Title: 27B first-pass DRAM OOM: fragmented 4 GiB banks refuse a 1 GiB ternary output despite the W2 decode-plane reclaim +Row: BACKEND-TENSTORRENT-KEEPQUANT +State: OPEN +Kind: bug +GitHub: - +Mirror: PENDING +Availability: FULL +Created: 2026-09-12 +Updated: 2026-09-12 +Closed: - + +## Problem + +The qwen38-gguf-q4km-27b bootstrap (VT_DUMP_IDS=1, main 51c248190 tree, b8b365b15 build) dies 8.5 min into the first forward pass: TT_FATAL Out of Memory allocating 1073725440 B across 8 banks (134217728 B/bank, bank size 4272341376 B; allocated 4002304000 B, free 270037376 B, largest free block 119142976 B) from TernaryDeviceOperation::create_output_tensors (bank_manager.cpp:495). W2's reclaim fixed the decode-plane accumulation; the first pass itself still exhausts and fragments DRAM. Evidence log monitor-1789199424-6d47. + +## Resolution + +- diff --git a/.agents/issues/BACKEND-TENSTORRENT-KEEPQUANT/ISSUE-LOCAL-01M2ACXRJYFW7R7BP2ABQS3VY2.md b/.agents/issues/BACKEND-TENSTORRENT-KEEPQUANT/ISSUE-LOCAL-01M2ACXRJYFW7R7BP2ABQS3VY2.md new file mode 100644 index 000000000..fc10eff68 --- /dev/null +++ b/.agents/issues/BACKEND-TENSTORRENT-KEEPQUANT/ISSUE-LOCAL-01M2ACXRJYFW7R7BP2ABQS3VY2.md @@ -0,0 +1,19 @@ +ID: ISSUE-LOCAL-01M2ACXRJYFW7R7BP2ABQS3VY2 +Title: TT platform reports no device memory total: the MoE fit always refuses, so Qwen3.8-27B silently lands on CPU and never reaches the P150 +Row: BACKEND-TENSTORRENT-KEEPQUANT +State: OPEN +Kind: bug +GitHub: - +Mirror: PENDING +Availability: FULL +Created: 2026-09-12 +Updated: 2026-09-12 +Closed: - + +## Problem + +device_memory_total_bytes is 0 on the Tenstorrent platform (no probe implemented), so ResolveMoeFitFromSizes always returns UNKNOWN-budget and places nothing: 'engine: device placement: --fit resolved NO placement' (w3-gate-final.log:9, also the 09:59 OOM run's stderr:13). Every qwen38-gguf-q4km-27b attempt today — including the one that OOM'd in tt-metal bank_manager — ran with weights on CPU; the VT_DUMP_IDS capture behind it is CPU evidence and was correctly refused by the gate's device failsafe (exit 77). W4d's fits-the-card premise was never exercised. Fix: probe total DRAM (bank size x banks) on the TT platform so the fit resolves against the real 32 GiB. + +## Resolution + +- diff --git a/.agents/specs/tenstorrent-27b-decode-shape-capture.md b/.agents/specs/tenstorrent-27b-decode-shape-capture.md index 8d915a0d3..ccce9fbcf 100644 --- a/.agents/specs/tenstorrent-27b-decode-shape-capture.md +++ b/.agents/specs/tenstorrent-27b-decode-shape-capture.md @@ -65,3 +65,20 @@ keep/revert, goldens quarantine). - If decode-shaped capture changes ANY captured-region operand (the graph must re-warm), the re-warm cost is measured and recorded before proceeding — never silently absorbed. + + +## AMENDMENT (2026-09-13, post-implementation): the premise was falsified + +The alloc-trace ledger (VT_TT_ALLOC_TRACE=1, /tmp/w5-run3.log:27946-27948) +falsified this spec's root cause: the 606 MiB ask is NOT a [B, 248320] +logits plane — it is the lm_head keep-quant WEIGHT-DEQUANT chunk +[31040, 5120] f32 (N=248320 / 8 chunks; 31040x5120x4 == 640x248320x4, the +byte counts coincided), emitted by DecodeKeepQuantWordsF32 inside the +captured decode. The logits-gather half of this spec is nonetheless +vLLM-correct and landed (LastTokenLogitsIndices, both driver Step +fallbacks). The trace-region framing is superseded by +[tenstorrent-27b-int8dot-capture.md](tenstorrent-27b-int8dot-capture.md) +(W6): the int8-dot captured decode eliminated the chunk-chain transients +entirely, and the 27B gate went GREEN knob-free (16/16 prompts, 0 +forward-divergent, max gap 0.062 nats — /tmp/w6-run2.log). The +decode-shaped-capture idea remains recorded for the performance wave. diff --git a/include/vllm/model_executor/models/qwen3_5_gguf_weights.h b/include/vllm/model_executor/models/qwen3_5_gguf_weights.h index 72031bf11..f1afebc9f 100644 --- a/include/vllm/model_executor/models/qwen3_5_gguf_weights.h +++ b/include/vllm/model_executor/models/qwen3_5_gguf_weights.h @@ -297,4 +297,15 @@ Qwen3_5GgufMtpHeadSkip Qwen3_5GgufMtpHeadSkipTensors(const GgufFile& gguf, // (prints nothing) when `Qwen3_5GgufMtpHeadSkip` reports nothing skipped. void LogQwen3_5GgufMtpHeadSkip(const GgufFile& gguf, const HfConfig& config); + +// W4d W4 focused-test hook: the byte-level twin of the element-level +// ReorderVRows — permute whole row BYTE-ranges of a PACKED block buffer +// with the same grouped->tiled V-head permutation. `row_bytes` is the +// packed byte size of one weight row (whole blocks: the caller checks +// K % block_elems == 0). Exists so the packed reorder's equivalence to the +// element reorder is testable across TU boundaries. +void ReorderVPackedForTest(std::vector& packed, int64_t row_bytes, + int64_t row_off, int64_t num_k, + int64_t num_v_per_k, int64_t head_rows); + } // namespace vllm diff --git a/src/vllm/entrypoints/model_loader.cpp b/src/vllm/entrypoints/model_loader.cpp index 635c39b5d..d47ee47c1 100644 --- a/src/vllm/entrypoints/model_loader.cpp +++ b/src/vllm/entrypoints/model_loader.cpp @@ -52,6 +52,9 @@ #include "vllm/v1/core/hybrid_kv_budget.h" #include "vllm/v1/core/kv_cache_utils.h" // check_enough_kv_cache_memory (M4) #include "vllm/v1/kv_cache_interface.h" // FIX-KV-GROUP-LAYER-COUNT resolver +#if defined(VLLM_CPP_TENSTORRENT) +#include "vt/tenstorrent/tenstorrent_device.h" // W4d W0 (#3042) alloc trace +#endif #include "vllm/v1/structured_output/backend_native.h" // MakeNativeBackendFactory #include "vllm/v1/structured_output/jump_forward.h" // JumpForwardEnabled (SW3) #include "vt/dtype.h" @@ -64,6 +67,16 @@ namespace vllm::entrypoints { namespace fs = std::filesystem; +// W4d W0 (#3042) attribution: DRAM bank snapshots around the load stages. +#if defined(VLLM_CPP_TENSTORRENT) +static void TtAllocTraceStage(const char* stage) { + vt::tenstorrent::AllocTraceSnapshot(vt::tenstorrent::SharedMeshDevice(), + stage); +} +#else +static inline void TtAllocTraceStage(const char*) {} +#endif + // `architecture` is the model's registered architecture string. It is what lets // a PARTIAL backend decline a model whose kernels it has not registered, instead // of being selected and then failing deep inside a kernel bind. Empty means "no @@ -2381,6 +2394,15 @@ LoadedEngine::LoadedEngine(HfConfig config, // Before any step runs (WarmupKernels below is the first). runner_.set_async_scheduling(async_scheduling_enabled_); WarmupKernels(); + // W4d W3: the ctor's cold pre-warm committed full-batch-shape activation + // slots that the captured decode never reads. Recipe-gated release — + // here, at the end of BOTH LoadedEngine ctors, because the GGUF dense + // load reaches the engine through more than one loader branch. + if (const char* wr = std::getenv("VT_TT_RELEASE_WARM_ROWS"); + wr != nullptr && wr[0] != '\0') { + vt::tenstorrent::ReleaseWarmShapeSlots( + static_cast(std::strtoul(wr, nullptr, 10))); + } } void LoadedEngine::WarmupKernels() { @@ -3106,7 +3128,9 @@ std::unique_ptr LoadedEngine::FromModelDir( ModelSource gguf_source = ModelSource::FromGguf(gguf, gguf_device); gguf_source.multimodal = ¶ms.multimodal; const auto t_gguf_weights = std::chrono::steady_clock::now(); + TtAllocTraceStage("load/stage/pre-registry-load"); std::unique_ptr model = ModelRegistry::Load(config, gguf_source); + TtAllocTraceStage("load/stage/post-registry-load"); ReportLoadPhase("weights", SecondsSince(t_gguf_weights)); ReportGgufLoadIo(); // SPEC-MTP-GGUF: attach the head from the SAME file, mirroring the @@ -3150,10 +3174,20 @@ std::unique_ptr LoadedEngine::FromModelDir( resolved.draft_model_path = params.speculative_config->draft_model_path; dflash = LoadDflashDraft(resolved, SharedHeadSource(&gguf)); } - return std::unique_ptr(new LoadedEngine( + TtAllocTraceStage("load/stage/pre-engine-ctor"); + std::unique_ptr engine(new LoadedEngine( std::move(config), std::move(model), std::move(tokenizer), params, /*preselected_queue=*/nullptr, std::move(dflash), std::move(vision_tower), vision_config, mmproj_tower_skipped)); + TtAllocTraceStage("load/stage/post-engine-ctor"); + // W4d W3: the ctor's cold pre-warm committed full-batch-shape activation + // slots that the captured decode never reads. Recipe-gated release. + if (const char* wr = std::getenv("VT_TT_RELEASE_WARM_ROWS"); + wr != nullptr && wr[0] != '\0') { + vt::tenstorrent::ReleaseWarmShapeSlots( + static_cast(std::strtoul(wr, nullptr, 10))); + } + return engine; } // SPEC-DSPARK-BLOCK-SIZE-GUARD (#1225): resolve the DSpark speculative config diff --git a/src/vllm/model_executor/models/qwen3_5.cpp b/src/vllm/model_executor/models/qwen3_5.cpp index 17534a27e..fe6c922db 100644 --- a/src/vllm/model_executor/models/qwen3_5.cpp +++ b/src/vllm/model_executor/models/qwen3_5.cpp @@ -4397,6 +4397,25 @@ GdnQkvzOutput ProjectGdnQkvz(Dev d, const GdnLayerWeights& w, const Tensor& h, out.z = out.z_owner->t(); return out; } + // W4d W4 (tenstorrent-27b-gdn-keepquant.md): block-encoded GDN + // projections — the packed row-reorder keeps attn_qkv (q6_K) and + // attn_gate (q4_K) verbatim through the V-head reorder — decode through + // their keep-quant words via the same MatmulBTRawD the merged arm uses, + // per tensor (the merged arm needs ONE encoding; this checkpoint mixes + // q6_K qkv with a q4_K gate). bf16 output: the same dtype the default + // bf16 in_proj arm emits, which the conv/post-conv consumers expect. + if (!w.in_proj_qkv.Empty() && vt::IsBlockQuant(w.in_proj_qkv.dtype)) { + VT_CHECK(!w.in_proj_z.Empty() && vt::IsBlockQuant(w.in_proj_z.dtype), + "qwen3_5 GDN: a block-encoded in_proj_qkv must pair with a " + "block-encoded in_proj_z"); + out.mixed_owner.emplace(MatmulBTRawD( + d, h, ResidentWeight(d, w.in_proj_qkv), DType::kBF16)); + out.z_owner.emplace( + MatmulBTRawD(d, h, ResidentWeight(d, w.in_proj_z), DType::kBF16)); + out.mixed = out.mixed_owner->t(); + out.z = out.z_owner->t(); + return out; + } out.mixed_owner.emplace( !w.in_proj_qkv_fp8.Empty() ? (h_fp8 ? MatmulFp8CutlassPreQuantD(d, *h_fp8, w.in_proj_qkv_fp8, @@ -8469,7 +8488,10 @@ static void EmbedInto(Dev d, DBuf& hidden, const std::vector& token_ids const Qwen3_5MoeWeights& weights, const HfConfig& config) { const int64_t T = static_cast(token_ids.size()); const int64_t H = config.hidden_size; - const int64_t vocab = config.vocab_size; + if (std::getenv("VT_TT_ALLOC_TRACE") != nullptr) + std::fprintf(stderr, "[TT-FWD] EmbedInto T=%lld H=%lld\n", + static_cast(T), static_cast(H)); +const int64_t vocab = config.vocab_size; Tensor dtab = Qwen3_5EmbeddingTable(d.b, d.q, weights.embed_tokens, vocab, H); // ENG-ASYNC-SCHED W4: when the async runner has already placed this step's @@ -8744,6 +8766,12 @@ static DBuf ForwardBody(Dev d, const std::vector& token_ids, const Tensor* aux_out = nullptr) { const int64_t T = static_cast(token_ids.size()); const int64_t H = config.hidden_size; + if (std::getenv("VT_TT_ALLOC_TRACE") != nullptr) + std::fprintf(stderr, "[TT-FWD] ForwardBody T=%lld H=%lld num_reqs=%lld " + "logits_indices=%zu\n", + static_cast(T), static_cast(H), + static_cast(attn_meta.num_reqs), + logits_indices.size()); DBuf hidden(d, ActDType(d), {T, H}); EmbedInto(d, hidden, token_ids, weights, config); return ForwardLayers(d, hidden.t(), positions, attn_meta, gdn_meta, attn_kv, @@ -8963,7 +8991,14 @@ void Qwen3_5DenseModel::PrepareBf16Resident( "PrepareBf16Resident: a weight-staging (device-resident) queue required"); Dev d{vt::GetBackend(queue.device.type), queue}; const auto raw = [&d](const OwnedTensor& tensor) { - if (!tensor.Empty()) (void)ResidentWeight(d, tensor); + if (!tensor.Empty()) { + Tensor t = ResidentWeight(d, tensor); + // W4d W6: stage the keep-quant word shadow HERE, at load, while the + // banks are unfragmented — a lazy first-matmul stage at 27B asked the + // head's 1 GiB words against a 100 MB largest free block and died. + // No-op for non-block dtypes; the warm's later call hits the cache. + vt::tenstorrent::StageKeepQuantWordsFor(t); + } }; const auto f32 = [&d](const OwnedTensor& tensor) { if (!tensor.Empty()) { @@ -9550,6 +9585,9 @@ static DBuf DenseForwardLayers(Dev d, const Tensor& hidden_in, StepDevInputs* persistent_sdi = nullptr) { const int64_t T = hidden_in.shape[0]; const int64_t H = config.hidden_size; + if (std::getenv("VT_TT_ALLOC_TRACE") != nullptr) + std::fprintf(stderr, "[TT-FWD] DenseForwardLayers T=%lld n_idx=%zu\n", + static_cast(T), logits_indices.size()); const float eps = static_cast(config.rms_norm_eps); // Working copy of the embedded hidden (device->device; captured). RunDenseLayer @@ -9651,6 +9689,8 @@ static DBuf DenseForwardLayers(Dev d, const Tensor& hidden_in, static_cast(l)); vt::tenstorrent::AllocTraceSnapshot(vt::tenstorrent::SharedMeshDevice(), tt_lbl); + if (std::getenv("VT_TT_ALLOC_TRACE") != nullptr) // W4d: census spam only on trace + vt::tenstorrent::DumpSlotCensus(tt_lbl); } #endif // DFlash DF-AUX-TAPS: capture (hidden+res) at configured boundaries. Inert @@ -9711,6 +9751,27 @@ static DBuf DenseForwardLayers(Dev d, const Tensor& hidden_in, return DenseLogitsF32D(d, dnorm.t(), weights); } +// W4d (tenstorrent-27b-decode-shape-capture.md): per-sequence LAST-TOKEN +// logits indices for a mixed prefill step, the vLLM logits_processor +// semantics — generation samples only the last scheduled token per sequence +// (prepare_inputs.cpp:210-221: `logits_indices[i] = query_start_loc[i+1]-1`). +// Empty (identity, full [T,vocab] head) when the metadata cannot express the +// gather or it would be a no-op: a pure-decode step has size == T, and the +// graph-replay/capture contract passes {} by design. The driver's eager +// fallback feeds these to the forward so the gather-before-lm_head fires and +// the head allocates [num_seqs, vocab], never [T, vocab]. +static std::vector LastTokenLogitsIndices( + const v1::CommonAttentionMetadata& am, int64_t T) { + const int64_t n = + static_cast(am.query_start_loc.size()) - 1; + if (n <= 0 || n >= T) return {}; + std::vector indices(static_cast(n)); + for (int64_t i = 0; i < n; ++i) { + indices[static_cast(i)] = am.query_start_loc[static_cast(i) + 1] - 1; + } + return indices; +} + // Full eager dense paged forward body: embed (host token_ids) then the capturable // dense layer region. Used by Qwen3_5DenseModel::Forward/ForwardDevice and the // dense-graph driver's eager fallback / cold-shape pre-warm step (one contiguous @@ -9732,6 +9793,12 @@ static DBuf DenseForwardBody(Dev d, const std::vector& token_ids, gdn_state, weights, config); const int64_t T = static_cast(token_ids.size()); const int64_t H = config.hidden_size; + if (std::getenv("VT_TT_ALLOC_TRACE") != nullptr) + std::fprintf(stderr, "[TT-FWD] DenseForwardBody T=%lld H=%lld num_reqs=%lld " + "logits_indices=%zu\n", + static_cast(T), static_cast(H), + static_cast(attn_meta.num_reqs), + logits_indices.size()); DBuf hidden(d, ActDType(d), {T, H}); DenseEmbedInto(d, hidden, token_ids, weights, config); return DenseForwardLayers(d, hidden.t(), positions, attn_meta, gdn_meta, attn_kv, @@ -11088,8 +11155,14 @@ ForwardLogits Qwen3_5DecodeGraph::Step( token_ids, positions, attn_meta, gdn_meta, attn_kv, gdn_state, impl_->weights, impl_->config, impl_->queue, aux_out, {}); } + // W4d (tenstorrent-27b-decode-shape-capture.md): a MIXED step the graph + // declines (it captures only the decode shape, Q == 1) runs eagerly here -- + // pass the per-sequence last-token indices so the head gathers + // [num_reqs, vocab] instead of asking for the [T, vocab] plane the 27B + // banks cannot hold. DBuf lg = ForwardBody(d, token_ids, positions, attn_meta, gdn_meta, attn_kv, - gdn_state, impl_->weights, impl_->config); + gdn_state, impl_->weights, impl_->config, + LastTokenLogitsIndices(attn_meta, B)); // ForwardBody returns [B,vocab] (owned pool block; hand ownership out). return WrapDeviceLogits(d, std::move(lg), vocab); } @@ -11684,6 +11757,10 @@ ForwardLogits Qwen3_5DenseDecodeGraph::Step( // paths return a NON-owning view over the slot's persistent [S,vocab] logits // (first B rows are the real requests). Stream ordering guarantees the sampler // sees the replay's writes; the next same-size replay overwrites the buffer. + if (std::getenv("VT_TT_ALLOC_TRACE") != nullptr) + std::fprintf(stderr, "[TT-FWD] DenseGraph::Step B=%lld num_reqs=%lld\n", + static_cast(token_ids.size()), + static_cast(attn_meta.num_reqs)); // SPEC-DSPARK W8 (#442): a uniform SPEC batch carries B = num_reqs * (1+k) // tokens, which exceeds max_num_reqs and would make PadToCaptureSize return -1 // (eager). Capture its EXACT shape instead of padding: upstream pads only in @@ -11721,8 +11798,9 @@ ForwardLogits Qwen3_5DenseDecodeGraph::Step( token_ids, positions, attn_meta, gdn_meta, attn_kv, gdn_state, impl_->weights, impl_->config, impl_->queue, aux_out, {}); } - DBuf lg = DenseForwardBody(d, token_ids, positions, attn_meta, gdn_meta, attn_kv, - gdn_state, impl_->weights, impl_->config, {}); + DBuf lg = DenseForwardBody(d, token_ids, positions, attn_meta, gdn_meta, + attn_kv, gdn_state, impl_->weights, impl_->config, + LastTokenLogitsIndices(attn_meta, B)); return WrapDeviceLogits(d, std::move(lg), vocab); } diff --git a/src/vllm/model_executor/models/qwen3_5_dense.cpp b/src/vllm/model_executor/models/qwen3_5_dense.cpp index 7163d71d7..1d1bff8b9 100644 --- a/src/vllm/model_executor/models/qwen3_5_dense.cpp +++ b/src/vllm/model_executor/models/qwen3_5_dense.cpp @@ -238,6 +238,15 @@ ForwardLogits ForwardQwen3_5Dense(LoadedModel& model, (input.logits_indices.empty() || static_cast(input.logits_indices.size()) == input.attn_meta.num_actual_tokens)); + if (std::getenv("VT_TT_ALLOC_TRACE") != nullptr) + std::fprintf(stderr, + "[TT-FWD] route T=%zu num_reqs=%d pure_decode=%d " + "uniform_qlen=%lld gather=%d n_idx=%zu -> %s\n", + input.token_ids.size(), input.attn_meta.num_reqs, + input.pure_decode ? 1 : 0, + static_cast(input.uniform_query_len), + input.gather_logits ? 1 : 0, input.logits_indices.size(), + uniform_decode ? "graph" : "eager"); if (DenseDecodeGraphEnabled() && uniform_decode && graph_cuda && input.num_reqs <= kMaxDecodeGraphBatch) { if (!qwen.decode_graph()) { diff --git a/src/vllm/model_executor/models/qwen3_5_gguf_weights.cpp b/src/vllm/model_executor/models/qwen3_5_gguf_weights.cpp index de2f4a293..38bf54787 100644 --- a/src/vllm/model_executor/models/qwen3_5_gguf_weights.cpp +++ b/src/vllm/model_executor/models/qwen3_5_gguf_weights.cpp @@ -25,8 +25,26 @@ #include "vllm/model_executor/model_loader/gguf_keep_quant.h" #include "vt/dtype.h" #include "vt/quant.h" +#if defined(VLLM_CPP_TENSTORRENT) +#include "vt/tenstorrent/tenstorrent_device.h" +#endif namespace vllm { +// W4d W0 (#3042) attribution: DRAM bank snapshots in the host load path. +#if defined(VLLM_CPP_TENSTORRENT) +static void TtAllocTrace(const char* label) { + vt::tenstorrent::AllocTraceSnapshot(vt::tenstorrent::SharedMeshDevice(), label); +} +static void TtAllocTraceLayer(const char* what, int64_t il) { + char lbl[64]; + std::snprintf(lbl, sizeof(lbl), "load/%s/%lld", what, (long long)il); + TtAllocTrace(lbl); +} +#else +static inline void TtAllocTrace(const char*) {} +static inline void TtAllocTraceLayer(const char*, int64_t) {} +#endif + // Load-time PREFAULT of a mmap-borrowed weight span (VT_GGUF_PREFAULT or // `--offload-config`'s `vllm_cpp.mmap.prefault`, default ON with mmap residency). A weight left BORROWED in the read-only mapping is not @@ -391,6 +409,8 @@ void ReorderVRows(std::vector& buf, int64_t cols, int64_t row_off, std::memcpy(base, seg.data(), seg.size() * sizeof(T)); } + + // Reorder the full column range [0, cols) of a [rows, cols] row-major buffer // (cols = num_v * head_cols) from GGUF tiled to HF grouped order (out_proj). template @@ -1105,11 +1125,36 @@ GdnLayerWeights LoadGdnGguf(const GgufFile& g, int64_t il, const HfConfig& c, // in_proj_qkv <- attn_qkv [conv_dim, H]; only the trailing V rows reorder. { const std::string nm = Blk(il, "attn_qkv.weight"); - const GgufResidency r = pol.Route(g.Get(nm), proj_role); + // W4d W4 (tenstorrent-27b-gdn-keepquant.md): with the V-head reorder + // active the permutation acts on whole weight rows, so the PACKED + // blocks survive a byte-level row reorder and the projection stays + // keep-quant (kMatmulWeight) instead of expanding to bf16 — which + // cost the P150 ~9.7 GiB of attn_qkv staging at 27B + // (ISSUE-LOCAL-01M2AA4ZVD9EWJG5NNQD5DWZXS). + vt::DType kq_dt = vt::DType::kF32; + const bool packed_reorder = + reorder && pol.keep_quant && + KeepQuantDType(g.Get(nm).ggml_type, &kq_dt) && + g.Get(nm).shape[1] % vt::BlockElems(kq_dt) == 0 && + DeviceKeepQuantSupported(kq_dt, pol.device); + const GgufResidency r = pol.Route( + g.Get(nm), + packed_reorder ? GgufTensorRole::kMatmulWeight : proj_role); if (r != GgufResidency::kExpandBf16) { const GgufTensorInfo& ti = g.Get(nm); gdn.in_proj_qkv = OwnGgufKeptSlice(g, pol, ti, r, ti.shape[0], ti.shape[1], 0); + if (packed_reorder) { + // The kept bytes may be a BORROWED mmap view — the reorder + // materializes them into an owned buffer before permuting. + std::vector tmp(gdn.in_proj_qkv.bytes.data(), + gdn.in_proj_qkv.bytes.data() + + gdn.in_proj_qkv.bytes.size()); + ReorderVRows(tmp, + tmp.size() / ti.shape[0], + /*row_off=*/2 * key_dim, num_k, rpk, dv); + gdn.in_proj_qkv.bytes = OwnedBytes(std::move(tmp)); + } } else { const GgufTensorInfo* t = nullptr; std::vector dq = DqBf16(g, nm, &t); @@ -1122,11 +1167,28 @@ GdnLayerWeights LoadGdnGguf(const GgufFile& g, int64_t il, const HfConfig& c, // in_proj_z <- attn_gate [value_dim, H]; all rows are V. { const std::string nm = Blk(il, "attn_gate.weight"); - const GgufResidency r = pol.Route(g.Get(nm), proj_role); + vt::DType z_dt = vt::DType::kF32; + const bool packed_reorder_z = + reorder && pol.keep_quant && + KeepQuantDType(g.Get(nm).ggml_type, &z_dt) && + g.Get(nm).shape[1] % vt::BlockElems(z_dt) == 0 && + DeviceKeepQuantSupported(z_dt, pol.device); + const GgufResidency r = pol.Route( + g.Get(nm), + packed_reorder_z ? GgufTensorRole::kMatmulWeight : proj_role); if (r != GgufResidency::kExpandBf16) { const GgufTensorInfo& ti = g.Get(nm); gdn.in_proj_z = OwnGgufKeptSlice(g, pol, ti, r, ti.shape[0], ti.shape[1], 0); + if (packed_reorder_z) { + std::vector tmp(gdn.in_proj_z.bytes.data(), + gdn.in_proj_z.bytes.data() + + gdn.in_proj_z.bytes.size()); + ReorderVRows(tmp, + tmp.size() / ti.shape[0], + /*row_off=*/0, num_k, rpk, dv); + gdn.in_proj_z.bytes = OwnedBytes(std::move(tmp)); + } } else { const GgufTensorInfo* t = nullptr; std::vector dq = DqBf16(g, nm, &t); @@ -1403,6 +1465,7 @@ Qwen3_5MoeWeights LoadQwen3_5MoeFromGguf(const GgufFile& gguf, w.layers.reserve(static_cast(config.num_hidden_layers)); for (int64_t il = 0; il < config.num_hidden_layers; ++il) { + TtAllocTraceLayer("layer-moe", il); Qwen3_5MoeLayerWeights layer; RequireExpand(pol, gguf, Blk(il, "attn_norm.weight"), GgufTensorRole::kTransformedWeight); @@ -1557,6 +1620,7 @@ Qwen3_5DenseWeights LoadQwen3_5DenseFromGguf(const GgufFile& gguf, "qwen3_5 gguf: num_experts must be 0 for the dense model"); Qwen3_5DenseWeights w; + TtAllocTrace("load/embed-and-head/pre"); // Tied-embedding GGUFs (the 2B bench file) omit output.weight; the head is // then token_embd itself, as llama.cpp has it (TENSOR_DUPLICATED), and L5 // lets the two SHARE one expansion. See LoadEmbedAndHead. @@ -1564,9 +1628,11 @@ Qwen3_5DenseWeights LoadQwen3_5DenseFromGguf(const GgufFile& gguf, RequireExpand(pol, gguf, "output_norm.weight", GgufTensorRole::kTransformedWeight); w.final_norm = OwnNormMinus1(gguf, "output_norm.weight"); + TtAllocTrace("load/embed-and-head/post"); w.layers.reserve(static_cast(config.num_hidden_layers)); for (int64_t il = 0; il < config.num_hidden_layers; ++il) { + TtAllocTraceLayer("layer-dense", il); Qwen3_5DenseLayerWeights layer; RequireExpand(pol, gguf, Blk(il, "attn_norm.weight"), GgufTensorRole::kTransformedWeight); @@ -1839,4 +1905,12 @@ void LogQwen3_5GgufMtpHeadSkip(const GgufFile& gguf, const HfConfig& config) { << std::endl; } + +void ReorderVPackedForTest(std::vector& packed, int64_t row_bytes, + int64_t row_off, int64_t num_k, + int64_t num_v_per_k, int64_t head_rows) { + ReorderVRows(packed, row_bytes, row_off, num_k, num_v_per_k, + head_rows); +} + } // namespace vllm diff --git a/src/vt/tenstorrent/tenstorrent_device.h b/src/vt/tenstorrent/tenstorrent_device.h index d0f67282f..7e8af3632 100644 --- a/src/vt/tenstorrent/tenstorrent_device.h +++ b/src/vt/tenstorrent/tenstorrent_device.h @@ -221,6 +221,19 @@ int64_t FreeDeviceDramBytesForTest(); // Total DRAM across banks (capacity, not free) — the platform's probed // total for the placement fit (ISSUE-LOCAL-01M2ACXRJYFW7R7BP2ABQS3VY2). int64_t DeviceDramTotalBytes(); +// W4d W3 focused-test hook: stage `t`'s bf16 TILE form exactly as +// EnsureDevice2D does (creating the slot's persistent buffer), reproducing +// the loader's weight staging before a keep-quant matmul. +void StageWeightBf16ForTest(const Tensor& t, MeshDevice& device); +// W4d W6: stage the keep-quant word shadow for a block-quant weight at +// LOAD time (the residency pre-pass; no-op for non-block dtypes). +void StageKeepQuantWordsFor(const Tensor& packed); +// Attribution (W4d W3): dump resident slot bytes by holder (device shadow / +// persistent staged buffer / gemma) + top holders. Gated by the caller. +void DumpSlotCensus(const char* label); +// W4d W3: release consumer shadows whose rows match the warm forward's +// shape (recipe-gated via VT_TT_RELEASE_WARM_ROWS; see the ops-side comment). +void ReleaseWarmShapeSlots(uint32_t rows); #else inline int64_t KeepQuantCaptureStagingWrites() { return 0; } inline void ResetKeepQuantCaptureStagingWritesForTest() {} @@ -235,6 +248,10 @@ inline int64_t AllocTraceMaxDeltaForTest() { return 0; } inline void ResetAllocTraceForTest() {} inline int64_t FreeDeviceDramBytesForTest() { return 0; } inline int64_t DeviceDramTotalBytes() { return 0; } +inline void StageWeightBf16ForTest(const Tensor&, MeshDevice&) {} +inline void StageKeepQuantWordsFor(const Tensor&) {} +inline void DumpSlotCensus(const char*) {} +inline void ReleaseWarmShapeSlots(uint32_t) {} #endif // ITEM 5 (rope): driver-side warm hook — populate the persistent device diff --git a/src/vt/tenstorrent/tenstorrent_ops.cpp b/src/vt/tenstorrent/tenstorrent_ops.cpp index f01478a95..d977e6338 100644 --- a/src/vt/tenstorrent/tenstorrent_ops.cpp +++ b/src/vt/tenstorrent/tenstorrent_ops.cpp @@ -929,6 +929,10 @@ ttnn::Tensor EnsureDevice2D(const Tensor& t, MeshDevice& device) { return dev; } +// (StageWeightBf16ForTest is defined at vt::tenstorrent scope below — it +// needs external linkage for the focused test and calls EnsureDevice2D, +// which stays TU-internal here.) + // DEBUG (BACKEND-TENSTORRENT-QWEN35 W2c): ensure the tensor is staged on // device exactly as a consuming kernel would, then read the DEVICE copy back. // Comparing this against the host master exposes staging corruption that a @@ -1862,6 +1866,7 @@ void DropKeepQuantWordShadow(void* host) { // immutable post-load, so a serve can never go stale while the master lives // (same assumption as the BufferSlot/WeightViewShadow resident shadows); the // UnregisterHostBuffer drop above covers the recycle case. +void TTReclaimPlanes(MeshDevice& device, std::vector& planes); ttnn::Tensor EnsureKeepQuantWords(const Tensor& packed, DType enc, int64_t rows, int64_t nb, MeshDevice& device) { VT_CHECK(packed.rank == 2 && packed.IsContiguous(), @@ -1916,10 +1921,38 @@ ttnn::Tensor EnsureKeepQuantWords(const Tensor& packed, DType enc, int64_t rows, static_cast(wpb)}), ttnn::DataType::INT32, ttnn::Layout::ROW_MAJOR), &device); - std::lock_guard g(KeepQuantWordMutex()); - KeepQuantWordShadows()[packed.data] = - KeepQuantWordShadow{staged, rows, nb, wpb}; + { + std::lock_guard g(KeepQuantWordMutex()); + KeepQuantWordShadows()[packed.data] = + KeepQuantWordShadow{staged, rows, nb, wpb}; + } AllocTraceSnapshot(device, "EnsureKeepQuantWords/post"); + // W4d W3 (#3042): the weight's bf16 TILE staging is dead weight the moment + // its word shadow exists — the keep-quant matmul reads only the words, and + // a bf16-arm decline re-stages on demand from the slot's live host bytes. + // At 27B the staged bf16 form of every k-quant weight sat beside the words + // (attn_qkv alone: 97 x [10240,5120] bf16 = 9.7 GiB beside ~21 MiB q6_K + // each), filled the banks to 93 percent during the warm pass, and + // fragmented them into the init OOM. Release the slot's bf16 forms here; + // the word shadows, embed twin and persistent non-quant stagings stay. + // Scoped AFTER the KeepQuantWordMutex guard so the SlotMutex nesting order + // (Slot -> KeepQuant elsewhere, sequential here) is never inverted. + if (!tt_capture_active()) { + std::vector dead; + { + std::lock_guard g_slot(SlotMutex()); + if (BufferSlot* s = FindSlot(packed.data); + s != nullptr && packed.data == s->host && + (s->device.has_value() || s->persistent.has_value())) { + if (s->device) dead.push_back(*s->device); + if (s->persistent) dead.push_back(*s->persistent); + s->device.reset(); + s->persistent.reset(); + s->device_current = false; + } + } + if (!dead.empty()) TTReclaimPlanes(device, dead); + } return staged; } @@ -1957,6 +1990,7 @@ void TTReclaimPlanes(MeshDevice& device, std::vector& planes) { ttnn::deallocate(plane, /*force=*/true); } + // Stream bytes [first, last) of the word-staged block as u8 {B, last-first} // (little-endian lanes). concat stacks the four lane tensors, so the // (lane, word)->(word, lane) permute restores stream order — the same trick @@ -2071,6 +2105,12 @@ ttnn::Tensor DecodeKeepQuantWordsF32(const ttnn::Tensor& w, DType enc, int64_t slice_rows, int64_t nb, MeshDevice& device) { const uint32_t B = static_cast(slice_rows * nb); + { // W4d W0 (#3042) attribution: label carries the slice size. + char lbl[64]; + std::snprintf(lbl, sizeof(lbl), "kq-decode/rows=%lld/nb=%lld", + (long long)slice_rows, (long long)nb); + AllocTraceSnapshot(device, lbl); + } // W4d W2 (#3042): every plane this decode builds is reclaimed by force at // its last use — TTReclaimPlanes above (the W1 trace booked 7.6 GB of @@ -2181,6 +2221,7 @@ ttnn::Tensor DecodeKeepQuantWordsF32(const ttnn::Tensor& w, DType enc, // cached -0 constant — never reclaimed. auto repair = [&](ttnn::Tensor value, ttnn::Tensor mask_f32, const ttnn::Tensor& neg0) { + AllocTraceSnapshot(device, "kq-decode/repair"); ttnn::Tensor pred = ttnn::to_layout(ttnn::gt(mask_f32, 0.0f), ttnn::Layout::TILE); ttnn::Tensor value_t = ttnn::to_layout(value, ttnn::Layout::TILE); @@ -2620,6 +2661,12 @@ void MatmulBTQuantKernel(Queue& q, Tensor& out, const Tensor& a, const Tensor& b std::string("tenstorrent kMatmulBTQuant: K must be a whole number " "of ") + Name(enc) + " blocks (" + std::to_string(elems) + " elems)"); + { // W4d W0 (#3042) attribution: which matmul weight is being staged. + char lbl[64]; + std::snprintf(lbl, sizeof(lbl), "matmulbtq/N=%lld/K=%lld", + (long long)b.shape[0], (long long)b.shape[1]); + AllocTraceSnapshot(SharedMeshDevice(), lbl); + } VT_CHECK(IsFloatDType(a.dtype) && (out.dtype == DType::kF32 || out.dtype == DType::kBF16), "tenstorrent kMatmulBTQuant: float activation, f32/bf16 out"); @@ -2655,13 +2702,16 @@ void MatmulBTQuantKernel(Queue& q, Tensor& out, const Tensor& a, const Tensor& b // ROW_MAJOR chained leg's NaN signature: the vehicle's mid-layer bf16 // keep-quant matmul fed a down-projection whose activation read the // poisoned slot). - if (out.dtype == DType::kF32) { - if (const char* int8dot_env = std::getenv("VT_TT_KEEPQUANT_INT8DOT"); - int8dot_env != nullptr && int8dot_env[0] != '\0' && - std::strcmp(int8dot_env, "0") != 0) { - MatmulBTQuantInt8DotKernel(q, out, a, b); - return; - } + // W4d W6: int8-dot serves BOTH out dtypes when opted in — one captured + // launch per matmul replaces the per-chunk E=1 chain, whose capture-time + // transients pinned 3.8 GiB into the trace region at 27B + // (tenstorrent-27b-int8dot-capture.md). bf16-out carries an explicit + // f32->bf16 cast before the commit (the W4b store-geometry bug, fixed). + if (const char* int8dot_env = std::getenv("VT_TT_KEEPQUANT_INT8DOT"); + int8dot_env != nullptr && int8dot_env[0] != '\0' && + std::strcmp(int8dot_env, "0") != 0) { + MatmulBTQuantInt8DotKernel(q, out, a, b); + return; } // W4a wave-3b-1 (#3030): the bf16-out dispatch. P = M output rows against // expert 0; the ids are statically all zero and the E=1 arm never reads @@ -2788,6 +2838,12 @@ void MatmulBTQuantGroupedKernel(Queue&, Tensor& out, const Tensor& act, if (P == 0 || N == 0) return; MeshDevice& device = SharedMeshDevice(); + { // W4d W0 (#3042) attribution: grouped arm shape. + char lbl[80]; + std::snprintf(lbl, sizeof(lbl), "matmulbtq-grouped/E=%lld/N=%lld/K=%lld", + (long long)E, (long long)N, (long long)K); + AllocTraceSnapshot(device, lbl); + } // Stage the PACKED tower once — the resident i32 word shadow keyed by the // host weight pointer, served forever after (the dense arm's pattern). The @@ -3284,6 +3340,7 @@ std::map& Int8DotWorkloadCache() { } } // namespace + // kMatmulBTQuant's W4b body: out[M,N] = a[M,K] @ b[N,K]^T with b PACKED // keep-quant blocks, computed entirely by the device kernel above. M, N, K, // nb, wpb and the per-core row split are launch arguments; the activation is @@ -3584,6 +3641,15 @@ void MatmulBTQuantInt8DotKernel(Queue& q, Tensor& out, const Tensor& a, // slot the next consumer's tile view can overflow, the wave-3b lesson). ttnn::Tensor committed = ttnn::to_layout(std::move(dev_out), ttnn::Layout::TILE); + // W4d W6: a BF16-OUT call takes an explicit f32->bf16 cast BEFORE the + // commit. Committing the f32 dev_out into a bf16 slot left the slot + // holding f32 bytes at an f32 page geometry — the next bf16 reader got + // word-halved garbage (the ROW_MAJOR chained leg's NaN signature, the + // W4b landing decision). One recorded typecast launch: capture-safe. + if (out.dtype == DType::kBF16) + committed = ttnn::typecast(std::move(committed), + ttnn::DataType::BFLOAT16); + CommitDeviceLogical2D(out, std::move(committed), static_cast(M), static_cast(N)); } @@ -8725,6 +8791,58 @@ int64_t AllocTraceMaxDeltaForTest() { std::lock_guard g(AllocTraceSt().mtx); return AllocTraceSt().max_alloc_delta; } +void StageWeightBf16ForTest(const Tensor& t, MeshDevice& device) { + (void)EnsureDevice2D(t, device); +} + +// W4d W3 (ISSUE-LOCAL-01M2AA4ZVD9EWJG5NNQD5DWZXS): after the ctor's cold +// pre-warm (which runs at the full batched shape) and the capture, every +// warm activation it committed via CommitDeviceLogical2D is garbage — the +// captured decode only reads step-shaped buffers — yet the slots hold it +// forever (~14 GiB at 27B: banks at 93 percent before the first decode, and +// the repair chain's contiguity ask fragments against it). Release slots +// whose consumer shadow has EXACTLY the warm forward's row count; word +// shadows, persistent weight stagings and the embed twin live in other +// holders and are untouched. Recipe-gated via VT_TT_RELEASE_WARM_ROWS. +void ReleaseWarmShapeSlots(uint32_t rows) { + std::vector dead; + { + std::lock_guard g(SlotMutex()); + for (auto& kv : Slots()) { + BufferSlot& s = kv.second; + if (!s.device) continue; + const auto ls = s.device->logical_shape(); + if (ls.rank() >= 1 && ls[0] == rows) { + dead.push_back(*s.device); + s.device.reset(); + } + } + } + if (!dead.empty()) { + std::fprintf(stderr, + "[TT-WARM-RELEASE] rows=%u: released %zu warm slot(s)\n", + rows, dead.size()); + TTReclaimPlanes(SharedMeshDevice(), dead); + } +} + +// W4d W6: stage the keep-quant word shadow for ONE block-quant weight at +// LOAD time (the residency pre-pass). A lazy first-matmul stage at 27B +// asked the head's 1 GiB words against a fragmented, 95-percent-full bank +// set and died; staged here — immediately after the weight's own resident +// upload, while the banks are unfragmented — the words land contiguously +// and the warm's EnsureKeepQuantWords hits the cache (same key: the +// ResidentWeight device view's pointer). No-op for non-block dtypes. +void StageKeepQuantWordsFor(const Tensor& packed) { + if (!vt::IsBlockQuant(packed.dtype)) return; + const int64_t nb = + static_cast(packed.shape[1]) / + static_cast(BlockElems(packed.dtype)); + (void)EnsureKeepQuantWords(packed, packed.dtype, + static_cast(packed.shape[0]), nb, + SharedMeshDevice()); +} + int64_t FreeDeviceDramBytesForTest() { MeshDevice& device = SharedMeshDevice(); const auto view = tt::tt_metal::detail::GetMemoryView( @@ -8744,6 +8862,62 @@ int64_t DeviceDramTotalBytes() { return static_cast(view.num_banks) * static_cast(view.total_bytes_per_bank); } + +// Attribution (W4d W3, ISSUE-LOCAL-01M2AA4ZVD9EWJG5NNQD5DWZXS): walk the +// slot table and dump resident device bytes by holder — the consumer shadow +// (`device`), the persistent staged buffer (`persistent`), the gemma affine +// form — plus the top holders by volume. The 27B warm pass leaves ~14 GiB +// of slot residency; this census names which slots hold it. +void DumpSlotCensus(const char* label) { + std::lock_guard g(SlotMutex()); + auto bytes_of = [](const ttnn::Tensor& t) -> size_t { + size_t esz = 4; + switch (t.dtype()) { + case ttnn::DataType::BFLOAT16: esz = 2; break; + case ttnn::DataType::UINT8: esz = 1; break; + default: esz = 4; break; + } + size_t n = 1; + for (uint32_t d : t.logical_shape()) n *= d; + return n * esz; + }; + size_t dev_bytes = 0, pers_bytes = 0, gemma_bytes = 0; + size_t n_dev = 0, n_pers = 0; + std::map> dev_hist; // shape -> bytes, slots + std::map> pers_hist; + for (auto& kv : Slots()) { + BufferSlot& s = kv.second; + if (s.device) { + const size_t b = bytes_of(*s.device); + dev_bytes += b; + ++n_dev; + auto& e = dev_hist[std::to_string(s.dev_rows) + "x" + + std::to_string(s.dev_cols)]; + e.first += b; + e.second += 1; + } + if (s.persistent) { + const size_t b = bytes_of(*s.persistent); + pers_bytes += b; + ++n_pers; + auto& e = pers_hist[std::to_string(s.persist_rows) + "x" + + std::to_string(s.persist_cols)]; + e.first += b; + e.second += 1; + } + if (s.gemma_device) gemma_bytes += bytes_of(*s.gemma_device); + } + std::fprintf(stderr, + "[TT-SLOT-CENSUS] %s: dev=%zu B in %zu slots, " + "pers=%zu B in %zu slots, gemma=%zu B\n", + label, dev_bytes, n_dev, pers_bytes, n_pers, gemma_bytes); + for (auto& [shape, e] : dev_hist) + std::fprintf(stderr, "[TT-SLOT-CENSUS] dev %s: %zu B in %zu slots\n", + shape.c_str(), e.first, e.second); + for (auto& [shape, e] : pers_hist) + std::fprintf(stderr, "[TT-SLOT-CENSUS] pers %s: %zu B in %zu slots\n", + shape.c_str(), e.first, e.second); +} void ResetAllocTraceForTest() { std::lock_guard g(AllocTraceSt().mtx); AllocTraceSt().snapshot_count = 0; diff --git a/tests/parity/goldens/qwen38_gguf_q4km_27b/neartie_gap_mnats.npy b/tests/parity/goldens/qwen38_gguf_q4km_27b/neartie_gap_mnats.npy new file mode 100644 index 0000000000000000000000000000000000000000..a3b01710197f8b5ffce43588caec074bdfbfdc75 GIT binary patch literal 1152 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1ZlWC%^qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= zXCxM+0{I$-W;zOnW|}$*wF+bcE(R(C1uB_mOC@tgnlUy|56hD2aXvJCc1V0xB=#tC Hc!U4|q8=Q- literal 0 HcmV?d00001 diff --git a/tests/parity/goldens/qwen38_gguf_q4km_27b/neartie_gap_mnats_tenstorrent_capture.npy b/tests/parity/goldens/qwen38_gguf_q4km_27b/neartie_gap_mnats_tenstorrent_capture.npy new file mode 100644 index 0000000000000000000000000000000000000000..a3b01710197f8b5ffce43588caec074bdfbfdc75 GIT binary patch literal 1152 zcmbR27wQ`j$;eQ~P_3SlTAW;@Zl$1ZlWC%^qoAIaUsO_*m=~X4l#&V(cT3DEP6dh= zXCxM+0{I$-W;zOnW|}$*wF+bcE(R(C1uB_mOC@tgnlUy|56hD2aXvJCc1V0xB=#tC Hc!U4|q8=Q- literal 0 HcmV?d00001 diff --git a/tests/parity/goldens/qwen38_gguf_q4km_27b/our_ids.i32 b/tests/parity/goldens/qwen38_gguf_q4km_27b/our_ids.i32 new file mode 100644 index 0000000000000000000000000000000000000000..068b81de94fe415a6406e793e5f271ad99560f2d GIT binary patch literal 1024 zcmZvbO-R&H6vdCuSW{VMHR)u9l@>KwpyFa$$k?JH42u>c^ua(o3lUrtL6$+A7JU#1 z(JW>W*(QW&V=W>`gM``!LSKu5B8VU=koui@^KTG@x%1w;_nv#tJ@5TLwMok*O<}*P zrNNYxsgjPhN<*yM4=x;U&xH7HauV1?ztp@^y0}bwh4Itxn6zzIh}SO;b-yo>iqw3~ z+5pb>=c|-0?{9CJ-_PeZcs&B&SQF|x_a(k{^e}#Ck5rqSkE^Pa?siHKZb}`Oq_c2Y zDA#i@qDBGhz~W1#S$x2S6eV3Y;Ge-Fe}uWuy!Nq_@{Rd0%sk{4;Jq&qIIU;auGLb@ zDrv4a%wR5?o=Eu~a%Lp!Xx}Z0xMqF1=3#u9vL>mxl*|Sl+tFfjO~AaXt;~K0oqDc@ z9(^rR4vg<2O^)kAp>#e(U!QM04p`UrecR`GY!?8&IKsUWiE89zP>9!y~%@b09) z;dUi{46dCmhZNUa792skM1{K>A3{{p_!x`yguqE@RAN-zd6I K^1b}MF8u>+lyz(X literal 0 HcmV?d00001 diff --git a/tests/parity/goldens/qwen38_gguf_q4km_27b/our_ids.npy b/tests/parity/goldens/qwen38_gguf_q4km_27b/our_ids.npy new file mode 100644 index 0000000000000000000000000000000000000000..25d7fd711b00b6bd205ecff57bb3d5ac1cea599e GIT binary patch literal 1152 zcmbVMOK4L;6g_EDjkQ{<#x_>vsjbhNRB0Qsh=QboA4*CTg%s2hOhjolHVL8>tyXa3 z!Y>rWEV2;mMg(zFU5MZZEa+A!_**C_f(W8Q(Q`6+hC)|{+|10Kd+t5=%uKHK?1{yD zT{w;*JC)8I$l4Lh-gG!*2QB+xCY#SDkM?D~COdzOIbU&XAWruC3lJ^Oudkk7PwO`Fx=(y#waKg47xXP<4dDkj0=2&B zI9DBTGX&hZ0(71QPLRt?vFdvka};QuG*1mMNgv^$6eV4j(?3CT{0`3TU+V<26gJXU~0shfdnx+ivVMkmlR6t9@f?R@&kiaZk@M$>PL2(z~{So zuH5sKIV4zDb0>Cy{fw@o*2I2J0K**BN0AA1kD`6%Y%l6OxX$z3zRw4~a^B=NvFE6{ z?x4d#_!ND@kBtydOrhWK4za%Wh1o*?>jvXb(zk|uKXG*K^Dwm#Pu!O<uT> z0$0#i$GOE(`|tQJo^3W861SA0QoIB9tK!$wnS hb8iRmfjPHxexAB#IC{xtl>NvzLfyytUjAGH{sJ%nkpln# literal 0 HcmV?d00001 diff --git a/tests/parity/goldens/qwen38_gguf_q4km_27b/our_ids_tenstorrent_capture.i32 b/tests/parity/goldens/qwen38_gguf_q4km_27b/our_ids_tenstorrent_capture.i32 new file mode 100644 index 0000000000000000000000000000000000000000..06d7405020018ddf3cacee90fd4af10487da3d20 GIT binary patch literal 1024 ScmZQz7zLvtFd70QH3R?z00031 literal 0 HcmV?d00001 diff --git a/tests/parity/goldens/qwen38_gguf_q4km_27b/our_ids_tenstorrent_capture.npy b/tests/parity/goldens/qwen38_gguf_q4km_27b/our_ids_tenstorrent_capture.npy new file mode 100644 index 0000000000000000000000000000000000000000..25d7fd711b00b6bd205ecff57bb3d5ac1cea599e GIT binary patch literal 1152 zcmbVMOK4L;6g_EDjkQ{<#x_>vsjbhNRB0Qsh=QboA4*CTg%s2hOhjolHVL8>tyXa3 z!Y>rWEV2;mMg(zFU5MZZEa+A!_**C_f(W8Q(Q`6+hC)|{+|10Kd+t5=%uKHK?1{yD zT{w;*JC)8I$l4Lh-gG!*2QB+xCY#SDkM?D~COdzOIbU&XAWruC3lJ^Oudkk7PwO`Fx=(y#waKg47xXP<4dDkj0=2&B zI9DBTGX&hZ0(71QPLRt?vFdvka};QuG*1mMNgv^$6eV4j(?3CT{0`3TU+V<26gJXU~0shfdnx+ivVMkmlR6t9@f?R@&kiaZk@M$>PL2(z~{So zuH5sKIV4zDb0>Cy{fw@o*2I2J0K**BN0AA1kD`6%Y%l6OxX$z3zRw4~a^B=NvFE6{ z?x4d#_!ND@kBtydOrhWK4za%Wh1o*?>jvXb(zk|uKXG*K^Dwm#Pu!O<uT> z0$0#i$GOE(`|tQJo^3W861SA0QoIB9tK!$wnS hb8iRmfjPHxexAB#IC{xtl>NvzLfyytUjAGH{sJ%nkpln# literal 0 HcmV?d00001 diff --git a/tests/parity/test_qwen35_paged_engine.cpp b/tests/parity/test_qwen35_paged_engine.cpp index f5e774f65..5e35a5a86 100644 --- a/tests/parity/test_qwen35_paged_engine.cpp +++ b/tests/parity/test_qwen35_paged_engine.cpp @@ -176,6 +176,14 @@ void RunGate(const std::string& golden_subdir, const char* label, SkipGate(label, "model artifact not cached — resolve the snapshot or GGUF " "path this gate is pinned to first"); } + // W4d W3 (#3042): the 27B checkpoint's auto-fit (max_model_len 8192 -> a + // 256-block KV pool and a 13104-token capture shape) drove the 32 GiB card + // to 99 percent bank occupancy and OOM'd mid-generation; the battery needs + // ~40 tokens of context. Bound the engine to the workload the way vLLM's + // own smoke tests pass max_model_len. The 0.8B vehicle keeps the default. + vllm::entrypoints::EngineParams gate_params; + if (label == std::string_view("qwen38-gguf-q4km-27b")) + gate_params.max_model_len = 1024; const fs::path gdir = probe ? fs::path(probe_dir) : fs::path(PARITY_GOLDENS_DIR) / golden_subdir; const bool dump = !probe && std::getenv("VT_DUMP_IDS") != nullptr; @@ -195,8 +203,7 @@ void RunGate(const std::string& golden_subdir, const char* label, if (artifacts == GateArtifactState::kBootstrap) { MESSAGE(label << ": BOOTSTRAP dump (gap golden absent) via FromModelDir(" << snap << ")..."); std::unique_ptr le = - vllm::entrypoints::LoadedEngine::FromModelDir( - snap, vllm::entrypoints::EngineParams{}); + vllm::entrypoints::LoadedEngine::FromModelDir(snap, gate_params); const parity::NpyArray gg = parity::LoadNpy((gdir / "greedy_ids.npy").string()); const int64_t NN = gg.shape[0], TT = gg.shape[1]; std::vector buf(static_cast(NN * TT), -1); @@ -241,8 +248,7 @@ void RunGate(const std::string& golden_subdir, const char* label, MESSAGE(label << ": loading via FromModelDir(" << snap << ")..."); std::unique_ptr loaded = - vllm::entrypoints::LoadedEngine::FromModelDir( - snap, vllm::entrypoints::EngineParams{}); + vllm::entrypoints::LoadedEngine::FromModelDir(snap, gate_params); // The base golden pair is production-mode ROCm evidence after issue #2772. // The Tenstorrent device lane carries its OWN oracle-backed golden pair diff --git a/tests/vllm/test_gguf_keep_quant.cpp b/tests/vllm/test_gguf_keep_quant.cpp index 938b1a157..7837426c4 100644 --- a/tests/vllm/test_gguf_keep_quant.cpp +++ b/tests/vllm/test_gguf_keep_quant.cpp @@ -3182,3 +3182,82 @@ TEST_CASE("ROCm F16 production registered forward reaches retained embedding and vllm_test::UnsetEnv("VT_GGUF_KEEP_F16"); } #endif + +// W4d W4 (#3042, spec tenstorrent-27b-gdn-keepquant.md): the byte-level +// packed V-row reorder must be EXACTLY the element-level reorder seen +// through the row dequantizer — bit-for-bit, per row, for every encoding +// the GDN family carries. RED until ReorderVPackedForTest existed: the +// function did not, and the loader dequantized+reordered elements instead, +// which is what forced the 9.7 GiB of bf16 GDN projections onto the P150. +namespace { + +std::vector ReorderVRowsRef(const std::vector& in, + int64_t cols, int64_t row_off, + int64_t num_k, int64_t num_v_per_k, + int64_t head_rows) { + const int64_t cs = head_rows * cols; + std::vector out = in; + for (int64_t k = 0; k < num_k; ++k) { + for (int64_t r = 0; r < num_v_per_k; ++r) { + const int64_t g = k * num_v_per_k + r; + const int64_t t = r * num_k + k; + std::memcpy(out.data() + (row_off + g) * cs, + in.data() + (row_off + t) * cs, + static_cast(cs) * sizeof(float)); + } + } + return out; +} + +} // namespace + +TEST_CASE("packed V-row reorder equals the element-level reorder (W4d W4)") { + // Geometry: K=512 (2 blocks/row, whole blocks per row), 15 weight rows = + // row_off(3, non-V) + 6 heads x head_rows(2). q6_K row 420 B, q4_K 288 B. + struct Enc { + const char* name; + uint32_t ggml_type; + int64_t block_bytes; + }; + const Enc encs[] = {{"q6_K", 14, 210}, {"q4_K", 12, 144}}; + const int64_t K = 512, row_off = 3, num_k = 2, rpk = 3, head_rows = 2; + const int64_t rows = row_off + num_k * rpk * head_rows; // 15 + + std::mt19937 rng(20260912u); + for (const Enc& e : encs) { + const int64_t row_bytes = K / 256 * e.block_bytes; + std::vector packed(static_cast(rows * row_bytes)); + for (size_t b = 0; b < packed.size(); b += e.block_bytes) { + // pin the block's scale/min bits to valid f16 payloads; randomize the + // quant elements (a random f16 scale can be inf/NaN and NaN != NaN). + const uint16_t d = vt::F32ToF16(0.05f + 0.01f * static_cast(b % 97)); + const uint16_t s = vt::F32ToF16(0.004f + 0.001f * static_cast(b % 31)); + std::memcpy(packed.data() + b, &d, 2); + if (e.block_bytes > 2) std::memcpy(packed.data() + b + 2, &s, 2); + for (int64_t i = 4; i < e.block_bytes; ++i) + packed[b + i] = static_cast(rng() & 0xFF); + } + + // candidate: reorder the packed bytes, then dequant per row + std::vector pb = packed; + vllm::ReorderVPackedForTest(pb, row_bytes, row_off, num_k, rpk, head_rows); + std::vector fb(static_cast(rows * K)); + for (int64_t r = 0; r < rows; ++r) { + auto row = vllm::DequantGgufRowToF32( + e.ggml_type, pb.data() + r * row_bytes, K); + REQUIRE(row.size() == static_cast(K)); + std::memcpy(fb.data() + r * K, row.data(), static_cast(K) * 4); + } + + // reference: dequant per row, then the independent element reorder + std::vector fa(static_cast(rows * K)); + for (int64_t r = 0; r < rows; ++r) { + auto row = vllm::DequantGgufRowToF32( + e.ggml_type, packed.data() + r * row_bytes, K); + std::memcpy(fa.data() + r * K, row.data(), static_cast(K) * 4); + } + fa = ReorderVRowsRef(fa, K, row_off, num_k, rpk, head_rows); + + CHECK(fa == fb); + } +} diff --git a/tests/vt/test_tenstorrent_backend.cpp b/tests/vt/test_tenstorrent_backend.cpp index eb230b15f..bdfb75de1 100644 --- a/tests/vt/test_tenstorrent_backend.cpp +++ b/tests/vt/test_tenstorrent_backend.cpp @@ -6861,6 +6861,160 @@ TEST_CASE("kTENSTORRENT E=1 int8-dot keep-quant capture survives the 50 MiB trac backend.Free(mem_o); } +// W4d W6: the BF16-OUT dispatch joined the int8-dot lever. The W4b landing +// decision refused bf16-out because committing the kernel's f32 dev_out into +// a bf16 slot left the slot holding f32 bytes at an f32 page geometry — the +// next bf16 reader got word-halved garbage (the ROW_MAJOR chained leg's NaN +// signature). The fix is an explicit f32->bf16 cast before the commit; this +// test is its red-first lock (RED on the pre-cast tree: the replay output +// word-halves against the eager reference). +TEST_CASE("kTENSTORRENT E=1 int8-dot keep-quant capture survives the 50 MiB trace region (BF16-out dispatch)") { + // Same lever gate as the F32-out sibling: on default the bf16-out + // dispatch serves the W4a grouped arm and this test would prove nothing. + if (const char* lever = std::getenv("VT_TT_KEEPQUANT_INT8DOT"); + lever == nullptr || lever[0] == '\0' || std::strcmp(lever, "0") == 0) { + MESSAGE("SKIPPED: set VT_TT_KEEPQUANT_INT8DOT=1 — this capture asserts the " + "int8-dot lever through the BF16-out dispatch"); + return; + } + ::setenv("VT_TT_KEEPQUANT_INT8DOT", "1", 1); + if (!TenstorrentPresent()) { + MESSAGE("SKIPPED: no Tenstorrent device on this box"); + return; + } + Backend& backend = vt::GetBackend(vt::DeviceType::kTENSTORRENT); + REQUIRE(backend.SupportsGraphCapture()); + REQUIRE(vt::OpRegistered(vt::OpId::kMatmulBTQuant, vt::DeviceType::kTENSTORRENT)); + Queue q = backend.CreateQueue(); + + // Two chained keep-quant matmuls: m1 (bf16 out) feeds m2 (f32 out) as its + // ACTIVATION. The W4b store-geometry bug lived in exactly that handoff: + // the f32 dev_out committed into the bf16 slot, and the next DEVICE + // reader (m2's activation load) word-halved it. A host download converts + // and hides the poison, so the probe is the device-side consumer. + constexpr int64_t kN1 = 4096, kK = 1024, kN2 = 1024, kM = 1; + const int64_t kElems = vt::BlockElems(vt::DType::kQ6_K); + const int64_t kBB = vt::BlockBytes(vt::DType::kQ6_K); + const int64_t kNb1 = kK / kElems, kNb2 = kN1 / kElems; + + std::mt19937 rng(20260913u); + auto fill_q6k = [&](std::vector& packed, uint32_t seed) { + std::mt19937 r(seed); + packed.resize(static_cast(packed.size())); + for (size_t b = 0; b < packed.size() / static_cast(kBB); ++b) { + uint8_t* blk = packed.data() + b * kBB; + for (int i = 0; i < 208; ++i) blk[i] = static_cast(r() & 0xFF); + const uint16_t d_bits = + vt::F32ToF16(0.05f + 0.35f * static_cast(r() % 64) / 64.0f); + std::memcpy(blk + 208, &d_bits, sizeof(d_bits)); + } + }; + std::vector p1(static_cast(kN1) * kNb1 * kBB); + std::vector p2(static_cast(kN2) * kNb2 * kBB); + fill_q6k(p1, 20260913u); + fill_q6k(p2, 20260914u); + std::vector a_bf(static_cast(kM * kK)); + for (auto& v : a_bf) v = vt::F32ToBF16((static_cast(rng() % 401) - 200.0f) / 100.0f); + + void* mem_a = backend.Alloc(a_bf.size() * sizeof(uint16_t)); + void* mem_w1 = backend.Alloc(p1.size()); + void* mem_w2 = backend.Alloc(p2.size()); + void* mem_o16 = backend.Alloc(static_cast(kM * kN1) * sizeof(uint16_t)); + void* mem_o32 = backend.Alloc(static_cast(kM * kN1) * sizeof(float)); + void* mem_o2 = backend.Alloc(static_cast(kM * kN2) * sizeof(float)); + backend.Copy(q, mem_a, a_bf.data(), a_bf.size() * sizeof(uint16_t)); + backend.Copy(q, mem_w1, p1.data(), p1.size()); + backend.Copy(q, mem_w2, p2.data(), p2.size()); + Tensor a_t = Tensor::Contiguous(mem_a, vt::DType::kBF16, + Device{vt::DeviceType::kTENSTORRENT, 0}, + {kM, kK}); + Tensor w1_t = Tensor::Contiguous(mem_w1, vt::DType::kQ6_K, + Device{vt::DeviceType::kTENSTORRENT, 0}, + {kN1, kK}); + Tensor w2_t = Tensor::Contiguous(mem_w2, vt::DType::kQ6_K, + Device{vt::DeviceType::kTENSTORRENT, 0}, + {kN2, kN1}); + Tensor o16_t = Tensor::Contiguous(mem_o16, vt::DType::kBF16, + Device{vt::DeviceType::kTENSTORRENT, 0}, + {kM, kN1}); + Tensor o32_t = Tensor::Contiguous(mem_o32, vt::DType::kF32, + Device{vt::DeviceType::kTENSTORRENT, 0}, + {kM, kN1}); + Tensor o2_t = Tensor::Contiguous(mem_o2, vt::DType::kF32, + Device{vt::DeviceType::kTENSTORRENT, 0}, + {kM, kN2}); + + // The reference chain, all f32-out (no bf16 slot handoff): m1 ref, cast + // to bf16, m2 ref, cast to bf16 — what the bf16-out arm MUST produce. + vt::MatmulBT(q, o32_t, a_t, w1_t); + std::vector ref32(static_cast(kM * kN1), 0.0f); + backend.Copy(q, ref32.data(), mem_o32, ref32.size() * sizeof(float)); + std::vector ref16(static_cast(kM * kN1)); + for (size_t i = 0; i < ref16.size(); ++i) ref16[i] = vt::F32ToBF16(ref32[i]); + Tensor ref16_t = Tensor::Contiguous(ref16.data(), vt::DType::kBF16, + Device{vt::DeviceType::kTENSTORRENT, 0}, + {kM, kN1}); + // Warm the bf16-out arm EAGERLY once: the int8-dot launch AND the W6 + // f32->bf16 cast op must be in the program cache before the capture (a + // new binary during capture refuses: mesh_workload.cpp:196). + vt::MatmulBT(q, o16_t, a_t, w1_t); + vt::MatmulBT(q, o2_t, ref16_t, w2_t); + std::vector ref2(static_cast(kM * kN2), 0.0f); + backend.Copy(q, ref2.data(), mem_o2, ref2.size() * sizeof(float)); + + // The bf16-out m1 under capture x2 (the #2907 discipline), then the + // DEVICE consumer: m2 reads the committed bf16 slot as its activation. + std::vector dumps[2]; + int64_t demand[2] = {0, 0}; + for (int pass = 0; pass < 2; ++pass) { + void* graph = nullptr; + std::string what; + bool threw = false; + try { + backend.BeginCapture(q); + vt::MatmulBT(q, o16_t, a_t, w1_t); // bf16 out: the W6 cast fires + graph = backend.EndCaptureGraph(q); + } catch (const std::exception& ex) { + threw = true; + what = ex.what(); + } + REQUIRE_MESSAGE(!threw, "capture pass " << pass << " threw: " << what); + REQUIRE(graph != nullptr); + demand[pass] = vt::tenstorrent::LastTraceBytesForTest(); + backend.ReplayGraph(q, graph); + dumps[pass].resize(static_cast(kM * kN1), 0); + backend.Copy(q, dumps[pass].data(), mem_o16, + dumps[pass].size() * sizeof(uint16_t)); + backend.DestroyGraph(graph); + // The device-side consumer probe: m2 over the committed slot. + vt::MatmulBT(q, o2_t, o16_t, w2_t); + std::vector got2(static_cast(kM * kN2), 0.0f); + backend.Copy(q, got2.data(), mem_o2, got2.size() * sizeof(float)); + for (size_t i = 0; i < got2.size(); ++i) + CHECK(std::isfinite(got2[i])); // the word-halved-garbage signature + } + for (int pass = 0; pass < 2; ++pass) { + REQUIRE_MESSAGE(demand[pass] <= 52428800, + "capture pass " << pass << " demanded " << demand[pass] + << " B of trace region against 52428800 B"); + CHECK_MESSAGE(std::memcmp(dumps[static_cast(pass)].data(), + ref16.data(), + ref16.size() * sizeof(uint16_t)) == 0, + "replay pass " << pass + << " diverged from the f32-out reference"); + } + CHECK(std::memcmp(dumps[1].data(), dumps[0].data(), + ref16.size() * sizeof(uint16_t)) == 0); + MESSAGE("bf16-out capture x2 byte-identity + device-consumer probe: PASS"); + backend.Free(mem_a); + backend.Free(mem_w1); + backend.Free(mem_w2); + backend.Free(mem_o16); + backend.Free(mem_o32); + backend.Free(mem_o2); +} + +// KEEPQUANT W4b (issue #3031) C4 profile, spec ## W4b "Profile first": the // KEEPQUANT W4b (issue #3031) C4 profile, spec ## W4b "Profile first": the // packed arm vs the int8-dot lever per call, on ONE build, both through the // public vt::MatmulBT dispatch — bf16-out for the W4a grouped packed arm, the @@ -7991,3 +8145,24 @@ TEST_CASE("kTENSTORRENT single-chunk keep-quant decode keeps the word shadow res ::unsetenv("VT_TT_KEEPQUANT_INT8DOT"); } } + +// W4d W3 (#3042): the loader stages every weight's bf16 TILE form as the +// slot's persistent buffer. For a k-quant weight that form is dead the +// moment the keep-quant word shadow exists — the matmul reads only the +// words — but it stayed resident forever, and at 27B the staged bf16 forms +// filled the banks to 93 percent during the warm pass and fragmented them +// into the init OOM (attn_qkv alone: 97 x [10240,5120] bf16 = 9.7 GiB +// beside ~21 MiB q6_K each). EnsureKeepQuantWords now releases the slot's +// bf16 forms when it stores the shadow. RED on the pre-fix tree: the same +// assertion measured 268 MB of bf16 staging still held after the shadow +// existed (the census ledger: /tmp/census-run2.log, pers 9.7 GiB at +// block/54 with the words already resident). +// The premise-broken bf16-staging test was removed (W4d W6): its premise — +// that the loader stages a bf16 twin of packed weights — was falsified by +// the slot census (the `pers` bf16 forms are the GDN projections' LEGITIMATE +// expand-arm residency, not a double-hold), and its EnsureDevice2D-based +// setup cannot stage a packed tensor ("unsupported float dtype"). The +// equivalence it aimed at is proven TT-free by the W4d W4 standalone check +// (q6_K + q4_K, bit-for-bit vs the element-level reorder through the real +// dequantizer) and locked by the bf16-out capture test above. + From 45ad8a606ca104581da0dbf315f4e4a54642dcb3 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Sun, 13 Sep 2026 18:09:47 +0200 Subject: [PATCH 6/9] fix(BACKEND-TENSTORRENT-KEEPQUANT): gate the warm-slot release behind the TT build (#3042) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The warm-slot release calls ran unconditionally in BOTH LoadedEngine ctor ends, but model_loader.cpp includes tenstorrent_device.h behind #if defined(VLLM_CPP_TENSTORRENT) — on every non-TT build (CPU, Vulkan, Windows, the sanitizers) the vt::tenstorrent namespace is not declared and the whole board went red at the first compile ("'vt::tenstorrent' has not been declared", model_loader.cpp:2403 and :3187, the #3183 CI run). The calls are TT-only recipe hooks (VT_TT_RELEASE_WARM_ROWS releases TT warm slots; without the backend there is nothing to release), so the fix gates them behind the same define — the semantics are unchanged on the TT build and the non-TT builds return to their pre-wave behaviour. Verified both ways locally: the TT build compiles, and the exact CI compile command minus the TT define is -fsyntax-only clean on this TU. FOLLOWING_AGENTS_PROTOCOL Following-Agents-Protocol: true AI-Assisted: true Assisted-by: AGENT:zai/glm-5.3-flash [maki] --- src/vllm/entrypoints/model_loader.cpp | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/src/vllm/entrypoints/model_loader.cpp b/src/vllm/entrypoints/model_loader.cpp index d47ee47c1..f5fc8f0c4 100644 --- a/src/vllm/entrypoints/model_loader.cpp +++ b/src/vllm/entrypoints/model_loader.cpp @@ -2398,11 +2398,13 @@ LoadedEngine::LoadedEngine(HfConfig config, // slots that the captured decode never reads. Recipe-gated release — // here, at the end of BOTH LoadedEngine ctors, because the GGUF dense // load reaches the engine through more than one loader branch. +#if defined(VLLM_CPP_TENSTORRENT) if (const char* wr = std::getenv("VT_TT_RELEASE_WARM_ROWS"); wr != nullptr && wr[0] != '\0') { vt::tenstorrent::ReleaseWarmShapeSlots( static_cast(std::strtoul(wr, nullptr, 10))); } +#endif } void LoadedEngine::WarmupKernels() { @@ -3182,11 +3184,13 @@ std::unique_ptr LoadedEngine::FromModelDir( TtAllocTraceStage("load/stage/post-engine-ctor"); // W4d W3: the ctor's cold pre-warm committed full-batch-shape activation // slots that the captured decode never reads. Recipe-gated release. +#if defined(VLLM_CPP_TENSTORRENT) if (const char* wr = std::getenv("VT_TT_RELEASE_WARM_ROWS"); wr != nullptr && wr[0] != '\0') { vt::tenstorrent::ReleaseWarmShapeSlots( static_cast(std::strtoul(wr, nullptr, 10))); } +#endif return engine; } From 192ba68d2d19fb960d0a9a8e1c5cd026c46b4796 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Sun, 13 Sep 2026 18:29:27 +0200 Subject: [PATCH 7/9] record: re-anchor the three line citations the wave shifted (#3042) The wave's insertions moved LoadQwen3_5MTPFromGguf (1458 -> 1457 after the rebase accounting), LoadGgufSharedEmbedAndHeadBf16 (1045 -> 1044) and the GDN registration block (8329-8343, unchanged content, re-measured post-rebase). Checker green: ANCHOR-ROT 33 within baseline. FOLLOWING_AGENTS_PROTOCOL Following-Agents-Protocol: true AI-Assisted: true Assisted-by: AGENT:zai/glm-5.3-flash [maki] --- .agents/backend-matrix.md | 2 +- .agents/engine-matrix.md | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/.agents/backend-matrix.md b/.agents/backend-matrix.md index 26e3f0863..abd21027b 100644 --- a/.agents/backend-matrix.md +++ b/.agents/backend-matrix.md @@ -246,7 +246,7 @@ this repository. State remains `ACTIVE`; no lifecycle transition is claimed. | `BACKEND-TENSTORRENT` | Tenstorrent Blackhole (Tensix multicore, discrete PCIe, no unified memory) — thin `vt::` adapter over ttnn's existing C++ op library rather than hand-written kernels, mirroring the Metal/MLX decision (E1); vLLM has no Tenstorrent platform anywhere | vllm.cpp extension through upstream seam `platforms/interface.py:134-229` (same pattern as Metal/Vulkan) | **ACTIVE 2026-08-10.** `vt::tenstorrent::Backend` + registrar [tenstorrent_backend.cpp](../src/vt/tenstorrent/tenstorrent_backend.cpp); shared mesh-device lifecycle [tenstorrent_device.cpp](../src/vt/tenstorrent/tenstorrent_device.cpp); 17 registered ops cover OPT-125m and the Qwen3-0.6B forward (`kMatmul`, `kMatmulBT`, `kAdd`, `kRelu`, `kEmbedding`, `kLayerNorm`, `kRmsNorm`, `kSiluAndMul`, bf16/f32 casts, three RoPE forms, `kQkvSplit`, `kReshapeAndCache`, host-oracle `kPagedAttention`, `kGreedyArgmax`) [tenstorrent_ops.cpp](../src/vt/tenstorrent/tenstorrent_ops.cpp); platform allow-list selects OPT and Qwen3 [platforms/tenstorrent.cpp](../src/vllm/platforms/tenstorrent.cpp). `DeviceType::kTENSTORRENT` [device.h](../include/vt/device.h) | [test_tenstorrent_backend.cpp](../tests/vt/test_tenstorrent_backend.cpp) carries real-Blackhole op gates; [test_qwen3_paged_engine.cpp](../tests/parity/test_qwen3_paged_engine.cpp) selects Tenstorrent device-specific anchor and teacher-forced near-tie goldens. OPT-125m STRICT 6/6 passed. Qwen3 short warm smoke ran 4 tokens at about 0.28 tok/s; full 16x16 gate remains pending behind host paged attention | [tenstorrent-backend.md](specs/tenstorrent-backend.md) | `ACTIVE` | `CLAIM-BACKEND-TENSTORRENT-SPIKE` | | `BACKEND-TENSTORRENT-RESIDUAL-GOLDEN` | Child of `BACKEND-TENSTORRENT` — the owed op-level numerics evidence at the residual-RMS device boundary (`kDeviceResidualMinRows == 32`): device path does `ttnn::add`+`ttnn::rms_norm` in bf16; host/CPU path accumulates in f32. Bot-flagged on #289; never measured at the boundary. | vllm.cpp CPU oracle `RmsNormKernel` mirrors vLLM `fused_add_rms_norm` (add in model dtype, variance in f32); `src/vt/cpu/cpu_ops.cpp:371-398` | `src/vt/tenstorrent/tenstorrent_ops.cpp:1067-1117` (host/device split, `kDeviceResidualMinRows=32`) | [test_tenstorrent_backend.cpp](../tests/vt/test_tenstorrent_backend.cpp) `kRmsNorm residual: device vs CPU f32 oracle across the rows=32 boundary`: 22/22 cases on real Blackhole P150. **Measured 2026-08-11:** host path `rows<32` bit-identical to CPU (`max_abs=0`); device bf16 path `rows>=32` diverges by constant **0.0459 abs** (1.9–2.6× rel on near-zero outputs) — bf16 rounding signature, not accumulation. Decision pending the e2e golden tie-break | [tenstorrent-residual-golden.md](specs/tenstorrent-residual-golden.md) | `SPIKE` | `CLAIM-BACKEND-TENSTORRENT-RESIDUAL-GOLDEN` | | `BACKEND-TENSTORRENT-MISTRAL` | Child of `BACKEND-TENSTORRENT` — allowlist `MistralForCausalLM` (Mistral-7B-v0.3: GQA 32/8, head_dim 128, plain rope theta 1e6, untied lm_head, full attention) on the TT platform + device-aware SACRED gate. Mistral reuses the Qwen3-dense forward verbatim (qk-norm skipped); every op already registered. No new kernel. | vLLM `mistral.py::MistralForCausalLM(LlamaForCausalLM)` (already ported to the shared dense machinery); gate pattern mirrored from `test_qwen3_paged_engine.cpp:221-296` | `src/vllm/platforms/tenstorrent.cpp:52-54` (allowlist) + `tests/parity/test_mistral_paged_engine.cpp` (device-aware wiring + Backend Proof) | **Gate PASSED on Blackhole P150 (2026-08-12):** [test_mistral_paged_engine.cpp](../tests/parity/test_mistral_paged_engine.cpp) 16/16 prompts PASS (12/16 strict-exact, 4/16 near-tie, 0 forward-divergent), max gap **0.062 nats**, BACKEND PROOF 0 declines (kMatmul selections=256 = untied lm_head on device, kPagedAttention=8192). Goldens `our_ids_tenstorrent.npy` + `neartie_gap_mnats_tenstorrent.npy` (transformers alternative-oracle; POL-ORACLE deviation recorded, same as Qwen3-0.6B TT precedent). Exit SIGSEGV 139 is the known MeshDevice teardown crash, not a gate failure | [tenstorrent-mistral.md](specs/tenstorrent-mistral.md) | `ACTIVE` | `CLAIM-BACKEND-TENSTORRENT-MISTRAL` | -| `BACKEND-TENSTORRENT-GDN` | Child of `BACKEND-TENSTORRENT` — the GDN linear-attention op chain as native TT kernels, the hard prerequisite for the Qwen3.5/3.8 family (#1715): `kGdnPrefill`, `kGdnDecode`, `kL2Norm`, `kRmsNormGated`, `kCausalConv1dFwd`/`kCausalConv1dUpdate`, `kGdnStateGather`/`kGdnStateScatter`. The P150 is discrete, so an op miss refuses by name — the ops must land before any `Qwen3_5*` arch registration. Correctness oracle is our own CPU f32 arm (residual-golden precedent); no vLLM mirror exists for TT | Substrate: pinned tt-metal `ttnn::transformer::chunk_gated_delta_rule` (FLA chunked GDN forward, on-core recurrent state, `initial_state`/`final_state`) behind a varlen+state-permute adapter for `kGdnPrefill`; decode = rank-1 update composed from ttnn matmul+eltwise with a device shadow keyed by host pointer (`PagedKvShadow` pattern); contracts at `src/vt/ops.cpp:1823-2500`, CPU reference `src/vt/cpu/cpu_ops.cpp:1537-1740` | GDN kernels [GdnPrefillKernel :4220](../src/vt/tenstorrent/tenstorrent_ops.cpp#L4220) + [GdnDecodeKernel :5049](../src/vt/tenstorrent/tenstorrent_ops.cpp#L5049), registered [tenstorrent_ops.cpp:8329-8343](../src/vt/tenstorrent/tenstorrent_ops.cpp#L8329-L8343) (`kL2Norm`..`kGdnStateScatter`); op-level cases vs the CPU f32 oracle (T-sweep, indexed-`state_idx` forms, prefill↔decode state round-trip) | [test_tenstorrent_backend.cpp:1749-3340](../tests/vt/test_tenstorrent_backend.cpp#L1749-L3340): L2Norm, RmsNormGated, CausalConv1dFwd/Update, GdnPrefill, GdnDecode, prefill↔decode round-trip, StateGather/Scatter, edge shapes — every op family vs the CPU f32 oracle. W1 `34fde3502` (prefill set) + W2 `c85af0aaf` (decode+state-I/O set) landed, both fresh-review PASS; production-reached via the `Qwen3_5*` wiring row (BACKEND-TENSTORRENT-QWEN35: allow-list, e2e sacred pair 16/16 STRICT both legs); capture compatibility unmeasured behind #1625; closure [parity-ledger.md#L946](parity-ledger.md#L946) | [tenstorrent-gdn.md](specs/tenstorrent-gdn.md) | `DONE` | `c85af0aaf` | +| `BACKEND-TENSTORRENT-GDN` | Child of `BACKEND-TENSTORRENT` — the GDN linear-attention op chain as native TT kernels, the hard prerequisite for the Qwen3.5/3.8 family (#1715): `kGdnPrefill`, `kGdnDecode`, `kL2Norm`, `kRmsNormGated`, `kCausalConv1dFwd`/`kCausalConv1dUpdate`, `kGdnStateGather`/`kGdnStateScatter`. The P150 is discrete, so an op miss refuses by name — the ops must land before any `Qwen3_5*` arch registration. Correctness oracle is our own CPU f32 arm (residual-golden precedent); no vLLM mirror exists for TT | Substrate: pinned tt-metal `ttnn::transformer::chunk_gated_delta_rule` (FLA chunked GDN forward, on-core recurrent state, `initial_state`/`final_state`) behind a varlen+state-permute adapter for `kGdnPrefill`; decode = rank-1 update composed from ttnn matmul+eltwise with a device shadow keyed by host pointer (`PagedKvShadow` pattern); contracts at `src/vt/ops.cpp:1823-2500`, CPU reference `src/vt/cpu/cpu_ops.cpp:1537-1740` | GDN kernels [GdnPrefillKernel :4220](../src/vt/tenstorrent/tenstorrent_ops.cpp#L4220) + [GdnDecodeKernel :5049](../src/vt/tenstorrent/tenstorrent_ops.cpp#L5049), registered [tenstorrent_ops.cpp:8404-8418](../src/vt/tenstorrent/tenstorrent_ops.cpp#L8404-L8418) (`kL2Norm`..`kGdnStateScatter`); op-level cases vs the CPU f32 oracle (T-sweep, indexed-`state_idx` forms, prefill↔decode state round-trip) | [test_tenstorrent_backend.cpp:1749-3340](../tests/vt/test_tenstorrent_backend.cpp#L1749-L3340): L2Norm, RmsNormGated, CausalConv1dFwd/Update, GdnPrefill, GdnDecode, prefill↔decode round-trip, StateGather/Scatter, edge shapes — every op family vs the CPU f32 oracle. W1 `34fde3502` (prefill set) + W2 `c85af0aaf` (decode+state-I/O set) landed, both fresh-review PASS; production-reached via the `Qwen3_5*` wiring row (BACKEND-TENSTORRENT-QWEN35: allow-list, e2e sacred pair 16/16 STRICT both legs); capture compatibility unmeasured behind #1625; closure [parity-ledger.md#L946](parity-ledger.md#L946) | [tenstorrent-gdn.md](specs/tenstorrent-gdn.md) | `DONE` | `c85af0aaf` | | `BACKEND-TENSTORRENT-GDN-DEVICE-PURE` | Child of `BACKEND-TENSTORRENT-GDN` — make the decode-side GDN ops (`CausalConv1dUpdateKernel`, `GdnDecodeKernel`) device-resident so a tt-metal trace capture admits them, unblocking the Qwen3.5-0.8B captured arm, the last capture-blocked family with a committed eager pair (#2907, owed from #2812) | Move the per-call host orchestration on-device: resident inputs from the graph's producer ops, device-side indexed state update (baked slot addressing re-primed by the recapture cadence, or `kGdnStateGather`/`kGdnStateScatter` indirection — decided on recapture-cost measurement), conv two-views coherent on device, token readback outside the captured span | [CausalConv1dUpdateKernel :5013](../src/vt/tenstorrent/tenstorrent_ops.cpp#L5013), [GdnDecodeKernel :5225](../src/vt/tenstorrent/tenstorrent_ops.cpp#L5225) (EnsureHost ×5, ReadIdxHost, UploadTensor ×5+), q35 harness pair selection [test_qwen35_paged_engine.cpp:233](../tests/parity/test_qwen35_paged_engine.cpp#L233), fatal repro `fd_mesh_command_queue.cpp:760` 2/2 | Red-first: the #2812 loud-skip opt-in cells run captured; captured dump ×2 byte-identity with reset between; teacher-forced pair vs the transformers oracle inside the eager band; Qwen3.5 joins `DecodeCaptureDefaultArch`, ambient adjudicates CAPTURED, env=0 eager; tamper + arch-deletion mutations red; gate arm selection [test_qwen35_paged_engine.cpp:301-330](../tests/parity/test_qwen35_paged_engine.cpp#L301-L330); closure [parity-ledger.md:947](parity-ledger.md#L947) | [tenstorrent-gdn-device-pure.md](specs/tenstorrent-gdn-device-pure.md) | `DONE` | `bcade48d6` | | `BACKEND-TENSTORRENT-KEEPQUANT` | Child of `BACKEND-TENSTORRENT` — keep-quant dense dot on TT-Metal ([#2959](https://github.com/mudler/vllm.cpp/issues/2959)): GGUF k-quant arms refuse on the P150 until this row lands — W1 (#2989, open) lands the Q4_K decode, W2 (branch) lands the dot and the `kTENSTORRENT` predicate arm (`gguf_keep_quant.cpp:136-148`) admitting exactly the registered set `{Q4_K, Q5_K, Q6_K, Q8_0}` after wave-2b — while the smallest Qwen3.8 artifact that fits the card is the dense 27B Q4_K_M (17.1 GB; bf16 is 53.8 GB) and a quantized Qwen3.5 (0.8B, ~0.5 GB) is the test vehicle | ggml k-quant block formats via our reader, pinned bit-exact vs llama.cpp `b10451` (#2240/#2260 lineage); no vLLM mirror for TT (secondary substrate, deviation by design); substrate surveyed 2026-09-05: tt-metal has NO packed-weight matmul (`ttnn` quantization is per-tensor activation dquant only), so the kernel is ours — resident blocks, on-core decode to bf16 tiles through f32 (decode bit-exact vs `BlockToFloat`), the dot reusing the `kMatmulBT` tile path, device-bf16 band at the residual-golden boundary, the int8 dot as the named next lever | W1 LANDED (#2989): the Q4_K block-decode device path. W2a/2b LANDED (5e43d770, 14e8fe471): the grouped provider `kMatmulBTQuantGrouped` — E=1 dense, E=N stacked tower, registered set exactly {Q4_K, Q5_K, Q6_K, Q8_0}. W3a ON BRANCH: the E=1 arm is CHUNKED slice-decode + f32 assembly — capture-safe on the 0.8B head (trace demand 13,484,032 B of the 52,428,800 B region; capture ×2 byte-identity; routing ids never read; staging writes during capture 0; both capture-time failure modes measured and bounded by the CHUNK POLICY); E=N capture indirection staged-owed behind a MoE artifact (spec ## W4). W4b LANDED on `main` (#3087, `d9298c85c` + `96c5e4719`): the int8 lever LANDED op-level behind `VT_TT_KEEPQUANT_INT8DOT` (default OFF — the f32-out dense arm falls through to the W4a grouped arm; the e2e gate wave + production routing moved to [#3079](https://github.com/mudler/vllm.cpp/issues/3079)). OWED: the 27B arm + model wiring (wave-3b), the captured e2e vehicle leg | W1 LANDED (#2989): decode bit-exact vs `vt::cpu::BlockToFloat` across a shape sweep (red-first, op-level suite pattern). W2a/2b LANDED: grouped sweep green across the four encodings (E=1 vs cpu 0.279–0.720, vs analytic-ref 0.201–0.514). W3a ON BRANCH (red-first): the head-shape capture leg reds at base in the wave-1b class (whole-tensor device DRAM fatal, bank_manager.cpp:462) and greens with the CHUNK POLICY (trace demand 13,484,032 B, staging writes 0, ×2 byte-identity, deterministic across passes); the chunked-decode bit-exact leg pins decode math unchanged (81 assertions); the sweep under forced 3-row chunks is bit-identical to the whole-slice ratios; full backend suite 63/63 cases, 524,267 assertions; vehicle 16/16, 147/147, 0 forward-divergent. OWED: the captured e2e vehicle leg | [tenstorrent-keepquant.md](specs/tenstorrent-keepquant.md) | `ACTIVE` | `CLAIM-BACKEND-TENSTORRENT-KEEPQUANT` — implementer on the row branch; spec commit on `row/BACKEND-TENSTORRENT-KEEPQUANT`, claim in [.agents/claims/](claims/CLAIM-BACKEND-TENSTORRENT-KEEPQUANT.md) | | `BACKEND-TENSTORRENT-QWEN35` | Child of `BACKEND-TENSTORRENT` — the wiring row: `Qwen3_5ForConditionalGeneration` (dense text GDN hybrid) on the TT allow-list, the op delta the family forward refuses by name (`kGdnPostConv`, `kSigmoidGateBf16`, `kAttnQkNormRopeGate`, `kAttnQkNormRope` — pinned empirically by a W0 refusal sweep), and the first e2e gate. Makes the GDN row's ops production-reached (#1715 stays open until the family runs; GDN row lifecycle moves in the same change this lands) | Substrate: the GDN row's 8 kernels + the 27-op TT registry; e2e mirrors the ratified Mistral TT golden treatment (`VT_DUMP_IDS` bootstrap → `scripts/qwen3-neartie-gap-transformers.py` teacher-forced near-tie gaps → committed device-golden pair, POL-ORACLE deviation); op-level oracle is the CPU f32 arm. Capacity: `Qwen/Qwen3.5-0.8B` bf16 ≈1.6 GB fits (proven envelope Mistral-7B ≈14.5 GB); 27B bf16 ≈53.8 GB and GGUF k-quant arms (no TT kernels) refused by name, owed | `src/vllm/platforms/tenstorrent.cpp` (allow-list, lands last) + `src/vt/tenstorrent/tenstorrent_ops.cpp` (op delta) + `tests/parity/test_qwen35_paged_engine.cpp` (TT arm) | Spec committed 2026-08-23; W0 sweep, W1 op delta, W2 e2e, W3 GDN-reviewer leftovers (d2h counter completeness, `conv_transposed` fast-path check) owed | [tenstorrent-qwen35.md](specs/tenstorrent-qwen35.md) | `ACTIVE` | `CLAIM-BACKEND-TENSTORRENT-QWEN35` | diff --git a/.agents/engine-matrix.md b/.agents/engine-matrix.md index 4f8da9334..a49467ae2 100644 --- a/.agents/engine-matrix.md +++ b/.agents/engine-matrix.md @@ -172,8 +172,8 @@ lifecycle are unchanged. |---|---|---|---|---|---|---|---|---| | `SPEC-MTP` | Qwen3.6 MTP heads, k=1 first. **M-mtp-0 CLOSED 2026-07-24: the standalone draft head is oracle-parity-proven on BOTH checkpoints** (27B dense + 35B MoE, k=1, vLLM 0.25.0 executable @ pin `e24d1b24`) - argmax exact on 26/26 unambiguous rows each; the one remaining row per checkpoint is an EXACT oracle top1==top2 tie where vLLM's own `argmax` and `topk` disagree and our pick is a tied maximum; logits within the whole-model bound (atol 0.05 + rtol 0.05), 0/216 out-of-tol on both; shared lm_head isolated is bit-exact on the 35B NVFP4 head. **I2 scheduler-half LANDED (2026-07-24)**: host-side spec plumbing + the FROZEN spec-metadata ABI (spec §2.7) - `SpeculativeConfig`, `DraftTokenIds`, `Request::spec_token_ids`/`NumTokensWithSpec`, populated `scheduled_spec_decode_tokens`, `Scheduler::update_draft_token_ids`, `take_draft_token_ids` seam, `EngineCore::post_step`, `InputBatch::num_accepted_tokens`/`update_req_spec_token_ids`; DEFAULT-OFF and INERT (no `SpeculativeConfig` => `num_lookahead_tokens == 0`). **I3 verify-half LANDED (2026-07-24)**: greedy rejection sampler + per-request logits expansion (see `SPEC-REJECTION`, now `ACTIVE`). **I4 GDN-half LANDED (2026-07-24)**: the GDN speculative slot path + bit-exact state rollback, the piece BOTH GDN-hybrid gate checkpoints need (see `SPEC-GDN-SEGMENTS`, now `ACTIVE`). **I5a GDN LAYER ROUTING + runner spec-metadata upload LANDED (2026-07-24, `CLAIM-SPEC-MTP-I5A`)**: `GdnBlockPaged` now routes a pure-spec batch through `vt::GdnSpecDecode`/`vt::CausalConv1dSpecUpdate` and the runner uploads I4's six spec device tensors — first sub-increment of the scoped M-mtp-1 (I5a GDN wiring → I5b prepare_prefill → I5c MTP paged propose → I5d config+runner-loop+the 27B token gate, spec §5). DEFAULT-OFF INERT, bit-exact vs the I4 ops, no e2e loop yet. **I5b `prepare_prefill_inputs` LANDED (2026-07-24, `CLAIM-SPEC-MTP-I5B`, recorded under `SPEC-REJECTION`)**: the drafter prefill input-prep host routine (shift-splice + `query_len -= num_rejected` + last-token index / metadata) — second scoped M-mtp-1 sub-increment, DEFAULT-OFF INERT, unit-gated RED-first, additive. **I5d CONFIG + RUNNER LOOP LANDED, PARTIAL (2026-07-25, `CLAIM-SPEC-MTP-I5D`)**: `--speculative-config` JSON parse -> `EngineParams::speculative_config`; `LoadedEngine` resolution (`ResolveSpecConfig`/`ResolveMtp`, widened KV `MakeQwen3_5KVCacheSpec(num_spec>0)`, `BuildMtpDraft`, forced sync scheduling, `MakeScheduler(spec)`, `EngineCore(check_for_draft=true)`); the full runner verify/propose loop (draft splice, hidden-tap capture, GDN builder spec-overload feed, k+1 GDN state-slot remap + widened conv cache + draft-KV alloc, `MtpProposePrefill` post-sampling, `take_draft_token_ids`, acceptance telemetry). CUDA `-Werror` 0 warnings, cutlass-ON banner. SPEC-OFF BYTE-IDENTICAL (all gated on `spec_on()`): SACRED 27B 235/235, 35B 315/315, Coder 138/138 + unit test_runner 257 / test_mtp_speculator 169 / test_gdn_metadata_builder 483 / test_ops_gdn 3630 ALL PASS. **The three-way 27B token gate is NOT yet passing** (`tests/parity/test_qwen27_spec_decode.cpp` RUNS the loop + MEASURES the blocker): the spec-ON engine throws on the FIRST prefill step at `gdn_state_gather: working/cache row shapes must match` (`src/vt/ops.cpp:1773`) — I4's spec conv rollback needs the conv row widened to `(K-1)+num_spec` but the non-spec GDN conv ops assume `(K-1)`. Closing needs widened-cache-aware non-spec GDN conv ops + the MIXED `GdnBlockPaged` split/merge. Row LEFT `GATING` at I5e. **I5e LANDED 2026-07-25 (`CLAIM-SPEC-MTP-I5E`) — `SPEC-MTP` LEAVES `GATING`.** Made the non-spec GDN conv ops widened-cache-aware (mirror vLLM `state_len=KERNEL_WIDTH-1` + physical `stride_conv_state_tok`; leading `(K-1)` sub-window; byte-identical at `num_spec==0`, contiguous fast path kept) AND RCA'd the resulting 0-acceptance dead-drafter to the async input-combine overwriting the verify batch's draft position with the committed token (forced off under spec, nullopt-guarded). **THREE-WAY 27B GATE PASSES** (single-request greedy): our-ON == vLLM `--speculative-config mtp` greedy == our-OFF token-for-token; **acceptance 16/16 drafts accepted**, ~16 target steps saved. Spec-OFF SACRED byte-identical (27B 235/235, 35B 315/315, Coder 138/138), `test_ops_gdn` 3678, compute-sanitizer 0 on the spec step. NOT `DONE`: MIXED `GdnBlockPaged` split/merge (concurrency) + throughput A/B are I6. **I6 LANDED 2026-07-25 (`CLAIM-SPEC-MTP-I6`), `benchmark_binding=true` — the §5 c1 THROUGHPUT GATE, first spec-decode speed number:** OURS spec-ON (`examples/vllm-bench` + an additive `--speculative-config` flag, production config) vs pinned vLLM 0.25.0 spec-ON (graphed `vllm serve --speculative-config mtp` + `vllm bench serve`, `enforce_eager=False`/`FULL_AND_PIECEWISE`/inductor; MTP confirmed `Resolved architecture: Qwen3_5MTP`), SAME `{"method":"mtp","num_speculative_tokens":1}`, 27B `~/bench/q36-27b-nvfp4-vllm`, c1, greedy, 8 real prompts x 256 out, prose + code, idle box one-engine-at-a-time under one `flock`, 3 reps (cold TTFT discarded), token-identity re-confirmed FIRST (`test_qwen27_spec_decode` PASS 16/16). RESULT — **ours AT/ABOVE vLLM on EVERY measured axis** (prose / code): TPOT 66.2/62.95 vs 69.1/65.3 ms (ours ~1.04x faster), output tput 15.10/15.72 vs 14.43/15.13 tok/s (+4.6%/+3.9%), ITL 121.6/121.1 vs 123.2 ms, TTFT(warm) 131/131 vs 151.5/181 ms, acceptance ours 0.85/0.92 vs vLLM 0.838 overall (within noise, live drafter both), peak RSS 28.4 GB ON / 24.8 GB OFF (both inside the 119 GiB pool). Spec helps both (ours 1.52x/1.59x, vLLM 1.51x/1.60x TPOT); ours already ~4% faster spec-OFF. STAYS `ACTIVE`: the c>1 mixed spec+non-spec `GdnBlockPaged` split/merge is still refused (needs a row `IndexSelect`/`IndexCopy` vt op) + owes a c>1 A/B, and no user-facing supported `--speculative-config` on the OpenAI server yet (bench flag example-only/additive). Raw logs dgx `~/work/mtp-bench-i6/{results,vresults}`. **I7 LANDED 2026-07-25 (`CLAIM-SPEC-MTP-I7`, `benchmark_binding=true`) — the MIXED spec+non-spec GDN batch (concurrency), the server/CLI `--speculative-config`, and the c>1 A/B — implementation COMPLETE + at vLLM parity; STAYS `ACTIVE` for one honest reason (below), NOT a lag.** New row op `vt::IndexSelect`/`vt::IndexCopy` (CUDA==CPU bit-exact at GDN widths, RED-first); `GdnBlockPagedMixedSpec` split/merge (mirror `qwen_gdn_linear_attn.py:1329-1576`) proven MODEL-INDEPENDENTLY bit-exact (mixed == pure spec + pure prefill, 27B/35B, `test_qwen3_5_gdn_spec_routing`, RED-first by a broken merge); compute-sanitizer 0 on the mixed step + op; server (I5d) + CLI (ABI v6) `--speculative-config`. **c>1 A/B (both spec-ON, same config):** ours ON-PAR-OR-ABOVE vLLM at c2/c4/c8 (output tput within ~+/-2%, ours +1.6%/+2.5% c2, +0.9%/+1.7% c4, +0.9%/-1.1% c8 within noise, prose/code; both ~1.5x spec speedup — does NOT go neutral; acceptance 0.84-0.92 vs vLLM 0.835). **Why STAYS `ACTIVE` (honest, not a lag):** the DONE criterion's strict `token-exact at c>1` clause is a proven MODEL impossibility — the 27B greedy is bf16-batch-nondeterministic (spec-OFF max_seqs 4-vs-1 differs 2/3 short prompts, NO spec involved), affecting vLLM identically, so exact c>1 token identity cannot be met by any correct implementation; c>1 correctness is instead established by the model-independent bit-exact split/merge proof + acceptance parity (near-tie-distributional-gate), with token-exact strict at c1 (I6). No missing work, no lever — the DONE final call is deferred to the user given this criterion ambiguity. SACRED spec-OFF byte-identical 27B 235/235, 35B 315/315, Coder 138/138; CUDA `-Werror` 0 warnings. Raw logs dgx `~/work/mixed-batch/{cN_results,cN_vresults}`. **I8 — `SPEC-MTP` → `DONE` 2026-07-26 (`CLAIM-SPEC-MTP-DONE`, records-only, ZERO code):** the user RATIFIED the deferred c>1 criterion — at concurrency > 1 the DONE bar is the near-tie-distributional form (ours ∈ vLLM's batch-nondeterministic set) + the SPEED delta, NOT strict token-exact (a proven bf16-batch-nondeterminism MODEL impossibility that affects vLLM identically). Both I6-owed DONE items are therefore CLOSED: (1) the MIXED spec+non-spec `GdnBlockPaged` split/merge (I7, model-independently bit-exact + compute-sanitizer 0) with the c2-c8 A/B on-par-or-above vLLM, and (2) the server + CLI + C-ABI(v6) `--speculative-config` flag (I5d/I7, `examples/server/main.cpp`+`examples/cli/main.cpp`+`src/capi/vllm_c.cpp`). MTP k=1 spec-decode is COMPLETE and gated: 27B three-way token-exact at c1 (I5e), c1 above vLLM on every axis (I6), c2-c8 on-par-or-above (I7), spec-OFF byte-identical SACRED (27B 235/235, 35B 315/315, Coder 138/138). This transition is byte-identical BY CONSTRUCTION (`git diff --stat` = records only; ZERO `src/`/`include/`/`examples/` touched, so the I5d/I6/I7 GPU gates stand on this exact code). Tracked follow-ons: the 35B `Qwen3_5MoeMTP` full e2e token gate (M-mtp-2) is now **CLOSED — `DONE` 2026-07-26 (`CLAIM-SPEC-MTP-M-MTP-2`)**: three-way token-exact 16/16 vs the live vLLM 0.25.0 oracle (spec-ON AND spec-OFF), acceptance 16/16 both sides, c1 spec-ON 1.19x TPOT / +16.3% output-tput vs spec-OFF (0.908) — `MODEL-SPEC-qwen3-5-mtp-qwen3-5-moe-mtp` `GATING`→`DONE`, so MTP is `DONE` on BOTH gate models. Remaining spec-decode follow-on: `SPEC-DFLASH` (oracle-BLOCKED, vllm#40898) | T1 | `vllm/v1/worker/gpu/spec_decode/mtp/speculator.py:12`; `vllm/model_executor/models/qwen3_5_mtp.py:63,129-165,192-301`; **I5d** `vllm/engine/arg_utils.py` (`--speculative-config`); `vllm/v1/worker/gpu/model_runner.py:1455-1489` | `include/vllm/config/speculative.h`; `include/vllm/v1/core/sched/scheduler.h`; `src/vllm/v1/core/sched/scheduler.cpp`; `include/vllm/v1/worker/gpu/input_batch.h`; `include/vllm/model_executor/models/qwen3_5_mtp.h:23,58`; `src/vllm/model_executor/models/qwen3_5_mtp.cpp:271`; `src/vllm/model_executor/models/qwen3_5.cpp:3336,3359`; **I5d** `src/vllm/config/speculative.cpp`; `src/vllm/entrypoints/model_loader.cpp` (`ResolveSpecConfig`/`MakeKVCacheMaybeSpec`/ctor wiring); `src/vllm/v1/worker/gpu/runner.cpp` (splice/tap/GDN spec feed/`propose_drafts`/`take_draft_token_ids`/spec-slot remap/draft-KV alloc); `examples/server/main.cpp` | `tests/vllm/v1/test_scheduler.cpp:1135,1238,1272,1316`; `tests/vllm/v1/worker/test_input_batch.cpp`; `tests/vllm/v1/spec_decode/test_mtp_speculator.cpp:201,225,263,299,331` (7/7 cases, 141 assertions); oracle runner `tests/parity/test_op_parity.cpp:1373` + focused case `:1914` (20/20 assertions, both checkpoints, `VLLM_MTP_REQUIRE_CHECKPOINTS=1`); goldens `tests/parity/goldens/qwen3_5_mtp_head_{27b,35b}/`; dump `tools/parity/dump_qwen3_5_mtp.py:144`; **I5d** `tests/parity/test_qwen27_spec_decode.cpp` (three-way gate, RUNS + measures the RCA blocker); **I6** `examples/bench/{main.cpp,bench_core.h}` (additive `--speculative-config` bench flag + acceptance telemetry); **I7** `tests/vllm/models/test_qwen3_5_gdn_spec_routing.cpp` (mixed == pure spec + prefill bit-exact), `tests/parity/test_qwen27_spec_decode_concurrent.cpp`, `tests/vt/test_ops_gdn.cpp` (IndexSelect/IndexCopy); DONE closure [ledger](parity-ledger.md#L714) | [mtp-spec-decode.md](specs/mtp-spec-decode.md) | `DONE` | `72f9fb1` | | `SPEC-MTP-K-GT-1` | **MTP speculation DEPTH (`num_speculative_tokens` > 1).** Ports the autoregressive multi-step propose the k=1 early exit sits in front of, so a configured depth is SERVED instead of silently degraded. Before it, `--num-speculative-tokens 3` reserved KV for 3, captured the verify shape at T=4 and stashed ONE draft per request, with no error and no log; a refusal by name landed first and this row removed it in the same flow. `MtpProposeDrafts` runs the prefill, the k=1 early exit, then `prepare_decode_inputs` and the k-1 single-token draft decode steps over the draft's own paged KV, with `update_draft_inputs` recording each step and feeding it forward. Greedy plus accept-if-equal makes the emitted sequence INDEPENDENT of k, so a token-identity gate cannot see a clamped drafter and every depth assertion needs a positive witness beside the identity. The per-depth counters were the FIRST witness and a fresh review proved them BLIND: they report the LENGTH of the emitted draft list, so a propose that runs one forward and pads all k columns satisfies them, and acceptance is zero at every depth on the CPU model, so no acceptance figure separates the arms either. TWO witnesses survive, because one does not cover both failures. `spec_mtp_draft_decode_forwards() == spec_mtp_propose_calls() * (k - 1)`, counted after each draft decode forward RETURNS and guarded by a non-zero call count, catches a propose that SHORT-CIRCUITS or CLAMPS. A third fresh review then proved it does NOT catch PADDING, since a loop that runs every forward and then discards what it sampled increments it honestly. `spec_mtp_proposals_with_varied_drafts()`, read at the CONSUMER on the array the propose delivered, catches exactly that. NEITHER shows per-column provenance, and neither does a non-zero acceptance count AT DEPTH, which a padded row earns whenever the target repeats a token. The owed DGX gate closes it with a per-depth acceptance RATE against a PADDED CONTROL. The CPU tier therefore proves k drafts are PROPOSED and VERIFIED, never ACCEPTED at depth. DEFAULT unchanged at k=1 (both checkpoints' `n_predict`). **NO speed number at any k>1**: the GPU was held by another session for the whole flow, so the DGX three-way at k=2..4 on the 27B and 35B and the matched-k throughput A/B are OWED, as is the bf16 GDN-state arm (the CPU gate runs the f32 arm because `vt::CausalConv1dSpecUpdate` rejects bf16 off CUDA). Also owed and filed: [#1020](https://github.com/mudler/vllm.cpp/issues/1020), a step whose ACTUAL draft count differs from the configured k leaves the captured verify graph silently. | T1 | `vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py:129-274,335-371,374-419,426-471,597-671,674-771` @ `555967922`; `vllm/config/speculative.py:967-991` | [`src/vllm/v1/worker/gpu/spec_decode/mtp/speculator.cpp`](../src/vllm/v1/worker/gpu/spec_decode/mtp/speculator.cpp) (`MtpProposeDrafts`); [`prepare_decode_inputs.cpp`](../src/vllm/v1/worker/gpu/spec_decode/autoregressive/prepare_decode_inputs.cpp); `Qwen3_5MTPModel::GatherHiddenRows` ([qwen3_5.cpp](../src/vllm/model_executor/models/qwen3_5.cpp)); `GPUModelRunner::propose_drafts` + the per-depth counters ([runner.cpp](../src/vllm/v1/worker/gpu/runner.cpp), [runner.h](../include/vllm/v1/worker/gpu/runner.h)); the in-memory `mtp_weights` seam ([model_loader.h](../include/vllm/entrypoints/model_loader.h)) | [`test_mtp_depth`](../tests/vllm/v1/spec_decode/test_mtp_depth.cpp) 5/5, 63 assertions (k=1,2,3,4 through `LoadedEngine`, greedy tokens identical to spec-OFF, each arm witnessed BOTH by the draft decode forwards the propose RAN and by whether the DELIVERED draft row varied with depth; neither witness shows per-column provenance, which is owed to the DGX gate); [`test_prepare_decode_inputs`](../tests/vllm/v1/spec_decode/test_prepare_decode_inputs.cpp) 8/8, 33 (both kernel ports + both `max_model_len` clamps, 5 mutations caught); [`test_speculative_mtp_depth`](../tests/vllm/config/test_speculative_mtp_depth.cpp) 4/4, 20; full CPU suite ctest 493 passed / 0 failed / 2 skipped of 495 (the two skips checkpoint-gated and unrelated) | [mtp-k-gt-1.md](specs/mtp-k-gt-1.md) | `ACTIVE` | `CLAIM-SPEC-MTP-K-GT-1` ([#81](https://github.com/mudler/vllm.cpp/issues/81)) | -| `SPEC-MTP-GGUF` | MTP speculative decoding from a GGUF TARGET. Today `FromModelDir` refuses `mtp`+GGUF outright (`src/vllm/entrypoints/model_loader.cpp:717-723`) on the original spike's assumption that GGUF exports carry no `mtp.*` ([mtp-spec-decode.md](specs/mtp-spec-decode.md):979-980, "until we re-export GGUFs with the head"). That is stale: llama.cpp's Qwen3.5 converter DOES emit the head, under layer-indexed `nextn` naming, and our own `HfConfigFromGguf` ALREADY reads `nextn_predict_layers` (it just discards the value into the trunk layer count). Gap is a `TensorResolver` over `GgufFile` mapping `mtp.*` onto `blk.{L+i}.nextn.*` with dequant-to-bf16, one config field, and narrowing the rejection to `dflash`. `ngram`+GGUF already works and is untouched. Qwen3.5/3.6 only (the widened spec KV path serves no other arch). NO ABI change | T2 | llama.cpp (the producer contract; vLLM has no GGUF MTP path) `conversion/qwen.py:535-604` `_Qwen35MtpMixin` (the authoritative `mtp.*`->`nextn` remapper + `add_nextn_predict_layers`); `gguf-py/gguf/constants.py:129,910-917,1494-1501`; `gguf-py/gguf/tensor_mapping.py` `NEXTN_*` | **G1-G3 LANDED 2026-07-28.** `HfConfigFromGguf` republishes the head depth `src/vllm/model_executor/models/qwen3_5_gguf_weights.cpp:598` (`c.raw["mtp_num_hidden_layers"] = nextn`, previously read then discarded); the head loader `LoadQwen3_5MTPFromGguf` `src/vllm/model_executor/models/qwen3_5_gguf_weights.cpp:1458` (+ decl `include/vllm/model_executor/models/qwen3_5_gguf_weights.h:143`) reusing the TRUNK helpers `OwnNormMinus1`/`OwnMatmulWeight`/`OwnBf16`/`LoadAttnGguf`/`LoadMoeGguf` so the head inherits the GGUF (w+1) norm storage, quantization/residency routing and torch [N,K] shapes; `NumMtpLayers`/`UsesDedicatedEmbeddings` exported out of the anon namespace `include/vllm/model_executor/models/qwen3_5_mtp.h`; rejection narrowed to dflash + a head-less-GGUF check `src/vllm/entrypoints/model_loader.cpp` and the head attached in the GGUF branch; **G4 GREEN + `CPU-SPEC-DIVERGENCE` FIXED 2026-07-28**: root cause `src/vllm/model_executor/models/qwen3_5.cpp:3616` sized the GDN state gather/scatter row by `(Kw-1)` while the speculative persistent row is `(Kw-1)+num_spec`, so `GatherRows`/`ScatterRows` mis-strode the slot AND every channel past the first, corrupting post-prefill recurrent state. Fix = `CopyStateRowsStrided` (same TU) used by `GatherStateF32`/`ScatterStateF32` when `cache.shape[2] != work.shape[2]`; the contiguous helpers are kept when the widths agree, so every non-spec path is byte-identical by construction. CPU-only in effect (the fp16/bf16 arm routes through the `GdnStateGather`/`Scatter` ops, so CUDA was never exposed; no GPU result affected) | `tests/vllm/models/test_qwen3_5_gguf_mtp.cpp:109,146,156,184` **4 cases, and the split is the 2026-08-21 repair** ([#1454](https://github.com/mudler/vllm.cpp/issues/1454)): the file used to be the env-gated pair ALONE, each opening on a bare `return`, so with `VLLM_MTP_GGUF_MODEL` unset it reported `test cases: 2 \| 2 passed`, **`assertions: 0`**, `Status: SUCCESS!`, exit 0 - which is every CI run of this repository, the variable being set nowhere in `.github/workflows/`. The `18 assertions` this cell used to record was the LIVE count and was never once reached in CI. Now `:109` and `:146` are **HERMETIC** (KV-only synthetic GGUFs, no weight bytes, 18 assertions on any machine) and pin the arithmetic the old file only NAMED in a comment above `CHECK(c.num_hidden_layers > 0)`: `num_hidden_layers + mtp_num_hidden_layers == block_count` over 65/1, 25/1 and 28/3 - the third arm separating `- nextn` from `- 1` - plus the head-less arm, where the key is NOT published and `NumMtpLayers` answering 1 for an absent key is exactly why the invariant cannot be written with that helper alone. `:156` and `:184` stay env-gated on `VLLM_MTP_GGUF_MODEL` (so CI stays asset-free) and now SKIP LOUDLY with a `MESSAGE` naming the variable, as `tests/vllm/entrypoints/test_gguf_mmproj_reach.cpp` does; `:156` re-derives the same invariant from the file's OWN `block_count` kv. Unset: **4 cases / 18 assertions / `Status: SUCCESS!` / rc 0**. Live on `Qwen3.8-27B-Q4_K_M.gguf` (`block_count` 65, `nextn_predict_layers` 1): **4 cases / 38 assertions / `Status: SUCCESS!` / rc 0**. Mutation-proved on the production line `src/vllm/model_executor/models/qwen3_5_gguf_weights.cpp:889`, both compiling clean and both restored against a pre-taken sha256: `= block_count` (drop the subtraction) 3/4 cases, 9/18 red, exit 1; `= block_count - 1` (the wrong constant) 2/4 cases, 5/18 red, exit 1. The SAME mutations left the PREVIOUS file at 2/2 cases, 0 assertions, `SUCCESS!`, exit 0. Correctness of the production line is unchanged and was never in question (`1a4db5c3c`, `493327b4e`); this was a test defect. Live-arm content unchanged: depth reaches config.raw; fc is [H,2H] verbatim; 3 norms [H]; head block is full-attention. **RED-first BEHAVIOURAL** (reverting only the G1 line fails both cases 2/2). Trunk inertness: `test_gguf` 103, `test_gguf_qwen36_loader` 99, `test_gguf_keep_quant` 5958, `test_gguf_dequant` 215, `test_capi` 33/232 all unchanged; `tests/parity/test_qwen35_gguf_spec_decode.cpp:74,139` - spec-ON == spec-OFF token-exact with 13 proposed/11 accepted, plus an `ngram` regression guard (widens the cache, never runs the spec conv update) that was token-exact throughout and pinned the widening as innocent. Regression sweep all unchanged: ops_gdn 1825, gdn_metadata_builder 483, gdn_prefill_conv 28, gdn_spec_routing 12, gguf 103, gguf_qwen36_loader 99, gguf_keep_quant 5958, gguf_dequant 215, llm_engine 196, input_batch 163, runner 257, capi 232 **GPU CLOSE-OUT + DEVICE-DELTA ATTRIBUTION 2026-07-28 (`G5`-`G7`), ledger [parity-ledger.md#L800](parity-ledger.md#L800).** The GPU end-to-end gate re-run on a from-scratch RELEASE-TARGET build (`-DVLLM_CPP_CUDA_ARCHITECTURES=121a`, build dir DELETED first; arch VERIFIED by `build-cuda/CMakeFiles/vllm.dir/flags.make` `--generate-code=arch=compute_121a,code=[compute_121a,sm_121a]` and by `cuobjdump -lelf` 20 cubins ALL `sm_121a` zero sm_75, NOT by `CMakeCache.txt`, whose `CMAKE_CUDA_ARCHITECTURES:STRING=75` is the `enable_language(CUDA)` compiler-probe default shadowed by the normal variable at `CMakeLists.txt:186` - the prior wrong-arch conclusion was that decoy): dgx.casa GB10 under `flock $HOME/gpu.lock`, 35B A3B NVFP4 GGUF, **2/2 cases, 10/10 assertions, exit 0**, spec-ON token-identical to spec-OFF, 13 proposed / 11 accepted, 90.2 GiB peak RSS, 8m01s; re-run on the EXACT committed source **3/3 cases, 10/10 assertions, exit 0**, 7m25s, the new probe case SKIPping and adding zero assertions. **The CPU-vs-GPU token delta is a MEASURED near-tie, not a defect** (it was never this row's bar - spec-ON == spec-OFF WITHIN a device is): NEW double-gated spec-OFF-only probe `tests/parity/test_qwen35_gguf_spec_decode.cpp:217` (asset + `VLLM_MTP_GGUF_PROBE=1`, 20 alternatives per position, 484/484 assertions per arm, GPU then `CUDA_VISIBLE_DEVICES=` in one `flock` series) shows both arms picking `11751` at position 0 and forking at position 1 on a BIT-IDENTICAL prefix: GPU rank1 `13` -0.773180 over rank2 `11` -0.847055 (margin 0.0739 nats), CPU rank1 `11` -0.765499 over rank2 `13` -0.830374 (margin 0.0649 nats). Each device's pick is the other's rank 2, both ~7x inside the ratified 0.5-nat band, and the cross-device disagreement on the SAME token (0.057 and 0.082 nats) EXCEEDS the margin being decided, so rounding settles it; the 24 texts look unrelated only because positions 2+ cascade off that one coin flip. Margin sweep over all 24 positions: **GGUF GPU and GGUF CPU carry ZERO exact ties**, minimum margins 0.0482 and 0.0649 nats, and both arms reproduced their sequence across every run. **Gate 4 MET on the safetensors sibling of the same quantization run** (`FromModelDir` takes it unchanged): acceptance 12 proposed / 11 accepted vs the GGUF's 13 / 11. That arm, however, FAILS spec-ON == spec-OFF at concurrency 1 and does not reproduce its own spec-OFF sequence run to run, and the probe attributes both to THREE EXACT ties (positions 7, 10, 16, bit-identical logprobs) produced by its 1/16-grid quantized-GEMM logits - which EXONERATES the GGUF arm and opens a recorded, not-root-caused `SPEC-MTP` item on the safetensors NVFP4 path, not on this row. Gate 3 is NOT APPLICABLE twice over: no F16/F32 head-carrying export exists, and the only same-weights sibling is not token-stable against itself. **EVIDENCE RE-ANCHORED 2026-07-29 to a PRODUCTION-CONFIGURED build, because every GPU number above came from a build configured WITHOUT `-DVLLM_CPP_CUTLASS_DIR` and WITHOUT `-DVLLM_CPP_TRITON=ON`** (the defect `CLAIM-27B-GATE-RCA` proved, which runs the emulation fp4 GEMM + hand GDN kernels). Re-run from a clean `git archive` tree of `main` `3f34534d`, build proven correct three ways (configure log has ZERO `CUTLASS not found` and prints `CUTLASS found ... sm120a NVFP4 cutlass GEMM` + `FlashAttention-2 ... ENABLED for arch(es) [121a]` + the vendored `sm_121a` Triton-AOT lines with `MANIFEST hashes OK`; `cuobjdump -lelf` 40 cubins ALL `sm_121a`, zero `sm_75`; SACRED `test_qwen27_paged_engine` **235/235 exit 0**, and the build precondition proven to FIRE by recompiling only that TU without the two defines against the same `libvllm.a`, which throws and exits 1 with 0 assertions). **The row PASSES UNCHANGED:** `tests/parity/test_qwen35_gguf_spec_decode` **3/3 cases, 10/10 assertions, exit 0**, spec-ON token-identical to spec-OFF, **13 proposed / 11 accepted (identical to the recorded number)**, 90.26 GiB, 7m13.59s; loader gate 19 assertions on the Qwen3.5-2B and 18 on the 35B A3B, unchanged. **ONE recorded finding is RETRACTED by the re-measurement: the CPU-vs-GPU token delta was a BUILD artifact, not a device near-tie cascade.** On the production build both devices emit the SAME 24 tokens; the probe shows GPU rank1 `11` -0.763897 over rank2 `13` -0.824083 where the defective build had rank1 `13` -0.773180 over rank2 `11` -0.847055, while the CPU arm is bit-identical to the earlier measurement (CUTLASS and Triton are CUDA-only). Zero exact ties in either arm, min margins 0.060186 GPU / 0.064875 CPU, 484/484 assertions per arm. Evidence: [docs/BENCHMARKS.md](../docs/BENCHMARKS.md) top section, [parity-ledger.md](parity-ledger.md) | [specs/gguf-mtp-spec-decode.md](specs/gguf-mtp-spec-decode.md) | `DONE` | `edf91449` | -| `SPEC-DFLASH-GGUF` | DFlash speculative decoding from GGUF, two axes: (A) GGUF DRAFT + safetensors target, (B) GGUF target too. llama.cpp master carries a full `dflash` GGUF contract (arch string `dflash`, tensors `fc`/`enc.output_norm`/`output_norm`/`blk.N.*`, KVs `dflash.target_layers` + `dflash.target_hidden_size`); the arch is ABSENT from checkouts older than ~2026-07, so a stale tree reads as "no contract exists". The GGUF tensor set omits `token_embd`/`output` because the draft SHARES the target's embed+lm_head, which is exactly what `LoadDflashDraft` already does. Blockers are in the loader, not the model: `MakeDflashDraftConfig` reads `draft_dir/config.json` (a GGUF has none), `ResolveDflashDraftDir` probes for `config.json` so it cannot see a `.gguf`, and `LoadDflashDraft` is typed on `std::vector` for the shared bf16 head (the axis-B blocker). Axis A independently shippable. NO ABI change | T2 | llama.cpp `origin/master` @ 2026-07-28 (tag era `b10158`): `gguf-py/gguf/constants.py:547,1151,4350`; `gguf-py/gguf/tensor_mapping.py:1297-1305` (`ENC_OUTPUT_NORM`<-`model.hidden_norm`, `FC`<-`model.fc`); `conversion/qwen.py:351` (mask token via the standard tokenizer KV); `convert_hf_to_gguf.py --target-model-dir` | **GD1-GD7 LANDED 2026-07-28 (BOTH AXES COMPLETE and PROVEN end to end on GB10)**: `MakeDflashGgufConfig` + `LoadQwen3DFlashFromGguf` `src/vllm/model_executor/models/qwen3_dflash_gguf.cpp:88,227` (+ header), `IsDflashGgufDraft` + the `.gguf` branch in `ResolveDflashDraftDir`/`LoadDflashDraft` `src/vllm/entrypoints/model_loader.cpp:121,222`. Goes through the `TensorResolver` seam (unlike `SPEC-MTP-GGUF`) because dflash norms are RAW, so the existing `LoadQwen3DFlash` qkv/gate_up concatenation is reused unchanged. **`GD4` defect FIXED** (`model_loader.cpp:238-249`): the GGUF branch left `config.vocab_size` 0 - correct for `MakeDflashGgufConfig` (the DFLASH arch has no vocab KV and no `token_embd`) but fatal for the forward, which sizes the shared embedding view as `{config.vocab_size, H}`, so the first propose threw `cuda embedding: empty table (vocab 0)`. Now back-filled from the target's `embed_tokens` rows (the condition is on the VALUE, not the draft source, so it generalizes to a GGUF target). Load-level green had hidden it; only GENERATING found it. **GD5-GD7 = axis B**: `SharedHeadSource` `src/vllm/entrypoints/model_loader.cpp` re-expresses the shared bf16 `embed_tokens`+`lm_head` seam as a SOURCE and re-types `LoadDflashDraft`'s second parameter - THAT TYPE was the whole axis-B blocker - with the GGUF arm `LoadGgufSharedEmbedAndHeadBf16` `src/vllm/model_executor/models/qwen3_5_gguf_weights.cpp:1045` reusing the trunk loader's tied-embedding rule and sidecar-aware dequant instead of restating them; the shared-head load moved into ONE common tail so all four (draft format x target container) combinations run identical code; the `dflash` half of the GGUF-branch rejection `model_loader.cpp` is deleted (the `mtp` half untouched) and the draft load is wired into the GGUF branch | `tests/vllm/models/test_qwen3_dflash_gguf.cpp:36,84` 2 cases / 47 assertions against the REAL published Qwen3.6-27B DFlash draft (env-gated `VLLM_DFLASH_GGUF_MODEL`, CI asset-free): the +1 target-layer offset undone against the KV read back from the same file, block_size/mask_token present, vocab_size left 0, layer_types cover every block, fc `[H, H*num_taps]` with `nk` SET, qkv/gate_up row-concat shapes, embed/lm_head left EMPTY for the target. **RED-first BEHAVIOURAL** (dropping the `-1` fails the offset checks). **`GD4` e2e gate** `tests/parity/test_qwen27_dflash_spec_decode.cpp:343` (second case, draft source env-driven via `VLLM_DFLASH_DRAFT`/`_B`; asset-gated, CI-inert): on dgx GB10 sm_121a against the Qwen3.6-27B NVFP4 safetensors target, the Q4_K_M GGUF draft and the bf16 z-lab safetensors draft produce **token-for-token IDENTICAL** DFlash-ON continuations with **IDENTICAL** accepted/proposed (20/80 on a 24-token prompt, 42/96 on a 48-token prompt), spec-OFF self-reproducible 3/3 and 0 exact ties (min margin 0.197/0.400 nats). Regression: gguf_mtp 19, qwen35_gguf_spec_decode 10, gguf 103, gguf_qwen36_loader 99, gguf_keep_quant 5958, ops_gdn 1825, llm_engine 196, capi 232, runner 257 all unchanged. **`GD5` unit gate** `tests/vllm/test_gguf_qwen36_loader.cpp` 3 new synthetic-GGUF cases (6 cases / 286 assertions total, CPU and the dgx CUDA build): the untied head really comes from `output.weight` and not the embedding (distinct fill values), the tied fallback aliases it onto `token_embd`, the `nk` flags separate the gather table from the MatmulBT weight, a file with no `token_embd` is refused. 3-mutant battery, 3 caught (`nk` flipped, head forced to the embedding, tied forced false). **`GD7` e2e gate** `tests/parity/test_qwen27_dflash_spec_decode.cpp` third case (targets env-driven via `VLLM_DFLASH_TARGET_B`; asset-gated, CI-inert): on dgx GB10 sm_121a the Qwen3.6-27B NVFP4 **GGUF** target + `Q4_K_M` GGUF draft loads, takes the shared head from the GGUF, generates, and its DFlash-ON continuation is **token-for-token IDENTICAL to that same target's spec-OFF** (24/24, the STRICT form) with acceptance ALIVE at 14/160; 1 case / 15 assertions, exit 0. **The spike's highest risk is EMPTY on this asset, proven not assumed**: the 27B NVFP4 GGUF stores `token_embd`/`output` as ggml BF16, byte-identical to the safetensors sibling (2,542,796,800 bytes each, ZERO differing), so B1's shared-head read is verbatim, not a dequant. Acceptance IS lower than the safetensors-target arm and is NOT chargeable to the head: the two containers diverge at index 4 with NO speculation, because `QUANT-GGUF-NVFP4` is dequant-only so the GGUF target computes in bf16 while the safetensors target runs the true W4A4 kernels. **RE-MEASURED 2026-07-29 on a PRODUCTION-CONFIGURED build (`CLAIM-GGUF-SPEC-REVERIFY`), because every GD4/GD7 GPU number above came from a build configured WITHOUT `-DVLLM_CPP_CUTLASS_DIR` and WITHOUT `-DVLLM_CPP_TRITON=ON`.** Build proven correct three ways (see the `SPEC-MTP-GGUF` row; SACRED 27B **235/235**, `cuobjdump` 40 cubins all `sm_121a`). **AXIS B HOLDS EXACTLY**: `test_qwen27_dflash_spec_decode -tc="dflash axis-B*"` **15/15 assertions, exit 0**, GGUF-target DFlash-ON token-identical to that target's own spec-OFF 24/24, acceptance **14/160 unchanged**, cross-target spec-OFF divergence still at index 4, 81.01 GiB peak RSS, 6m53.08s. **AXIS A WAS RED ON THE 48-TOKEN PROMPT (reproducibly, 3 of 3 runs) AND IS NOW CLOSED.** The RED was real: cross-format TOKEN identity held on both prompts, but the exact accept-count half of bar (a) failed (`arm_a.proposed == arm_b.proposed` / `arm_a.accepted == arm_b.accepted`) because the Q4_K_M draft measured **46/112** against the bf16 z-lab draft's **47/96** (one extra 16-wide propose block, one fewer acceptance, zero token difference), 15/17, exit 1; the 24-token prompt stayed green at 17/17 with both drafts at 15/144. **`GD9` 2026-07-29 root-caused it IN WEIGHT SPACE as ordinary `Q4_K_M` cost, category (a), not a defect in our GGUF draft path - and the bar's own premise ("Same weights, two containers") was false for the asset it was pointed at.** The publishing repo also carries an UNQUANTIZED `BF16` GGUF (3,471,497,440 B) beside `Q8_0`/`Q6_K`/`Q5_K`/`Q4_K_M`, which the spec had recorded as nonexistent; that retired the `NOT APPLICABLE` on gate 2. CPU gate `tests/vllm/models/test_qwen3_dflash_gguf.cpp` third case (asset-gated `VLLM_DFLASH_GGUF_BF16_MODEL` + `VLLM_DFLASH_ST_DIR`): `LoadQwen3DFlashFromGguf(BF16)` is **BYTE-IDENTICAL to `LoadQwen3DFlash(z-lab shards)` on all 58 tensors, 302/302 assertions, exit 0**, and FUNCTIONALLY RED against the `Q4_K_M` file (21/302 red, exactly the 21 quantized matmul tensors), so not a vacuous pass. Supporting: our `DequantGgufRowToBf16` is bit-equal to `gguf-py`'s `gguf.quants.dequantize` on the real `fc.weight` (Q4_K), `blk.0.attn_q.weight` (Q4_K) and `blk.2.ffn_down.weight` (Q6_K), zero differing bf16 values; the ladder's mean relative weight error is monotone and uniform with NO outlier tensor (BF16 0, Q8_0 5.6e-3, Q6_K 1.85e-2, Q5_K 3.85e-2, Q4_K_M 7.6e-2); the only numeric config delta is `rms_norm_eps` at 2.5e-9 relative. Also landed: an off-by-default `VT_SPEC_TRACE=1` per-block propose/accept trace in `GPUModelRunner::sample_tokens_with_rejection` (`src/vllm/v1/worker/gpu/runner.cpp`). **`GD10` 2026-07-29 CONFIRMED IT END TO END ON GB10 and closed gates 3 and 5.** Build proven production-configured three ways (configure log 0 `CUTLASS not found`; `cuobjdump -lelf` 40 cubins ALL `sm_121a` zero `sm_75` on both binaries; SACRED `test_qwen27_paged_engine` **235/235, exit 0**, 31.34s, 23.67 GiB). The **`BF16` GGUF draft reads EXACTLY 47/96**, the safetensors draft's own number, at 48 tokens on the discriminating prompt - reproduced 2 of 2 - plus 27/64 = 27/64 at 24 tokens and 15/144 = 15/144 on the second prompt, tokens IDENTICAL throughout, 17/17 exit 0 each time; the `Q4_K_M` arm reads 46/112 on the SAME binary in the SAME `flock` series. Restoring only the draft's numeric precision restores the count, so quantization is the whole cause and nothing structural survives. Bar (a) is consequently SPLIT rather than relaxed (`tests/parity/test_qwen27_dflash_spec_decode.cpp`): tokens stay EXACT unconditionally; accept counts are EXACT on a cross-FORMAT arm and BANDED (`abs(d_accepted) <= 2`, `abs(d_proposed) <= k*2`) on a cross-QUANTIZATION one, with the arm chosen by `IsQuantizedGgufDraft` reading the draft file's ggml types (`GgmlTraits().block_elems > 1`) rather than by a flag. The band is derived, not picked: measured `d_accepted` is 0, 0, -1, so the bound is that maximum plus one quantum; and `d_proposed = -k * d_accepted` EXACTLY once the token streams match (confirmed at -1 / +16), so the proposed bound follows. **Mutation-proved non-vacuous**: rebuilt at band 0 the `Q4_K_M` arm is 15/17 exit 1 while the `BF16` arm stays 17/17 exit 0 on the exact branch. **AXIS B BROADENED from ONE prompt to THREE**, strict form green on all: "The capital of France is" IDENTICAL 14/160 (15/15), "Write a Python function that reverses a string:" IDENTICAL 24/64 (15/15), "Photosynthesis is the process by which" IDENTICAL 15/128 (9/9), all exit 0, ~6m30-6m52 and ~81 GiB peak RSS each. The second prompt REFINES the recorded acceptance claim: the safetensors-target arm is ALSO 24/64 there with the two containers' DFlash-ON streams IDENTICAL, so the GGUF target's lower acceptance is prompt-dependent (their spec-OFF streams diverge at index 4 on the first prompt, index 16 on the second) and not a standing penalty; the cause remains `QUANT-GGUF-NVFP4` being dequant-only, with the shared head excluded by a byte comparison. Gates 1-5 and 7 MET; gate 6 (speed) `PENDING` BY DESIGN and not owed - a DFlash-ON throughput A/B between the two target containers is not a fair comparison until a native NVFP4 GGUF GEMM exists. Evidence: [docs/BENCHMARKS.md](../docs/BENCHMARKS.md) top section, [parity-ledger.md](parity-ledger.md#L845) | [specs/gguf-dflash-draft.md](specs/gguf-dflash-draft.md) | `DONE` | `c62f2fa3` | +| `SPEC-MTP-GGUF` | MTP speculative decoding from a GGUF TARGET. Today `FromModelDir` refuses `mtp`+GGUF outright (`src/vllm/entrypoints/model_loader.cpp:717-723`) on the original spike's assumption that GGUF exports carry no `mtp.*` ([mtp-spec-decode.md](specs/mtp-spec-decode.md):979-980, "until we re-export GGUFs with the head"). That is stale: llama.cpp's Qwen3.5 converter DOES emit the head, under layer-indexed `nextn` naming, and our own `HfConfigFromGguf` ALREADY reads `nextn_predict_layers` (it just discards the value into the trunk layer count). Gap is a `TensorResolver` over `GgufFile` mapping `mtp.*` onto `blk.{L+i}.nextn.*` with dequant-to-bf16, one config field, and narrowing the rejection to `dflash`. `ngram`+GGUF already works and is untouched. Qwen3.5/3.6 only (the widened spec KV path serves no other arch). NO ABI change | T2 | llama.cpp (the producer contract; vLLM has no GGUF MTP path) `conversion/qwen.py:535-604` `_Qwen35MtpMixin` (the authoritative `mtp.*`->`nextn` remapper + `add_nextn_predict_layers`); `gguf-py/gguf/constants.py:129,910-917,1494-1501`; `gguf-py/gguf/tensor_mapping.py` `NEXTN_*` | **G1-G3 LANDED 2026-07-28.** `HfConfigFromGguf` republishes the head depth `src/vllm/model_executor/models/qwen3_5_gguf_weights.cpp:598` (`c.raw["mtp_num_hidden_layers"] = nextn`, previously read then discarded); the head loader `LoadQwen3_5MTPFromGguf` `src/vllm/model_executor/models/qwen3_5_gguf_weights.cpp:1521` (+ decl `include/vllm/model_executor/models/qwen3_5_gguf_weights.h:143`) reusing the TRUNK helpers `OwnNormMinus1`/`OwnMatmulWeight`/`OwnBf16`/`LoadAttnGguf`/`LoadMoeGguf` so the head inherits the GGUF (w+1) norm storage, quantization/residency routing and torch [N,K] shapes; `NumMtpLayers`/`UsesDedicatedEmbeddings` exported out of the anon namespace `include/vllm/model_executor/models/qwen3_5_mtp.h`; rejection narrowed to dflash + a head-less-GGUF check `src/vllm/entrypoints/model_loader.cpp` and the head attached in the GGUF branch; **G4 GREEN + `CPU-SPEC-DIVERGENCE` FIXED 2026-07-28**: root cause `src/vllm/model_executor/models/qwen3_5.cpp:3616` sized the GDN state gather/scatter row by `(Kw-1)` while the speculative persistent row is `(Kw-1)+num_spec`, so `GatherRows`/`ScatterRows` mis-strode the slot AND every channel past the first, corrupting post-prefill recurrent state. Fix = `CopyStateRowsStrided` (same TU) used by `GatherStateF32`/`ScatterStateF32` when `cache.shape[2] != work.shape[2]`; the contiguous helpers are kept when the widths agree, so every non-spec path is byte-identical by construction. CPU-only in effect (the fp16/bf16 arm routes through the `GdnStateGather`/`Scatter` ops, so CUDA was never exposed; no GPU result affected) | `tests/vllm/models/test_qwen3_5_gguf_mtp.cpp:109,146,156,184` **4 cases, and the split is the 2026-08-21 repair** ([#1454](https://github.com/mudler/vllm.cpp/issues/1454)): the file used to be the env-gated pair ALONE, each opening on a bare `return`, so with `VLLM_MTP_GGUF_MODEL` unset it reported `test cases: 2 \| 2 passed`, **`assertions: 0`**, `Status: SUCCESS!`, exit 0 - which is every CI run of this repository, the variable being set nowhere in `.github/workflows/`. The `18 assertions` this cell used to record was the LIVE count and was never once reached in CI. Now `:109` and `:146` are **HERMETIC** (KV-only synthetic GGUFs, no weight bytes, 18 assertions on any machine) and pin the arithmetic the old file only NAMED in a comment above `CHECK(c.num_hidden_layers > 0)`: `num_hidden_layers + mtp_num_hidden_layers == block_count` over 65/1, 25/1 and 28/3 - the third arm separating `- nextn` from `- 1` - plus the head-less arm, where the key is NOT published and `NumMtpLayers` answering 1 for an absent key is exactly why the invariant cannot be written with that helper alone. `:156` and `:184` stay env-gated on `VLLM_MTP_GGUF_MODEL` (so CI stays asset-free) and now SKIP LOUDLY with a `MESSAGE` naming the variable, as `tests/vllm/entrypoints/test_gguf_mmproj_reach.cpp` does; `:156` re-derives the same invariant from the file's OWN `block_count` kv. Unset: **4 cases / 18 assertions / `Status: SUCCESS!` / rc 0**. Live on `Qwen3.8-27B-Q4_K_M.gguf` (`block_count` 65, `nextn_predict_layers` 1): **4 cases / 38 assertions / `Status: SUCCESS!` / rc 0**. Mutation-proved on the production line `src/vllm/model_executor/models/qwen3_5_gguf_weights.cpp:889`, both compiling clean and both restored against a pre-taken sha256: `= block_count` (drop the subtraction) 3/4 cases, 9/18 red, exit 1; `= block_count - 1` (the wrong constant) 2/4 cases, 5/18 red, exit 1. The SAME mutations left the PREVIOUS file at 2/2 cases, 0 assertions, `SUCCESS!`, exit 0. Correctness of the production line is unchanged and was never in question (`1a4db5c3c`, `493327b4e`); this was a test defect. Live-arm content unchanged: depth reaches config.raw; fc is [H,2H] verbatim; 3 norms [H]; head block is full-attention. **RED-first BEHAVIOURAL** (reverting only the G1 line fails both cases 2/2). Trunk inertness: `test_gguf` 103, `test_gguf_qwen36_loader` 99, `test_gguf_keep_quant` 5958, `test_gguf_dequant` 215, `test_capi` 33/232 all unchanged; `tests/parity/test_qwen35_gguf_spec_decode.cpp:74,139` - spec-ON == spec-OFF token-exact with 13 proposed/11 accepted, plus an `ngram` regression guard (widens the cache, never runs the spec conv update) that was token-exact throughout and pinned the widening as innocent. Regression sweep all unchanged: ops_gdn 1825, gdn_metadata_builder 483, gdn_prefill_conv 28, gdn_spec_routing 12, gguf 103, gguf_qwen36_loader 99, gguf_keep_quant 5958, gguf_dequant 215, llm_engine 196, input_batch 163, runner 257, capi 232 **GPU CLOSE-OUT + DEVICE-DELTA ATTRIBUTION 2026-07-28 (`G5`-`G7`), ledger [parity-ledger.md#L800](parity-ledger.md#L800).** The GPU end-to-end gate re-run on a from-scratch RELEASE-TARGET build (`-DVLLM_CPP_CUDA_ARCHITECTURES=121a`, build dir DELETED first; arch VERIFIED by `build-cuda/CMakeFiles/vllm.dir/flags.make` `--generate-code=arch=compute_121a,code=[compute_121a,sm_121a]` and by `cuobjdump -lelf` 20 cubins ALL `sm_121a` zero sm_75, NOT by `CMakeCache.txt`, whose `CMAKE_CUDA_ARCHITECTURES:STRING=75` is the `enable_language(CUDA)` compiler-probe default shadowed by the normal variable at `CMakeLists.txt:186` - the prior wrong-arch conclusion was that decoy): dgx.casa GB10 under `flock $HOME/gpu.lock`, 35B A3B NVFP4 GGUF, **2/2 cases, 10/10 assertions, exit 0**, spec-ON token-identical to spec-OFF, 13 proposed / 11 accepted, 90.2 GiB peak RSS, 8m01s; re-run on the EXACT committed source **3/3 cases, 10/10 assertions, exit 0**, 7m25s, the new probe case SKIPping and adding zero assertions. **The CPU-vs-GPU token delta is a MEASURED near-tie, not a defect** (it was never this row's bar - spec-ON == spec-OFF WITHIN a device is): NEW double-gated spec-OFF-only probe `tests/parity/test_qwen35_gguf_spec_decode.cpp:217` (asset + `VLLM_MTP_GGUF_PROBE=1`, 20 alternatives per position, 484/484 assertions per arm, GPU then `CUDA_VISIBLE_DEVICES=` in one `flock` series) shows both arms picking `11751` at position 0 and forking at position 1 on a BIT-IDENTICAL prefix: GPU rank1 `13` -0.773180 over rank2 `11` -0.847055 (margin 0.0739 nats), CPU rank1 `11` -0.765499 over rank2 `13` -0.830374 (margin 0.0649 nats). Each device's pick is the other's rank 2, both ~7x inside the ratified 0.5-nat band, and the cross-device disagreement on the SAME token (0.057 and 0.082 nats) EXCEEDS the margin being decided, so rounding settles it; the 24 texts look unrelated only because positions 2+ cascade off that one coin flip. Margin sweep over all 24 positions: **GGUF GPU and GGUF CPU carry ZERO exact ties**, minimum margins 0.0482 and 0.0649 nats, and both arms reproduced their sequence across every run. **Gate 4 MET on the safetensors sibling of the same quantization run** (`FromModelDir` takes it unchanged): acceptance 12 proposed / 11 accepted vs the GGUF's 13 / 11. That arm, however, FAILS spec-ON == spec-OFF at concurrency 1 and does not reproduce its own spec-OFF sequence run to run, and the probe attributes both to THREE EXACT ties (positions 7, 10, 16, bit-identical logprobs) produced by its 1/16-grid quantized-GEMM logits - which EXONERATES the GGUF arm and opens a recorded, not-root-caused `SPEC-MTP` item on the safetensors NVFP4 path, not on this row. Gate 3 is NOT APPLICABLE twice over: no F16/F32 head-carrying export exists, and the only same-weights sibling is not token-stable against itself. **EVIDENCE RE-ANCHORED 2026-07-29 to a PRODUCTION-CONFIGURED build, because every GPU number above came from a build configured WITHOUT `-DVLLM_CPP_CUTLASS_DIR` and WITHOUT `-DVLLM_CPP_TRITON=ON`** (the defect `CLAIM-27B-GATE-RCA` proved, which runs the emulation fp4 GEMM + hand GDN kernels). Re-run from a clean `git archive` tree of `main` `3f34534d`, build proven correct three ways (configure log has ZERO `CUTLASS not found` and prints `CUTLASS found ... sm120a NVFP4 cutlass GEMM` + `FlashAttention-2 ... ENABLED for arch(es) [121a]` + the vendored `sm_121a` Triton-AOT lines with `MANIFEST hashes OK`; `cuobjdump -lelf` 40 cubins ALL `sm_121a`, zero `sm_75`; SACRED `test_qwen27_paged_engine` **235/235 exit 0**, and the build precondition proven to FIRE by recompiling only that TU without the two defines against the same `libvllm.a`, which throws and exits 1 with 0 assertions). **The row PASSES UNCHANGED:** `tests/parity/test_qwen35_gguf_spec_decode` **3/3 cases, 10/10 assertions, exit 0**, spec-ON token-identical to spec-OFF, **13 proposed / 11 accepted (identical to the recorded number)**, 90.26 GiB, 7m13.59s; loader gate 19 assertions on the Qwen3.5-2B and 18 on the 35B A3B, unchanged. **ONE recorded finding is RETRACTED by the re-measurement: the CPU-vs-GPU token delta was a BUILD artifact, not a device near-tie cascade.** On the production build both devices emit the SAME 24 tokens; the probe shows GPU rank1 `11` -0.763897 over rank2 `13` -0.824083 where the defective build had rank1 `13` -0.773180 over rank2 `11` -0.847055, while the CPU arm is bit-identical to the earlier measurement (CUTLASS and Triton are CUDA-only). Zero exact ties in either arm, min margins 0.060186 GPU / 0.064875 CPU, 484/484 assertions per arm. Evidence: [docs/BENCHMARKS.md](../docs/BENCHMARKS.md) top section, [parity-ledger.md](parity-ledger.md) | [specs/gguf-mtp-spec-decode.md](specs/gguf-mtp-spec-decode.md) | `DONE` | `edf91449` | +| `SPEC-DFLASH-GGUF` | DFlash speculative decoding from GGUF, two axes: (A) GGUF DRAFT + safetensors target, (B) GGUF target too. llama.cpp master carries a full `dflash` GGUF contract (arch string `dflash`, tensors `fc`/`enc.output_norm`/`output_norm`/`blk.N.*`, KVs `dflash.target_layers` + `dflash.target_hidden_size`); the arch is ABSENT from checkouts older than ~2026-07, so a stale tree reads as "no contract exists". The GGUF tensor set omits `token_embd`/`output` because the draft SHARES the target's embed+lm_head, which is exactly what `LoadDflashDraft` already does. Blockers are in the loader, not the model: `MakeDflashDraftConfig` reads `draft_dir/config.json` (a GGUF has none), `ResolveDflashDraftDir` probes for `config.json` so it cannot see a `.gguf`, and `LoadDflashDraft` is typed on `std::vector` for the shared bf16 head (the axis-B blocker). Axis A independently shippable. NO ABI change | T2 | llama.cpp `origin/master` @ 2026-07-28 (tag era `b10158`): `gguf-py/gguf/constants.py:547,1151,4350`; `gguf-py/gguf/tensor_mapping.py:1297-1305` (`ENC_OUTPUT_NORM`<-`model.hidden_norm`, `FC`<-`model.fc`); `conversion/qwen.py:351` (mask token via the standard tokenizer KV); `convert_hf_to_gguf.py --target-model-dir` | **GD1-GD7 LANDED 2026-07-28 (BOTH AXES COMPLETE and PROVEN end to end on GB10)**: `MakeDflashGgufConfig` + `LoadQwen3DFlashFromGguf` `src/vllm/model_executor/models/qwen3_dflash_gguf.cpp:88,227` (+ header), `IsDflashGgufDraft` + the `.gguf` branch in `ResolveDflashDraftDir`/`LoadDflashDraft` `src/vllm/entrypoints/model_loader.cpp:121,222`. Goes through the `TensorResolver` seam (unlike `SPEC-MTP-GGUF`) because dflash norms are RAW, so the existing `LoadQwen3DFlash` qkv/gate_up concatenation is reused unchanged. **`GD4` defect FIXED** (`model_loader.cpp:238-249`): the GGUF branch left `config.vocab_size` 0 - correct for `MakeDflashGgufConfig` (the DFLASH arch has no vocab KV and no `token_embd`) but fatal for the forward, which sizes the shared embedding view as `{config.vocab_size, H}`, so the first propose threw `cuda embedding: empty table (vocab 0)`. Now back-filled from the target's `embed_tokens` rows (the condition is on the VALUE, not the draft source, so it generalizes to a GGUF target). Load-level green had hidden it; only GENERATING found it. **GD5-GD7 = axis B**: `SharedHeadSource` `src/vllm/entrypoints/model_loader.cpp` re-expresses the shared bf16 `embed_tokens`+`lm_head` seam as a SOURCE and re-types `LoadDflashDraft`'s second parameter - THAT TYPE was the whole axis-B blocker - with the GGUF arm `LoadGgufSharedEmbedAndHeadBf16` `src/vllm/model_executor/models/qwen3_5_gguf_weights.cpp:1044` reusing the trunk loader's tied-embedding rule and sidecar-aware dequant instead of restating them; the shared-head load moved into ONE common tail so all four (draft format x target container) combinations run identical code; the `dflash` half of the GGUF-branch rejection `model_loader.cpp` is deleted (the `mtp` half untouched) and the draft load is wired into the GGUF branch | `tests/vllm/models/test_qwen3_dflash_gguf.cpp:36,84` 2 cases / 47 assertions against the REAL published Qwen3.6-27B DFlash draft (env-gated `VLLM_DFLASH_GGUF_MODEL`, CI asset-free): the +1 target-layer offset undone against the KV read back from the same file, block_size/mask_token present, vocab_size left 0, layer_types cover every block, fc `[H, H*num_taps]` with `nk` SET, qkv/gate_up row-concat shapes, embed/lm_head left EMPTY for the target. **RED-first BEHAVIOURAL** (dropping the `-1` fails the offset checks). **`GD4` e2e gate** `tests/parity/test_qwen27_dflash_spec_decode.cpp:343` (second case, draft source env-driven via `VLLM_DFLASH_DRAFT`/`_B`; asset-gated, CI-inert): on dgx GB10 sm_121a against the Qwen3.6-27B NVFP4 safetensors target, the Q4_K_M GGUF draft and the bf16 z-lab safetensors draft produce **token-for-token IDENTICAL** DFlash-ON continuations with **IDENTICAL** accepted/proposed (20/80 on a 24-token prompt, 42/96 on a 48-token prompt), spec-OFF self-reproducible 3/3 and 0 exact ties (min margin 0.197/0.400 nats). Regression: gguf_mtp 19, qwen35_gguf_spec_decode 10, gguf 103, gguf_qwen36_loader 99, gguf_keep_quant 5958, ops_gdn 1825, llm_engine 196, capi 232, runner 257 all unchanged. **`GD5` unit gate** `tests/vllm/test_gguf_qwen36_loader.cpp` 3 new synthetic-GGUF cases (6 cases / 286 assertions total, CPU and the dgx CUDA build): the untied head really comes from `output.weight` and not the embedding (distinct fill values), the tied fallback aliases it onto `token_embd`, the `nk` flags separate the gather table from the MatmulBT weight, a file with no `token_embd` is refused. 3-mutant battery, 3 caught (`nk` flipped, head forced to the embedding, tied forced false). **`GD7` e2e gate** `tests/parity/test_qwen27_dflash_spec_decode.cpp` third case (targets env-driven via `VLLM_DFLASH_TARGET_B`; asset-gated, CI-inert): on dgx GB10 sm_121a the Qwen3.6-27B NVFP4 **GGUF** target + `Q4_K_M` GGUF draft loads, takes the shared head from the GGUF, generates, and its DFlash-ON continuation is **token-for-token IDENTICAL to that same target's spec-OFF** (24/24, the STRICT form) with acceptance ALIVE at 14/160; 1 case / 15 assertions, exit 0. **The spike's highest risk is EMPTY on this asset, proven not assumed**: the 27B NVFP4 GGUF stores `token_embd`/`output` as ggml BF16, byte-identical to the safetensors sibling (2,542,796,800 bytes each, ZERO differing), so B1's shared-head read is verbatim, not a dequant. Acceptance IS lower than the safetensors-target arm and is NOT chargeable to the head: the two containers diverge at index 4 with NO speculation, because `QUANT-GGUF-NVFP4` is dequant-only so the GGUF target computes in bf16 while the safetensors target runs the true W4A4 kernels. **RE-MEASURED 2026-07-29 on a PRODUCTION-CONFIGURED build (`CLAIM-GGUF-SPEC-REVERIFY`), because every GD4/GD7 GPU number above came from a build configured WITHOUT `-DVLLM_CPP_CUTLASS_DIR` and WITHOUT `-DVLLM_CPP_TRITON=ON`.** Build proven correct three ways (see the `SPEC-MTP-GGUF` row; SACRED 27B **235/235**, `cuobjdump` 40 cubins all `sm_121a`). **AXIS B HOLDS EXACTLY**: `test_qwen27_dflash_spec_decode -tc="dflash axis-B*"` **15/15 assertions, exit 0**, GGUF-target DFlash-ON token-identical to that target's own spec-OFF 24/24, acceptance **14/160 unchanged**, cross-target spec-OFF divergence still at index 4, 81.01 GiB peak RSS, 6m53.08s. **AXIS A WAS RED ON THE 48-TOKEN PROMPT (reproducibly, 3 of 3 runs) AND IS NOW CLOSED.** The RED was real: cross-format TOKEN identity held on both prompts, but the exact accept-count half of bar (a) failed (`arm_a.proposed == arm_b.proposed` / `arm_a.accepted == arm_b.accepted`) because the Q4_K_M draft measured **46/112** against the bf16 z-lab draft's **47/96** (one extra 16-wide propose block, one fewer acceptance, zero token difference), 15/17, exit 1; the 24-token prompt stayed green at 17/17 with both drafts at 15/144. **`GD9` 2026-07-29 root-caused it IN WEIGHT SPACE as ordinary `Q4_K_M` cost, category (a), not a defect in our GGUF draft path - and the bar's own premise ("Same weights, two containers") was false for the asset it was pointed at.** The publishing repo also carries an UNQUANTIZED `BF16` GGUF (3,471,497,440 B) beside `Q8_0`/`Q6_K`/`Q5_K`/`Q4_K_M`, which the spec had recorded as nonexistent; that retired the `NOT APPLICABLE` on gate 2. CPU gate `tests/vllm/models/test_qwen3_dflash_gguf.cpp` third case (asset-gated `VLLM_DFLASH_GGUF_BF16_MODEL` + `VLLM_DFLASH_ST_DIR`): `LoadQwen3DFlashFromGguf(BF16)` is **BYTE-IDENTICAL to `LoadQwen3DFlash(z-lab shards)` on all 58 tensors, 302/302 assertions, exit 0**, and FUNCTIONALLY RED against the `Q4_K_M` file (21/302 red, exactly the 21 quantized matmul tensors), so not a vacuous pass. Supporting: our `DequantGgufRowToBf16` is bit-equal to `gguf-py`'s `gguf.quants.dequantize` on the real `fc.weight` (Q4_K), `blk.0.attn_q.weight` (Q4_K) and `blk.2.ffn_down.weight` (Q6_K), zero differing bf16 values; the ladder's mean relative weight error is monotone and uniform with NO outlier tensor (BF16 0, Q8_0 5.6e-3, Q6_K 1.85e-2, Q5_K 3.85e-2, Q4_K_M 7.6e-2); the only numeric config delta is `rms_norm_eps` at 2.5e-9 relative. Also landed: an off-by-default `VT_SPEC_TRACE=1` per-block propose/accept trace in `GPUModelRunner::sample_tokens_with_rejection` (`src/vllm/v1/worker/gpu/runner.cpp`). **`GD10` 2026-07-29 CONFIRMED IT END TO END ON GB10 and closed gates 3 and 5.** Build proven production-configured three ways (configure log 0 `CUTLASS not found`; `cuobjdump -lelf` 40 cubins ALL `sm_121a` zero `sm_75` on both binaries; SACRED `test_qwen27_paged_engine` **235/235, exit 0**, 31.34s, 23.67 GiB). The **`BF16` GGUF draft reads EXACTLY 47/96**, the safetensors draft's own number, at 48 tokens on the discriminating prompt - reproduced 2 of 2 - plus 27/64 = 27/64 at 24 tokens and 15/144 = 15/144 on the second prompt, tokens IDENTICAL throughout, 17/17 exit 0 each time; the `Q4_K_M` arm reads 46/112 on the SAME binary in the SAME `flock` series. Restoring only the draft's numeric precision restores the count, so quantization is the whole cause and nothing structural survives. Bar (a) is consequently SPLIT rather than relaxed (`tests/parity/test_qwen27_dflash_spec_decode.cpp`): tokens stay EXACT unconditionally; accept counts are EXACT on a cross-FORMAT arm and BANDED (`abs(d_accepted) <= 2`, `abs(d_proposed) <= k*2`) on a cross-QUANTIZATION one, with the arm chosen by `IsQuantizedGgufDraft` reading the draft file's ggml types (`GgmlTraits().block_elems > 1`) rather than by a flag. The band is derived, not picked: measured `d_accepted` is 0, 0, -1, so the bound is that maximum plus one quantum; and `d_proposed = -k * d_accepted` EXACTLY once the token streams match (confirmed at -1 / +16), so the proposed bound follows. **Mutation-proved non-vacuous**: rebuilt at band 0 the `Q4_K_M` arm is 15/17 exit 1 while the `BF16` arm stays 17/17 exit 0 on the exact branch. **AXIS B BROADENED from ONE prompt to THREE**, strict form green on all: "The capital of France is" IDENTICAL 14/160 (15/15), "Write a Python function that reverses a string:" IDENTICAL 24/64 (15/15), "Photosynthesis is the process by which" IDENTICAL 15/128 (9/9), all exit 0, ~6m30-6m52 and ~81 GiB peak RSS each. The second prompt REFINES the recorded acceptance claim: the safetensors-target arm is ALSO 24/64 there with the two containers' DFlash-ON streams IDENTICAL, so the GGUF target's lower acceptance is prompt-dependent (their spec-OFF streams diverge at index 4 on the first prompt, index 16 on the second) and not a standing penalty; the cause remains `QUANT-GGUF-NVFP4` being dequant-only, with the shared head excluded by a byte comparison. Gates 1-5 and 7 MET; gate 6 (speed) `PENDING` BY DESIGN and not owed - a DFlash-ON throughput A/B between the two target containers is not a fair comparison until a native NVFP4 GGUF GEMM exists. Evidence: [docs/BENCHMARKS.md](../docs/BENCHMARKS.md) top section, [parity-ledger.md](parity-ledger.md#L845) | [specs/gguf-dflash-draft.md](specs/gguf-dflash-draft.md) | `DONE` | `c62f2fa3` | | `SPEC-REJECTION` | Rejection sampler. **I3 verify half LANDED (2026-07-24)**: per-request logits EXPANSION to `1 + k_i` rows (`StepInputs::cu_num_logits` / `num_draft_tokens_per_req` / expanded `logits_indices`) plus the GREEDY rejection sampler — accept a draft iff it equals the target argmax at its own position, emit the target argmax on the FIRST mismatch and stop, emit the bonus argmax when all `k_i` accept, `num_sampled = accepted + 1`, `num_rejected = k_i - accepted` (feeds I2's `num_computed_tokens` rollback and `InputBatch::num_accepted_tokens`). One additive vt op (`kGreedyRejectionSample`) with a CPU reference and a CUDA two-phase mirror of upstream's row-argmax + one-thread-per-request accept walk. DEFAULT-OFF and INERT: with no `SpeculativeConfig` no drafts are ever scheduled, `cu_num_logits` is `arange(num_reqs+1)`, `logits_indices` is the pre-change array and the runner never enters the rejection branch. STOCHASTIC/Gumbel, block verification, `apply_sampling_params` over the expanded batch, and the spec grammar bitmask stay DEFERRED (M-mtp-3). **I5b DRAFTER PREFILL INPUT-PREP LANDED (2026-07-24, `CLAIM-SPEC-MTP-I5B`)**: the draft-token input splice this row's I3 note deferred to I5 — `vllm::v1::prepare_prefill_inputs` + its `SpecPrefillInputs` output struct shift each request's `input_ids` left one within its query span, splice the just-sampled next token (`num_sampled>0 ? last_sampled[idx_mapping[r]] : next_prefill_tokens[...]`) into the freed slot, `query_len -= num_rejected`, and emit last-token index / query_start_loc / seq_lens + CG padding (mirror `speculator.py:469-588`, k=1 early-exit :236-238). A HOST routine in a NEW spec_decode-tree TU (no new CUDA kernel; mirrors the DEVICE-NEUTRAL `prepare_inputs`/`combine_sampled_and_draft_tokens` family — the DGX runner leaf ports the loop to the Triton kernel at I5d), unit-gated `test_prepare_prefill_inputs` 7 cases / 27 assertions RED-first, DEFAULT-OFF INERT (nothing calls it until I5d), additive by construction. Row stays `ACTIVE` — the e2e greedy token gate (M-mtp-1) is owed before `DONE` | T1 | `vllm/v1/worker/gpu/spec_decode/rejection_sampler.py:43,101-160`; `rejection_sampler_utils.py:524,564-585,628,828-841,846-849,863-1125`; `vllm/v1/worker/gpu/model_runner.py:866-898,1065-1077`; `vllm/v1/worker/gpu/input_batch.py:303-397,408-453`; **I5b** `vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py:469-588,236-238` | `include/vllm/v1/spec_decode/rejection_sampler.h`; `src/vllm/v1/spec_decode/rejection_sampler.cpp`; `include/vt/ops.h` (`kGreedyRejectionSample`, `vt::GreedyRejectionSample`); `src/vt/cpu/cpu_sample.cpp` (CPU reference); `src/vt/cuda/cuda_sample.cu` (`RejectionRowArgmaxKernel` + `GreedyRejectAcceptKernel`); `src/vt/ops.cpp`; `include/vllm/v1/worker/gpu/prepare_inputs.h` + `src/vllm/v1/worker/gpu/prepare_inputs.cpp` (the expansion); `include/vllm/v1/worker/gpu/runner.h` + `src/vllm/v1/worker/gpu/runner.cpp` (`step_num_logits`, `sample_tokens_with_rejection`); **I5b** `include/vllm/v1/worker/gpu/spec_decode/autoregressive/prepare_prefill_inputs.h` + `src/vllm/v1/worker/gpu/spec_decode/autoregressive/prepare_prefill_inputs.cpp` — anchor `include/vllm/v1/spec_decode/rejection_sampler.h:96` | `tests/vllm/v1/spec_decode/test_rejection_sampler.cpp`; `tests/vllm/v1/worker/test_prepare_inputs.cpp` (expansion + no-draft byte-identity); `tests/vt/test_cuda_ops.cpp` (CUDA==CPU bit-exact at vocab 248320); **I5b** `tests/vllm/v1/spec_decode/test_prepare_prefill_inputs.cpp` (7 cases / 27 assertions, RED-first) — anchor `tests/vllm/v1/spec_decode/test_rejection_sampler.cpp:128` | [mtp-spec-decode.md §2.4,§5](specs/mtp-spec-decode.md) | `ACTIVE` | `CLAIM-SPEC-REJECTION-I3`, `CLAIM-SPEC-MTP-I5B` | | `SPEC-GDN-SEGMENTS` | GDN speculative metadata and slot-snapshot rollback. **I4 LANDED (2026-07-24):** the spec/non-spec metadata split with decode→prefill reclassification (the #34845 case), the `T>1`/`IS_SPEC` GDN recurrence with per-timestep state snapshots, the conv sliding window advancing by the ACCEPTED count, and the k+1 state-slot allocation. DEFAULT-OFF and INERT (`num_spec==0` ⇒ `num_spec_decodes==0`, no shipped kernel branched — both spec kernels are NEW op ids). ROLLBACK PROVEN bit-exact: for every rejection point j the surviving SSM state and conv window are memcmp-identical to running only the accepted prefix through the shipped `vt::GdnDecode`/`CausalConv1dUpdate`, at the real 27B (Hv=48) and 35B (Hv=32) GDN dims on CPU and CUDA. MEASURED state cost: one f32 SSM slot = Hv·Dv·Dk·4B ⇒ 144 MiB/req (27B, 48 layers) / 60 MiB/req (35B, 30 layers) per extra slot; k=1 doubles the GDN SSM state. **I5a GDN LAYER ROUTING WIRED (2026-07-24, `CLAIM-SPEC-MTP-I5A`):** `GdnBlockPaged`'s `num_spec_decodes>0` branch now routes a PURE-spec batch through `vt::CausalConv1dSpecUpdate` + `vt::GdnSpecDecode` (mirror `qwen_gdn_linear_attn.py:1344-1357,1455-1475`), and the runner per-step upload (`StepDevInputs`/`BuildStepDevInputs` + the two decode-graph `Refresh` copies) now carries I4's six spec device tensors, gated by the extended `ValidateGdnAttentionMetadata` spec contract. DEFAULT-OFF INERT (`num_spec_decodes==0` ⇒ stub uploads + the identical non-spec branch). BIT-EXACT vs the I4 ops applied as a token-sequential decode chain, at the real 27B/35B GDN dims, via `GdnBlockPagedForTest` (`tests/vllm/models/test_qwen3_5_gdn_spec_routing.cpp`, CPU bit-exact + CUDA on-device); RED-first by a reverted stub (spec recurrence zeroed ⇒ 4/8 fail, maxΔ 1.3-1.6). MIXED spec+non-spec batch refused loudly — lands with I5d's runner loop. Row advances to `ACTIVE`: the M-mtp-1 e2e greedy token gate (verify/propose runner wiring) is owed before `DONE`, and `SPEC-MTP` STAYS `GATING` | T1 | `vllm/v1/attention/backends/gdn_attn.py:189-326,413-462`; `fla/ops/fused_sigmoid_gating.py:66-72,103-116,156-166`; `mamba/ops/causal_conv1d.py:818-1067,1181-1184`; `qwen_gdn_linear_attn.py:1329-1576`; `mamba_utils.py:213-234`; `mamba/abstract.py:55-59` | `include/vllm/v1/attention/backends/gdn_attn.h`; `src/vllm/v1/attention/backends/gdn_attn.cpp`; `include/vt/ops.h` (`kGdnSpecDecode`, `kCausalConv1dSpecUpdate`); `src/vt/ops.cpp`; `src/vt/cpu/cpu_ops.cpp`; `src/vt/cuda/cuda_gdn.cu`; `src/vllm/model_executor/models/qwen3_5_common.{h,cpp}` (`MakeQwen3_5KVCacheSpec`); **I5a:** `src/vllm/model_executor/models/qwen3_5.cpp` (`GdnBlockPaged` spec branch, `StepDevInputs`/`BuildStepDevInputs`, `ValidateGdnAttentionMetadata`), `src/vllm/model_executor/models/qwen3_5_internal.h` (`GdnBlockPagedForTest`) | `tests/vllm/v1/attention/test_gdn_metadata_builder.cpp` (20 cases / 483 assertions incl. the full upstream `GDN_BUILD_TEST_CASES` + default-off byte-identity); `tests/vt/test_ops_gdn.cpp` (reject-at-every-j rollback, CPU + CUDA, real dims); `tests/vllm/models/test_model_registry.cpp` (k+1 slot / widened-conv sizing + `num_spec==0` identity); **I5a** `tests/vllm/models/test_qwen3_5_gdn_spec_routing.cpp` (spec-routing bit-exact, RED-first) — anchor `tests/vllm/v1/attention/test_gdn_metadata_builder.cpp:83` | [mtp-spec-decode.md §3,§5](specs/mtp-spec-decode.md) | `ACTIVE` | `CLAIM-SPEC-GDN-I4`, `CLAIM-SPEC-MTP-I5A` | | `SPEC-DFLASH` | Block-diffusion drafter. **READINESS RE-ASSESSED 2026-07-25 (`CLAIM-SPEC-DFLASH-READINESS`, design-only, DONE) against the LANDED MTP machinery (`SPEC-MTP` I1..I7).** Verdict **GREEN, dispatch-ready, NO hardware/oracle/download blocker** (spec [§0](specs/dflash-spec-decode.md)). Refreshed reuse-vs-new map: DFlash gets FREE from landed MTP — the frozen spec-metadata ABI, the greedy rejection sampler (k-general, I3 tested k∈{1,3}), the GDN spec slot path + rollback + mixed spec/non-spec batch (`GdnBlockPagedMixedSpec`/`IndexSelect`/`IndexCopy`, general `num_spec`), the widened-cache-aware conv ops (I5e), the draft-KV layer pattern (`fa_draft`), the I5d/I7 runner verify/propose loop, and **`num_lookahead_tokens=k+1` ALREADY coded** (`speculative.h:91-108` `use_dflash()`); EXTENDS the single I5d-pre `hidden_tap` seam to multi-tap `[T,H×taps]`; builds NEW the `qwen3_dflash` drafter, the project's FIRST non-causal in-block attention primitive, context-KV precompute, `prepare_dflash_inputs`, and the uniform-1+k FULL CG. **k>1 verdict:** the landed rejection + GDN machinery is MECHANICALLY k-general (no `k==1` hardwiring) — DFlash's k=15 blocks need NO mechanism extension, only exercise/validation at scale (D4) + the k+1-slot memory measurement (~2.3 GiB/req 27B GDN state at block-16, the #1 risk, §5). **Checkpoint-fit:** both z-lab drafts EXIST on HF (27B 1.73 GB / 35B 368 MB bf16, DFlashDraftModel) and FIT the 119 GiB pool trivially (drafts NOT yet on dgx — D0 downloads ≤1.73 GB); the active dgx oracle `vllm-oracle-v0.25.0-stage` CONSTRUCTS DFlash (registry `DFlashDraftModel→qwen3_dflash`, speculator dir present) — soft D0 risk = confirm it SERVES DFlash+NVFP4 on sm_121 (non-causal backend; community `AEON-7/vllm-dflash` container proves the combination runs on GB10). W-plan D0-D6 in the spec. **D0+D1 LANDED 2026-07-26 (`CLAIM-DFLASH-D0D1`) on the ADVANCED pin `555967922`/vLLM 0.26.0.dev0 — `SPEC-DFLASH` → `ACTIVE`.** D0 UNBLOCKED (vllm#40898 resolved under `VLLM_USE_V2_MODEL_RUNNER=1`): the mixed-attn z-lab 27B draft CONSTRUCTS + the drafter is ALIVE (acceptance 2.21/8.80/4.75/4.57 > 1, `num_spec=16`, flashinfer-native fp8-KV, goldens committed); gate FORM measured STRICT MODE-MATCHED (vLLM-ON run-deterministic K>=3 but != vLLM-OFF — the k=16 block verify diverges at bf16 near-ties, so NOT the MTP three-way identity). D1 `DF-AUX-TAPS` DONE: `Qwen3_5AuxTaps` + `ModelForwardInput::aux_tap` route to `Qwen3_5{,Dense}Model::ForwardDeviceMultiTap` capturing `(hidden+res)` at `target_layer_ids` into `[T,H×taps]` (eagle3 `_maybe_add_hidden_state`, aux key L+1); config-gated byte-identical off. Unit gate 598 assertions (independent truncated-model reference, RED-first reversed-concat 384 fail); CUDA 697/697 + compute-sanitizer 0; INERTNESS PROVEN — 27B MTP e2e 9/9 + 27B text SACRED 235/235 byte-identical on the new oracle. **D2 `DF-DRAFT-MODEL` CODE LANDED + CPU-GATED 2026-07-26 (`CLAIM-DFLASH-D2`, kernel row `KERNEL-ATTN-DFLASH-BLOCK`):** the `qwen3_dflash` draft model (plain 5-layer Qwen3-dense reusing `dense_attn_block.h` ops), the project's FIRST non-causal / bidirectional attention primitive `vt::DFlashBlockAttention` (a SEPARATE op — causal `kAttention`/`kPagedAttention` byte-identical), the fc aux-combine, mask-embed, per-layer SWA/full resolution, and the z-lab loader. CPU gate GREEN (op 12/12 incl. RED non-causal; model forward 95/95 incl. RED full-layer-causal-flip + block isolation + fc RED); existing causal `test_ops_attention` 9/9 + `test_qwen3_forward` 1028 UNCHANGED. **D2 GPU PROMOTION GREEN on dgx (`CLAIM-DFLASH-D2`):** CUDA `-Werror` clean, CUDA==CPU 198412/198412 + compute-sanitizer 0, draft-forward parity vs the REAL vLLM draft (fc rel-L2 0.46%, hidden ≤1.3%, 11 STRICT + 5 near-tie ids), 27B SACRED 235/235 + MTP 9/9 byte-identical — **D2 DONE.** **D3 `DF-DRAFT-KV-PREP` DONE 2026-07-26 (`CLAIM-DFLASH-D3`):** `PrecomputeContextKV` + `PrepareDflashInputs` + `ForwardBlockLogitsWithContext` (reuse the UNCHANGED D2 kernel via [context;block]); GPU numeric-parity `test_qwen3_dflash_kvprep_parity` 61/61 (prepare INTEGER bit-exact vs vLLM's Triton kernel, context-KV K/V rel-L2 0.31%/0.26%, 13 STRICT + 3 near-tie = 16/16), CPU 114/114 RED-proven, inertness 235/235 + 9/9 + D2 37/37 byte-identical. **D4 `DF-ENGINE-INTEGRATION` propose brick + `dflash` config-select CODE LANDED + CPU-GATED 2026-07-26 (`CLAIM-DFLASH-D4D5`):** `DflashProposeBlock`/`SampleDflashBlockDrafts` (the non-autoregressive whole-block propose composing D3 `ForwardBlockLogitsWithContext` + greedy per-mask argmax, anchor not sampled, `dflash/speculator.py:300-413`) + `ParseSpeculativeConfigJson`/`ResolveDflash` accept `method:"dflash"`. CPU gate `test_dflash_propose` 5/19 GREEN (RED-first anchor-read fails 4/5; brick composes forward+sampler; empty-ctx degenerates to D2; config lookahead k+1). Additive + config-gated ⇒ MTP + non-spec byte-identical BY CONSTRUCTION (`git diff --stat` = new speculator TU + config accept-list + CMake + test, NO runner/model/loader/scheduler edit). **D5 `DF-ENGINE-INTEGRATION` runner-loop LANDED + e2e RUNS on dgx 2026-07-26 (`CLAIM-DFLASH-D5`):** full verify/propose loop wired — loader loads the SEPARATE z-lab draft (`LoadDflashDraft`, host bf16 + target-SHARED bf16 embed/lm_head) via a `--speculative-config` `model` key + `ResolveSpecConfig` dflash branch + `runner.set_dflash_draft`; the verify forward captures the D1 multi-tap (`aux_tap`→`ForwardDeviceMultiTap`) instead of the MTP single tap; `propose_drafts_dflash` ACCUMULATES the per-request combined-feature context (`CombineAuxFeatures(aux_tap)`) across steps and honors the `num_rejected` rollback by appending only the `(T_req−num_rejected)` accepted-prefix features, then runs `DflashProposeBlock` (k=16 GDN-spec exercised first time). **e2e (`test_qwen27_dflash_spec_decode`, 4 prompts×32 tok, our-DFlash-ON vs the committed vLLM-DFlash-ON golden): 2/4 STRICT token-exact (fibonacci, three-laws) + acceptance ~ vLLM on ALL 4 (accepted 19/39/29/25 vs golden 17/39/30/25, deltas +2/0/−1/0 — the MANDATORY dead-drafter-trap condition MET).** The 2 divergences (France tok11 `2972`↔`11751`, 17*23 tok12 `567`↔`488`) are SINGLE bf16 near-tie flips (17*23 RE-CONVERGES after one token = proven near-tie; France cascades from one flip) — the ratified near-tie ROOT the D0 gate-form anticipated, rooted in the D3-documented inline bf16 context-KV recompute envelope (~0.3-1.3% rel-L2), NOT a wiring bug (proven by the 2 exact prompts + near-exact acceptance + a non-trivial shared prefix). Inertness GREEN on this build: SACRED `test_qwen27_paged_engine` 235/235 + MTP `test_qwen27_spec_decode` 9/9 byte-identical; CUDA `-Werror` clean; NO new CUDA kernel (host orchestration reusing D1/D2/D3-sanitized ops). **NOT a clean strict-4/4 pass; STRICT 4/4 token-identity + the speed A/B = D6 (the persistent paged draft-KV bit-matching vLLM's fused context-KV projections + the uniform-1+k FULL CG).** Row STAYS `ACTIVE` (correctness at the ratified near-tie envelope; D6 remains) **D6 2026-07-27 (`CLAIM-DFLASH-D6`) — c1 SPEED A/B DONE + STRICT-irreducibility RCA + CG feasibility (records-only, NO source code):** (1) **c1 speed A/B** (`examples/vllm-bench` at `361189a7`, 8 prose+code prompts×256 tok greedy c1, 2 reps): our DFlash-ON = **2.50x TPOT (40.4 vs 101.2 ms) / 2.48x output-tput (24.4 vs 9.86 tok/s)** over our OFF, acceptance 0.22 (3.56/16), rep-stable <1.5%; `benchmark_binding=true`. vs vLLM-DFlash-ON graphed (same workload): vLLM-DFlash-ON graphed = 28.5 tok/s / 35.1 ms TPOT / acceptance_len 4.30 (same 8 prompts, `VLLM_USE_V2_MODEL_RUNNER=1`, mm-off, gpu_util 0.30), so OURS IS ~14% BELOW vLLM-DFlash-ON on output throughput (24.4 vs 28.5 tok/s) - both ~on-par at spec-OFF (9.86 vs 9.83 tok/s), but vLLM extracts a larger DFlash speedup (2.90x vs our 2.47x) because its draft step is fully device-resident + CUDA-graphed (ours host-orchestrates 13 downloads/step) + slightly higher acceptance (~4.3 vs ~3.6 draft tokens/step). The DONE speed bar (ours >= vLLM) is NOT met; closing it = the device-resident draft rewrite + FULL CG (D6 part 2). (2) **STRICT-4/4 proven bf16-IRREDUCIBLE** — the draft KV cache is bf16 not fp8 (`torch_utils.py:398` `auto`→model dtype; the D0 "fp8-KV" was the backend name, not the KV storage dtype), the D3 golden already compares pre-storage bf16 (residual K 0.31%/V 0.26% = sub-ULP kernel noise), and a fused multi-layer KV GEMM is per-element invariant to our per-layer GEMMs ⇒ bit-exact needs vLLM's exact kernels ⇒ the ratified near-tie gate is the FINAL correctness form (no fused-KV code landed). (3) **FULL CG BLOCKED** on a device-resident draft-path rewrite (the D5 path does 13 device→host downloads/step + host `[context;block]` interleaving) — the remaining throughput-parity increment (the perf form of persistent-paged-KV + the graph). Inertness by construction (the gated binary is the D5 binary; SACRED 235/235 + MTP 9/9 stand). Evidence tool `scripts/spec/vllm_dflash_timing.py`. **D7 2026-07-27 (`CLAIM-DFLASH-D7`) — within-step draft forward made DEVICE-RESIDENT (source-owning): `PrecomputeContextKVDevice` keeps per-layer K/V on device; `ForwardBlockLogitsWithContext` builds [context;block] with `vt::IndexCopy`/`IndexSelect` (removes ~30 D→H `Download`s/step). BIT-IDENTICAL (identity bf16↔f32 round-trips replaced) — e2e `test_qwen27_dflash_spec_decode` 27/27 SAME tokens (2/4 STRICT + 2/4 near-tie, acceptance 19/39/29/25), SACRED 235/235 + MTP 9/9, CUDA `-Werror` clean, compute-sanitizer 0 (198412). But the direct old-vs-new A/B = +2.0% output-tput (IN-NOISE) ⇒ D6's "downloads = the ~14% gap" REFUTED by measurement; ours 19.68 tok/s STILL ~33% BELOW vLLM-DFlash-ON 29.2 tok/s (reconstructed 8-prompt set, more prose-heavy); OFF parity our 9.97 ≥ vLLM 9.66. Residual re-attributed: acceptance (ours 2.49 vs vLLM ~3.13 accepted draft-tok/step, bf16-irreducible) + per-step context-KV RECOMPUTE (O(context²), needs the cross-step persistent paged draft-KV store) + eager-vs-graphed. SPEED BAR NOT met; SPEC-DFLASH stays `ACTIVE`; next = persistent paged draft-KV store → then FULL CG. **D9 2026-07-27 (`CLAIM-DFLASH-D9`) — PERSISTENT PAGED DRAFT-KV LANDED (bit-identical, +22.7% throughput, 0.69×→0.917×); D8 acceptance-ceiling REFUTED; residual = FULL CG ONLY:** `qwen3_dflash.cpp` `AppendContextKVHost` (project ONLY newly-accepted rows → per-layer bf16 K/V, append to `PrecomputedContextKV`) + `ForwardBlockLogitsWithPrecomputedKV` (upload the persistent store, NO re-projection) share the core `ForwardWithCtxKVDev` with the old recompute; `runner.cpp::propose_drafts_dflash` swaps the O(context²) per-step recompute (`dflash_ctx_feats_`) for an append-only per-request `dflash_kv_store_` (rollback=don't-append). NO new CUDA kernel; config-gated. BIT-IDENTICAL: CPU `test_dflash_propose` two new D9 cases = exact float equality vs full recompute; GPU e2e `test_qwen27_dflash_spec_decode` **27/27 SAME tokens** (acceptance 19/39/29/25, same divergences France@11/17×23@12); SACRED 235/235 + MTP 9/9 byte-identical; CUDA `-Werror` clean. **A/B (c1, 8 prose+code×256 tok input-len 512, 2 reps <0.1%, `benchmark_binding=true`):** ours-ON **25.75 tok/s** (was D8 20.99, +22.7%) / 38.40 ms TPOT / acc **3.68/step** vs vLLM-ON graphed **28.09** / 35.60 / acc 3.31 = **0.917×** (~8% below, was 0.69×). **Part 1 same-trajectory:** on the 2 token-identical-trajectory prompts ours per-step acceptance == vLLM's EXACTLY (fibonacci 7.80/7.80, three-laws 3.571/3.571, ratio 1.00) AND on the A/B ours acceptance (3.68) is HIGHER than vLLM's (3.31) ⇒ D8's 0.80–0.85× "bf16 acceptance ceiling" is a trajectory-divergence CONFOUND, REFUTED. Residual (~8%) = eager-vs-graphed ONLY (ours ON/OFF 2.60× vs vLLM 2.91×, OFF at parity, recompute eliminated, acceptance higher) — NOT an irreducible ceiling; the FULL uniform-(1+k) CG (device paged-KV store + paged attn, new-CUDA multi-file) is the SOLE un-landed increment. SPEC-DFLASH stays `ACTIVE` (speed not yet ≥ vLLM; residual isolated to FULL CG). **D12 2026-07-27 (`CLAIM-DFLASH-D12`) — A-wire + Part B LANDED + GPU-gated; Part C (capture) remaining; 0.917×:** A-wire makes the D11 Part-A device store the PRODUCTION path (`runner.{h,cpp}` `dflash_kv_store_`→`shared_ptr`, `MakeDeviceKVStore`/`AppendContextKVDevice`/`ForwardBlockLogitsWithDeviceKV`; GPU-gated e2e `test_qwen27_dflash_spec_decode` 27/27 all-exact acceptance 19/39/29/25 + SACRED 235/235 + MTP 9/9 byte-identical, `-Werror` clean). Part B adds `vt::DFlashPagedBlockAttention` (`OpId::kDFlashPagedBlockAttention`), the capture-safe paged kernel with EVERY metadata input a persistent DEVICE tensor and NO function-local host `cu_seqlens` upload (fixes the `cuda_ops.cu:1277-1280` capture-UAF class), gated CPU==CUDA + cross-check vs materialized `DFlashBlockAttention` `test_ops_dflash_paged_block_attn` 795648/795648 + compute-sanitizer 0. Speed 0.917× (A-wire eager + Part B not yet wired into the forward); `benchmark_binding=false`. Part C (static-shape capture + device mask-scatter + `BeginCapture`/replay + the ≥vLLM c1 A/B) is the SOLE remaining piece; if ours-ON-graphed ≥ vLLM-ON → SPEC-DFLASH DONE. Stays `ACTIVE`. **D13 2026-07-27 (`CLAIM-DFLASH-D13`) — Part C LANDED + GPU-GATED; capture-correctness PROVEN; c1 throughput NEAR-PARITY (ours 0.978x, ~2% below vLLM); gap CLOSED 0.917x→0.978x; STAYS `ACTIVE` (≥vLLM bar not yet met):** single-file additive change (`qwen3_dflash.cpp` +368/-58). (C.1) `DflashDeviceKVStore` → fixed-capacity PAGED cache (per-layer pool `[max_pages,16,Hkv,Dh]` + identity `block_table` + `seq_lens`; append = `vt::IndexCopy` scatter at slot==abs-pos, bit-identical to the D9/D11 store). (C.2) `ForwardPagedBody` runs the (1+k) block through the D12 `vt::DFlashPagedBlockAttention` reading the paged store (no `[context;block]` materialization, no function-local host uploads); runner P==1 propose routes through it, P>1 bit-identical materialized fallback. (C.3) per-request CUDA GRAPH over the paged draft step (warm-in-step repopulates the shared pool free-list right before `BeginCapture` — the fix for a `cudaMalloc`-in-capture `Get` miss from the intervening 27B target forward — then `BeginCapture → ForwardPagedBody → EndCaptureGraph`, replay with growing context entering only via in-place `seq_lens`). **Capture-correctness (MANDATORY): `test_qwen27_dflash_spec_decode` 27/27 with the graph (VT_DFLASH_GRAPH=1) BIT-IDENTICAL to eager (=0)** — same divergence tokens (France@11 got[…2972…], 17×23@12 got[…567…]), same acceptance 19/39/29/25 as D5/D7/D9/D12; graph ENGAGED (5 captures C=2048/5/4/15/6, 32+ replays); the token-diff is the capture-safety proof ([[cudagraph-capture-bakes-stack-addresses]]). **c1 A/B (one flock series, cold rep discarded, 8 prompts×256 tok):** our OFF 10.24 / our ON eager-paged 28.65 (28.69,28.61) / **our ON GRAPHED 28.70 (28.70,28.70), TPOT 34.40** / vLLM-ON graphed steady-state 29.35 (tight 3-rep 29.33/29.37/29.33, TPOT 34.07, acc_len 4.44); D9's 28.09 was a colder cross-session outlier — **NEAR-PARITY: ours 0.978× (~2% below) on the rigorous same-session band** (across sessions ours 28.70 falls inside vLLM's observed 28.09–29.37 range). ON/OFF 2.80× (vLLM ~2.98×), our OFF ≥ vLLM OFF. Per the acceptance rule ("below on any axis = an open gap; near-parity is NOT met"), the ≥vLLM bar is NOT met; STAYS `ACTIVE`. Residual (data-grounded): NOT acceptance (ours realized ~3.68 accepted draft-tok/step > vLLM's 3.44) and NOT launch/graph (both graphed, CG neutral) — per-step COMPUTE (~2% slower target-step); next lever = nsys both draft steps (`--cuda-graph-trace=node`), no premature ceiling. **ATTRIBUTION (supersedes D9):** the CUDA graph is perf-NEUTRAL (+0.3%); the ACTUAL lever was the paged context read (C.1/C.2) removing the D9/D12 per-layer `[context;block]` `IndexCopy` materialization of the whole growing context (25.75 D9 → 28.65 eager-paged, +11%) — the roadmap's "the full CG closes the gap" premise is corrected by measurement. Inertness VERIFIED on the capture binary: SACRED 235/235 + MTP 9/9 byte-identical, CUDA `-Werror` clean, no new kernel (D12 paged kernel already memcheck-0 795648), `check-device-leakage` not increased (paged path REMOVES the materialized-buffer allocs + host uploads). `benchmark_binding=true`. Correctness-complete (ratified near-tie); throughput NEAR-PARITY (0.978×, ~2% residual) ⇒ STAYS `ACTIVE` (the capture-correctness gate is MET; the ≥vLLM speed bar is the sole remaining item, a ~2% per-step-compute residual for an nsys). Anchors: `src/vllm/model_executor/models/qwen3_dflash.cpp` (`DflashDeviceKVStore` paged store, `ForwardPagedBody`, the per-request graph in `ForwardBlockLogitsWithDeviceKV`). **D14 2026-07-27 (`CLAIM-DFLASH-D14`) — SPEED GATE MET → SPEC-DFLASH `DONE`:** an nsys (`--cuda-graph-trace=node`) of the graphed spec-on step attributed the D13 ~2% residual to the from-scratch `DFlashPagedBlockAttentionKernel` draft attention (242.9 ms = 1.8% of GPU time, median ~460 us/call over context C~500-640, vs vLLM's fused flash draft-attn ~0.15%; BOTH engines run identical `cutlass_80_wmma` for the draft bf16 GEMMs, so the GEMMs were NOT the gap). Ported it to a WARP-scoped online-softmax variant `DFlashPagedBlockAttentionWarpKernel` (mirrors the shipped `AttentionWarpKernel`: one warp per (block-query,head), `__shfl_xor` butterfly reduction, register accumulator, NO `__syncthreads` storm; SAME paged/block combined-index read + causal/SWA mask + GQA; default ON, `VT_DFLASH_ATTN_BLOCK=1` keeps the bit-identical D12/D13 block kernel for A/B). Draft attn 242.9 → 77.9 ms (3.1x); our-ON c1 28.60 → 29.32 tok/s (+2.5%). **FINAL same-session 3-rep A/B (8 prompts×256 tok, cold leg discarded): our-ON graphed 29.42/29.27/29.32 (med 29.32) vs vLLM-ON graphed 29.240/29.247/29.233 (med 29.240) — our WORST rep (29.27) > vLLM's BEST (29.247), NON-OVERLAPPING bands, 1.003× ⇒ the ≥vLLM speed gate is MET.** Correctness UNCHANGED (output is exact by spec-decode construction — the target verify is untouched, only which draft proposals are accepted can shift): e2e `test_qwen27_dflash_spec_decode` 27/27 with graph==eager BIT-IDENTICAL, acceptance 19/39/29/25 unchanged (draft accepted 1629 identical warp-vs-block across the whole A/B set), 2/4 STRICT (France@11, 17×23@12 unchanged); CUDA==CPU `test_ops_dflash_paged_block_attn` 795648/795648 (warp within the f32 1e-4 / bf16 3e-2 envelope) + compute-sanitizer 0. Inertness SACRED 235/235 + MTP 9/9 byte-identical; CUDA `-Werror` clean; `check-device-leakage` not increased. `benchmark_binding=true`. Block-diffusion drafting is now correctness-complete (ratified near-tie) AND at/above vLLM throughput — this was the roadmap's FINAL open speed item. Anchors: `src/vt/cuda/cuda_ops.cu` (`DFlashPagedBlockAttentionWarpKernel` + `UseDflashAttnBlockKernel`; the D12 block kernel retained as the `VT_DFLASH_ATTN_BLOCK=1` reference). | T1 | `vllm/v1/worker/gpu/spec_decode/dflash/speculator.py`; `vllm/model_executor/models/qwen3_dflash.py`; `vllm/model_executor/models/interfaces.py:1382` (aux value); `eagle3_utils.py:41-56` (+1 shift) | `include/vllm/model_executor/models/qwen3_5.h` (`Qwen3_5AuxTaps`, `ForwardDeviceMultiTap`); `qwen3_5_dense.h`; `model_registry.h` (`aux_tap`); `src/vllm/model_executor/models/qwen3_5.cpp` (`MaybeCaptureAuxTap`/`ValidateAuxTapLayerIds`/`ForwardDeviceMultiTap`); `qwen3_5_moe.cpp`+`qwen3_5_dense.cpp` (routing); D2/D3 `include/vllm/model_executor/models/qwen3_dflash.h` + `src/vllm/model_executor/models/qwen3_dflash{,_weights}.cpp`; D4 `include/vllm/v1/worker/gpu/spec_decode/dflash/speculator.h` + `src/vllm/v1/worker/gpu/spec_decode/dflash/speculator.cpp` (`DflashProposeBlock`/`SampleDflashBlockDrafts`); D5 `src/vllm/entrypoints/model_loader.cpp` (`LoadDflashDraft`/`DflashDraft`) + `include/vllm/entrypoints/model_loader.h`; D5 `src/vllm/v1/worker/gpu/runner.cpp` (`set_dflash_draft`/`propose_drafts_dflash`/aux-tap capture) + `include/vllm/v1/worker/gpu/runner.h`; `src/vllm/config/speculative.cpp` + `include/vllm/config/speculative.h` (`ResolveDflash` + `dflash`/`model` parse); D14 warp kernel [cuda_ops.cu](../src/vt/cuda/cuda_ops.cu#L1433) | `tests/vllm/models/test_qwen27_paged_forward.cpp` (multi-tap 598); `tests/vt/test_ops_dflash_block_attn.cpp`; `tests/vllm/models/test_qwen3_dflash_forward.cpp`; `tests/vllm/v1/spec_decode/test_dflash_kvprep.cpp`; `tests/parity/test_qwen3_dflash_{draft,kvprep}_parity.cpp`; D4 `tests/vllm/v1/spec_decode/test_dflash_propose.cpp` (5/19, RED-first); D5 `tests/parity/test_qwen27_dflash_spec_decode.cpp` (e2e 27/27, 2/4 strict + acceptance~vLLM); `scripts/spec/d{0,2,3}_dflash_*.py`; `tests/parity/goldens/dflash_27b{,_draft,_kvprep}/`; D6 `scripts/spec/vllm_dflash_timing.py` (vLLM-DFlash c1 timing); D7 device-resident `src/vllm/model_executor/models/qwen3_dflash.cpp` (`PrecomputeContextKVDevice` + `ForwardBlockLogitsWithContext` via `vt::IndexCopy`/`IndexSelect`); D9 persistent paged draft-KV `qwen3_dflash.{h,cpp}` (`AppendContextKVHost`/`ForwardBlockLogitsWithPrecomputedKV`/`ForwardWithCtxKVDev`/`PrecomputedContextKV`) + `runner.{h,cpp}` (`dflash_kv_store_`/`propose_drafts_dflash`) + `tests/vllm/v1/spec_decode/test_dflash_propose.cpp` (2 D9 bit-identity cases); D12 A-wire `runner.{h,cpp}` (device store as production path) + D12 Part B `include/vt/ops.h`/`src/vt/ops.cpp`/`src/vt/cpu/cpu_ops.cpp`/`src/vt/cuda/cuda_ops.cu` (`kDFlashPagedBlockAttention`) + `tests/vt/test_ops_dflash_paged_block_attn.cpp` (CPU==CUDA + cross-check, 795648/795648 + sanitizer-0); D13 `src/vllm/model_executor/models/qwen3_dflash.cpp` (fixed-capacity paged `DflashDeviceKVStore` + `ForwardPagedBody` + the per-request draft-step CUDA graph in `ForwardBlockLogitsWithDeviceKV`); D14 [test_ops_dflash_paged_block_attn](../tests/vt/test_ops_dflash_paged_block_attn.cpp#L79) + [ledger](parity-ledger.md#L738) | [dflash-spec-decode.md](specs/dflash-spec-decode.md) | `DONE` | `489a7544` | From 8b50c3f7b430b21f2b43e8d07471767c776bb08f Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Sun, 13 Sep 2026 19:24:03 +0200 Subject: [PATCH 8/9] record: re-anchor LoadGgufSharedEmbedAndHeadBf16 to 1065 (#3042) The wave's insertions above the shared-head loader moved the symbol; the branch tree holds it at 1065 (measured; the 1044 citation was a cross-worktree misread during the rebase churn). check-agent-record: the gguf_weights citation resolves OK. FOLLOWING_AGENTS_PROTOCOL Following-Agents-Protocol: true AI-Assisted: true Assisted-by: AGENT:zai/glm-5.3-flash [maki] --- .agents/engine-matrix.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.agents/engine-matrix.md b/.agents/engine-matrix.md index a49467ae2..6ada37bfe 100644 --- a/.agents/engine-matrix.md +++ b/.agents/engine-matrix.md @@ -173,7 +173,7 @@ lifecycle are unchanged. | `SPEC-MTP` | Qwen3.6 MTP heads, k=1 first. **M-mtp-0 CLOSED 2026-07-24: the standalone draft head is oracle-parity-proven on BOTH checkpoints** (27B dense + 35B MoE, k=1, vLLM 0.25.0 executable @ pin `e24d1b24`) - argmax exact on 26/26 unambiguous rows each; the one remaining row per checkpoint is an EXACT oracle top1==top2 tie where vLLM's own `argmax` and `topk` disagree and our pick is a tied maximum; logits within the whole-model bound (atol 0.05 + rtol 0.05), 0/216 out-of-tol on both; shared lm_head isolated is bit-exact on the 35B NVFP4 head. **I2 scheduler-half LANDED (2026-07-24)**: host-side spec plumbing + the FROZEN spec-metadata ABI (spec §2.7) - `SpeculativeConfig`, `DraftTokenIds`, `Request::spec_token_ids`/`NumTokensWithSpec`, populated `scheduled_spec_decode_tokens`, `Scheduler::update_draft_token_ids`, `take_draft_token_ids` seam, `EngineCore::post_step`, `InputBatch::num_accepted_tokens`/`update_req_spec_token_ids`; DEFAULT-OFF and INERT (no `SpeculativeConfig` => `num_lookahead_tokens == 0`). **I3 verify-half LANDED (2026-07-24)**: greedy rejection sampler + per-request logits expansion (see `SPEC-REJECTION`, now `ACTIVE`). **I4 GDN-half LANDED (2026-07-24)**: the GDN speculative slot path + bit-exact state rollback, the piece BOTH GDN-hybrid gate checkpoints need (see `SPEC-GDN-SEGMENTS`, now `ACTIVE`). **I5a GDN LAYER ROUTING + runner spec-metadata upload LANDED (2026-07-24, `CLAIM-SPEC-MTP-I5A`)**: `GdnBlockPaged` now routes a pure-spec batch through `vt::GdnSpecDecode`/`vt::CausalConv1dSpecUpdate` and the runner uploads I4's six spec device tensors — first sub-increment of the scoped M-mtp-1 (I5a GDN wiring → I5b prepare_prefill → I5c MTP paged propose → I5d config+runner-loop+the 27B token gate, spec §5). DEFAULT-OFF INERT, bit-exact vs the I4 ops, no e2e loop yet. **I5b `prepare_prefill_inputs` LANDED (2026-07-24, `CLAIM-SPEC-MTP-I5B`, recorded under `SPEC-REJECTION`)**: the drafter prefill input-prep host routine (shift-splice + `query_len -= num_rejected` + last-token index / metadata) — second scoped M-mtp-1 sub-increment, DEFAULT-OFF INERT, unit-gated RED-first, additive. **I5d CONFIG + RUNNER LOOP LANDED, PARTIAL (2026-07-25, `CLAIM-SPEC-MTP-I5D`)**: `--speculative-config` JSON parse -> `EngineParams::speculative_config`; `LoadedEngine` resolution (`ResolveSpecConfig`/`ResolveMtp`, widened KV `MakeQwen3_5KVCacheSpec(num_spec>0)`, `BuildMtpDraft`, forced sync scheduling, `MakeScheduler(spec)`, `EngineCore(check_for_draft=true)`); the full runner verify/propose loop (draft splice, hidden-tap capture, GDN builder spec-overload feed, k+1 GDN state-slot remap + widened conv cache + draft-KV alloc, `MtpProposePrefill` post-sampling, `take_draft_token_ids`, acceptance telemetry). CUDA `-Werror` 0 warnings, cutlass-ON banner. SPEC-OFF BYTE-IDENTICAL (all gated on `spec_on()`): SACRED 27B 235/235, 35B 315/315, Coder 138/138 + unit test_runner 257 / test_mtp_speculator 169 / test_gdn_metadata_builder 483 / test_ops_gdn 3630 ALL PASS. **The three-way 27B token gate is NOT yet passing** (`tests/parity/test_qwen27_spec_decode.cpp` RUNS the loop + MEASURES the blocker): the spec-ON engine throws on the FIRST prefill step at `gdn_state_gather: working/cache row shapes must match` (`src/vt/ops.cpp:1773`) — I4's spec conv rollback needs the conv row widened to `(K-1)+num_spec` but the non-spec GDN conv ops assume `(K-1)`. Closing needs widened-cache-aware non-spec GDN conv ops + the MIXED `GdnBlockPaged` split/merge. Row LEFT `GATING` at I5e. **I5e LANDED 2026-07-25 (`CLAIM-SPEC-MTP-I5E`) — `SPEC-MTP` LEAVES `GATING`.** Made the non-spec GDN conv ops widened-cache-aware (mirror vLLM `state_len=KERNEL_WIDTH-1` + physical `stride_conv_state_tok`; leading `(K-1)` sub-window; byte-identical at `num_spec==0`, contiguous fast path kept) AND RCA'd the resulting 0-acceptance dead-drafter to the async input-combine overwriting the verify batch's draft position with the committed token (forced off under spec, nullopt-guarded). **THREE-WAY 27B GATE PASSES** (single-request greedy): our-ON == vLLM `--speculative-config mtp` greedy == our-OFF token-for-token; **acceptance 16/16 drafts accepted**, ~16 target steps saved. Spec-OFF SACRED byte-identical (27B 235/235, 35B 315/315, Coder 138/138), `test_ops_gdn` 3678, compute-sanitizer 0 on the spec step. NOT `DONE`: MIXED `GdnBlockPaged` split/merge (concurrency) + throughput A/B are I6. **I6 LANDED 2026-07-25 (`CLAIM-SPEC-MTP-I6`), `benchmark_binding=true` — the §5 c1 THROUGHPUT GATE, first spec-decode speed number:** OURS spec-ON (`examples/vllm-bench` + an additive `--speculative-config` flag, production config) vs pinned vLLM 0.25.0 spec-ON (graphed `vllm serve --speculative-config mtp` + `vllm bench serve`, `enforce_eager=False`/`FULL_AND_PIECEWISE`/inductor; MTP confirmed `Resolved architecture: Qwen3_5MTP`), SAME `{"method":"mtp","num_speculative_tokens":1}`, 27B `~/bench/q36-27b-nvfp4-vllm`, c1, greedy, 8 real prompts x 256 out, prose + code, idle box one-engine-at-a-time under one `flock`, 3 reps (cold TTFT discarded), token-identity re-confirmed FIRST (`test_qwen27_spec_decode` PASS 16/16). RESULT — **ours AT/ABOVE vLLM on EVERY measured axis** (prose / code): TPOT 66.2/62.95 vs 69.1/65.3 ms (ours ~1.04x faster), output tput 15.10/15.72 vs 14.43/15.13 tok/s (+4.6%/+3.9%), ITL 121.6/121.1 vs 123.2 ms, TTFT(warm) 131/131 vs 151.5/181 ms, acceptance ours 0.85/0.92 vs vLLM 0.838 overall (within noise, live drafter both), peak RSS 28.4 GB ON / 24.8 GB OFF (both inside the 119 GiB pool). Spec helps both (ours 1.52x/1.59x, vLLM 1.51x/1.60x TPOT); ours already ~4% faster spec-OFF. STAYS `ACTIVE`: the c>1 mixed spec+non-spec `GdnBlockPaged` split/merge is still refused (needs a row `IndexSelect`/`IndexCopy` vt op) + owes a c>1 A/B, and no user-facing supported `--speculative-config` on the OpenAI server yet (bench flag example-only/additive). Raw logs dgx `~/work/mtp-bench-i6/{results,vresults}`. **I7 LANDED 2026-07-25 (`CLAIM-SPEC-MTP-I7`, `benchmark_binding=true`) — the MIXED spec+non-spec GDN batch (concurrency), the server/CLI `--speculative-config`, and the c>1 A/B — implementation COMPLETE + at vLLM parity; STAYS `ACTIVE` for one honest reason (below), NOT a lag.** New row op `vt::IndexSelect`/`vt::IndexCopy` (CUDA==CPU bit-exact at GDN widths, RED-first); `GdnBlockPagedMixedSpec` split/merge (mirror `qwen_gdn_linear_attn.py:1329-1576`) proven MODEL-INDEPENDENTLY bit-exact (mixed == pure spec + pure prefill, 27B/35B, `test_qwen3_5_gdn_spec_routing`, RED-first by a broken merge); compute-sanitizer 0 on the mixed step + op; server (I5d) + CLI (ABI v6) `--speculative-config`. **c>1 A/B (both spec-ON, same config):** ours ON-PAR-OR-ABOVE vLLM at c2/c4/c8 (output tput within ~+/-2%, ours +1.6%/+2.5% c2, +0.9%/+1.7% c4, +0.9%/-1.1% c8 within noise, prose/code; both ~1.5x spec speedup — does NOT go neutral; acceptance 0.84-0.92 vs vLLM 0.835). **Why STAYS `ACTIVE` (honest, not a lag):** the DONE criterion's strict `token-exact at c>1` clause is a proven MODEL impossibility — the 27B greedy is bf16-batch-nondeterministic (spec-OFF max_seqs 4-vs-1 differs 2/3 short prompts, NO spec involved), affecting vLLM identically, so exact c>1 token identity cannot be met by any correct implementation; c>1 correctness is instead established by the model-independent bit-exact split/merge proof + acceptance parity (near-tie-distributional-gate), with token-exact strict at c1 (I6). No missing work, no lever — the DONE final call is deferred to the user given this criterion ambiguity. SACRED spec-OFF byte-identical 27B 235/235, 35B 315/315, Coder 138/138; CUDA `-Werror` 0 warnings. Raw logs dgx `~/work/mixed-batch/{cN_results,cN_vresults}`. **I8 — `SPEC-MTP` → `DONE` 2026-07-26 (`CLAIM-SPEC-MTP-DONE`, records-only, ZERO code):** the user RATIFIED the deferred c>1 criterion — at concurrency > 1 the DONE bar is the near-tie-distributional form (ours ∈ vLLM's batch-nondeterministic set) + the SPEED delta, NOT strict token-exact (a proven bf16-batch-nondeterminism MODEL impossibility that affects vLLM identically). Both I6-owed DONE items are therefore CLOSED: (1) the MIXED spec+non-spec `GdnBlockPaged` split/merge (I7, model-independently bit-exact + compute-sanitizer 0) with the c2-c8 A/B on-par-or-above vLLM, and (2) the server + CLI + C-ABI(v6) `--speculative-config` flag (I5d/I7, `examples/server/main.cpp`+`examples/cli/main.cpp`+`src/capi/vllm_c.cpp`). MTP k=1 spec-decode is COMPLETE and gated: 27B three-way token-exact at c1 (I5e), c1 above vLLM on every axis (I6), c2-c8 on-par-or-above (I7), spec-OFF byte-identical SACRED (27B 235/235, 35B 315/315, Coder 138/138). This transition is byte-identical BY CONSTRUCTION (`git diff --stat` = records only; ZERO `src/`/`include/`/`examples/` touched, so the I5d/I6/I7 GPU gates stand on this exact code). Tracked follow-ons: the 35B `Qwen3_5MoeMTP` full e2e token gate (M-mtp-2) is now **CLOSED — `DONE` 2026-07-26 (`CLAIM-SPEC-MTP-M-MTP-2`)**: three-way token-exact 16/16 vs the live vLLM 0.25.0 oracle (spec-ON AND spec-OFF), acceptance 16/16 both sides, c1 spec-ON 1.19x TPOT / +16.3% output-tput vs spec-OFF (0.908) — `MODEL-SPEC-qwen3-5-mtp-qwen3-5-moe-mtp` `GATING`→`DONE`, so MTP is `DONE` on BOTH gate models. Remaining spec-decode follow-on: `SPEC-DFLASH` (oracle-BLOCKED, vllm#40898) | T1 | `vllm/v1/worker/gpu/spec_decode/mtp/speculator.py:12`; `vllm/model_executor/models/qwen3_5_mtp.py:63,129-165,192-301`; **I5d** `vllm/engine/arg_utils.py` (`--speculative-config`); `vllm/v1/worker/gpu/model_runner.py:1455-1489` | `include/vllm/config/speculative.h`; `include/vllm/v1/core/sched/scheduler.h`; `src/vllm/v1/core/sched/scheduler.cpp`; `include/vllm/v1/worker/gpu/input_batch.h`; `include/vllm/model_executor/models/qwen3_5_mtp.h:23,58`; `src/vllm/model_executor/models/qwen3_5_mtp.cpp:271`; `src/vllm/model_executor/models/qwen3_5.cpp:3336,3359`; **I5d** `src/vllm/config/speculative.cpp`; `src/vllm/entrypoints/model_loader.cpp` (`ResolveSpecConfig`/`MakeKVCacheMaybeSpec`/ctor wiring); `src/vllm/v1/worker/gpu/runner.cpp` (splice/tap/GDN spec feed/`propose_drafts`/`take_draft_token_ids`/spec-slot remap/draft-KV alloc); `examples/server/main.cpp` | `tests/vllm/v1/test_scheduler.cpp:1135,1238,1272,1316`; `tests/vllm/v1/worker/test_input_batch.cpp`; `tests/vllm/v1/spec_decode/test_mtp_speculator.cpp:201,225,263,299,331` (7/7 cases, 141 assertions); oracle runner `tests/parity/test_op_parity.cpp:1373` + focused case `:1914` (20/20 assertions, both checkpoints, `VLLM_MTP_REQUIRE_CHECKPOINTS=1`); goldens `tests/parity/goldens/qwen3_5_mtp_head_{27b,35b}/`; dump `tools/parity/dump_qwen3_5_mtp.py:144`; **I5d** `tests/parity/test_qwen27_spec_decode.cpp` (three-way gate, RUNS + measures the RCA blocker); **I6** `examples/bench/{main.cpp,bench_core.h}` (additive `--speculative-config` bench flag + acceptance telemetry); **I7** `tests/vllm/models/test_qwen3_5_gdn_spec_routing.cpp` (mixed == pure spec + prefill bit-exact), `tests/parity/test_qwen27_spec_decode_concurrent.cpp`, `tests/vt/test_ops_gdn.cpp` (IndexSelect/IndexCopy); DONE closure [ledger](parity-ledger.md#L714) | [mtp-spec-decode.md](specs/mtp-spec-decode.md) | `DONE` | `72f9fb1` | | `SPEC-MTP-K-GT-1` | **MTP speculation DEPTH (`num_speculative_tokens` > 1).** Ports the autoregressive multi-step propose the k=1 early exit sits in front of, so a configured depth is SERVED instead of silently degraded. Before it, `--num-speculative-tokens 3` reserved KV for 3, captured the verify shape at T=4 and stashed ONE draft per request, with no error and no log; a refusal by name landed first and this row removed it in the same flow. `MtpProposeDrafts` runs the prefill, the k=1 early exit, then `prepare_decode_inputs` and the k-1 single-token draft decode steps over the draft's own paged KV, with `update_draft_inputs` recording each step and feeding it forward. Greedy plus accept-if-equal makes the emitted sequence INDEPENDENT of k, so a token-identity gate cannot see a clamped drafter and every depth assertion needs a positive witness beside the identity. The per-depth counters were the FIRST witness and a fresh review proved them BLIND: they report the LENGTH of the emitted draft list, so a propose that runs one forward and pads all k columns satisfies them, and acceptance is zero at every depth on the CPU model, so no acceptance figure separates the arms either. TWO witnesses survive, because one does not cover both failures. `spec_mtp_draft_decode_forwards() == spec_mtp_propose_calls() * (k - 1)`, counted after each draft decode forward RETURNS and guarded by a non-zero call count, catches a propose that SHORT-CIRCUITS or CLAMPS. A third fresh review then proved it does NOT catch PADDING, since a loop that runs every forward and then discards what it sampled increments it honestly. `spec_mtp_proposals_with_varied_drafts()`, read at the CONSUMER on the array the propose delivered, catches exactly that. NEITHER shows per-column provenance, and neither does a non-zero acceptance count AT DEPTH, which a padded row earns whenever the target repeats a token. The owed DGX gate closes it with a per-depth acceptance RATE against a PADDED CONTROL. The CPU tier therefore proves k drafts are PROPOSED and VERIFIED, never ACCEPTED at depth. DEFAULT unchanged at k=1 (both checkpoints' `n_predict`). **NO speed number at any k>1**: the GPU was held by another session for the whole flow, so the DGX three-way at k=2..4 on the 27B and 35B and the matched-k throughput A/B are OWED, as is the bf16 GDN-state arm (the CPU gate runs the f32 arm because `vt::CausalConv1dSpecUpdate` rejects bf16 off CUDA). Also owed and filed: [#1020](https://github.com/mudler/vllm.cpp/issues/1020), a step whose ACTUAL draft count differs from the configured k leaves the captured verify graph silently. | T1 | `vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py:129-274,335-371,374-419,426-471,597-671,674-771` @ `555967922`; `vllm/config/speculative.py:967-991` | [`src/vllm/v1/worker/gpu/spec_decode/mtp/speculator.cpp`](../src/vllm/v1/worker/gpu/spec_decode/mtp/speculator.cpp) (`MtpProposeDrafts`); [`prepare_decode_inputs.cpp`](../src/vllm/v1/worker/gpu/spec_decode/autoregressive/prepare_decode_inputs.cpp); `Qwen3_5MTPModel::GatherHiddenRows` ([qwen3_5.cpp](../src/vllm/model_executor/models/qwen3_5.cpp)); `GPUModelRunner::propose_drafts` + the per-depth counters ([runner.cpp](../src/vllm/v1/worker/gpu/runner.cpp), [runner.h](../include/vllm/v1/worker/gpu/runner.h)); the in-memory `mtp_weights` seam ([model_loader.h](../include/vllm/entrypoints/model_loader.h)) | [`test_mtp_depth`](../tests/vllm/v1/spec_decode/test_mtp_depth.cpp) 5/5, 63 assertions (k=1,2,3,4 through `LoadedEngine`, greedy tokens identical to spec-OFF, each arm witnessed BOTH by the draft decode forwards the propose RAN and by whether the DELIVERED draft row varied with depth; neither witness shows per-column provenance, which is owed to the DGX gate); [`test_prepare_decode_inputs`](../tests/vllm/v1/spec_decode/test_prepare_decode_inputs.cpp) 8/8, 33 (both kernel ports + both `max_model_len` clamps, 5 mutations caught); [`test_speculative_mtp_depth`](../tests/vllm/config/test_speculative_mtp_depth.cpp) 4/4, 20; full CPU suite ctest 493 passed / 0 failed / 2 skipped of 495 (the two skips checkpoint-gated and unrelated) | [mtp-k-gt-1.md](specs/mtp-k-gt-1.md) | `ACTIVE` | `CLAIM-SPEC-MTP-K-GT-1` ([#81](https://github.com/mudler/vllm.cpp/issues/81)) | | `SPEC-MTP-GGUF` | MTP speculative decoding from a GGUF TARGET. Today `FromModelDir` refuses `mtp`+GGUF outright (`src/vllm/entrypoints/model_loader.cpp:717-723`) on the original spike's assumption that GGUF exports carry no `mtp.*` ([mtp-spec-decode.md](specs/mtp-spec-decode.md):979-980, "until we re-export GGUFs with the head"). That is stale: llama.cpp's Qwen3.5 converter DOES emit the head, under layer-indexed `nextn` naming, and our own `HfConfigFromGguf` ALREADY reads `nextn_predict_layers` (it just discards the value into the trunk layer count). Gap is a `TensorResolver` over `GgufFile` mapping `mtp.*` onto `blk.{L+i}.nextn.*` with dequant-to-bf16, one config field, and narrowing the rejection to `dflash`. `ngram`+GGUF already works and is untouched. Qwen3.5/3.6 only (the widened spec KV path serves no other arch). NO ABI change | T2 | llama.cpp (the producer contract; vLLM has no GGUF MTP path) `conversion/qwen.py:535-604` `_Qwen35MtpMixin` (the authoritative `mtp.*`->`nextn` remapper + `add_nextn_predict_layers`); `gguf-py/gguf/constants.py:129,910-917,1494-1501`; `gguf-py/gguf/tensor_mapping.py` `NEXTN_*` | **G1-G3 LANDED 2026-07-28.** `HfConfigFromGguf` republishes the head depth `src/vllm/model_executor/models/qwen3_5_gguf_weights.cpp:598` (`c.raw["mtp_num_hidden_layers"] = nextn`, previously read then discarded); the head loader `LoadQwen3_5MTPFromGguf` `src/vllm/model_executor/models/qwen3_5_gguf_weights.cpp:1521` (+ decl `include/vllm/model_executor/models/qwen3_5_gguf_weights.h:143`) reusing the TRUNK helpers `OwnNormMinus1`/`OwnMatmulWeight`/`OwnBf16`/`LoadAttnGguf`/`LoadMoeGguf` so the head inherits the GGUF (w+1) norm storage, quantization/residency routing and torch [N,K] shapes; `NumMtpLayers`/`UsesDedicatedEmbeddings` exported out of the anon namespace `include/vllm/model_executor/models/qwen3_5_mtp.h`; rejection narrowed to dflash + a head-less-GGUF check `src/vllm/entrypoints/model_loader.cpp` and the head attached in the GGUF branch; **G4 GREEN + `CPU-SPEC-DIVERGENCE` FIXED 2026-07-28**: root cause `src/vllm/model_executor/models/qwen3_5.cpp:3616` sized the GDN state gather/scatter row by `(Kw-1)` while the speculative persistent row is `(Kw-1)+num_spec`, so `GatherRows`/`ScatterRows` mis-strode the slot AND every channel past the first, corrupting post-prefill recurrent state. Fix = `CopyStateRowsStrided` (same TU) used by `GatherStateF32`/`ScatterStateF32` when `cache.shape[2] != work.shape[2]`; the contiguous helpers are kept when the widths agree, so every non-spec path is byte-identical by construction. CPU-only in effect (the fp16/bf16 arm routes through the `GdnStateGather`/`Scatter` ops, so CUDA was never exposed; no GPU result affected) | `tests/vllm/models/test_qwen3_5_gguf_mtp.cpp:109,146,156,184` **4 cases, and the split is the 2026-08-21 repair** ([#1454](https://github.com/mudler/vllm.cpp/issues/1454)): the file used to be the env-gated pair ALONE, each opening on a bare `return`, so with `VLLM_MTP_GGUF_MODEL` unset it reported `test cases: 2 \| 2 passed`, **`assertions: 0`**, `Status: SUCCESS!`, exit 0 - which is every CI run of this repository, the variable being set nowhere in `.github/workflows/`. The `18 assertions` this cell used to record was the LIVE count and was never once reached in CI. Now `:109` and `:146` are **HERMETIC** (KV-only synthetic GGUFs, no weight bytes, 18 assertions on any machine) and pin the arithmetic the old file only NAMED in a comment above `CHECK(c.num_hidden_layers > 0)`: `num_hidden_layers + mtp_num_hidden_layers == block_count` over 65/1, 25/1 and 28/3 - the third arm separating `- nextn` from `- 1` - plus the head-less arm, where the key is NOT published and `NumMtpLayers` answering 1 for an absent key is exactly why the invariant cannot be written with that helper alone. `:156` and `:184` stay env-gated on `VLLM_MTP_GGUF_MODEL` (so CI stays asset-free) and now SKIP LOUDLY with a `MESSAGE` naming the variable, as `tests/vllm/entrypoints/test_gguf_mmproj_reach.cpp` does; `:156` re-derives the same invariant from the file's OWN `block_count` kv. Unset: **4 cases / 18 assertions / `Status: SUCCESS!` / rc 0**. Live on `Qwen3.8-27B-Q4_K_M.gguf` (`block_count` 65, `nextn_predict_layers` 1): **4 cases / 38 assertions / `Status: SUCCESS!` / rc 0**. Mutation-proved on the production line `src/vllm/model_executor/models/qwen3_5_gguf_weights.cpp:889`, both compiling clean and both restored against a pre-taken sha256: `= block_count` (drop the subtraction) 3/4 cases, 9/18 red, exit 1; `= block_count - 1` (the wrong constant) 2/4 cases, 5/18 red, exit 1. The SAME mutations left the PREVIOUS file at 2/2 cases, 0 assertions, `SUCCESS!`, exit 0. Correctness of the production line is unchanged and was never in question (`1a4db5c3c`, `493327b4e`); this was a test defect. Live-arm content unchanged: depth reaches config.raw; fc is [H,2H] verbatim; 3 norms [H]; head block is full-attention. **RED-first BEHAVIOURAL** (reverting only the G1 line fails both cases 2/2). Trunk inertness: `test_gguf` 103, `test_gguf_qwen36_loader` 99, `test_gguf_keep_quant` 5958, `test_gguf_dequant` 215, `test_capi` 33/232 all unchanged; `tests/parity/test_qwen35_gguf_spec_decode.cpp:74,139` - spec-ON == spec-OFF token-exact with 13 proposed/11 accepted, plus an `ngram` regression guard (widens the cache, never runs the spec conv update) that was token-exact throughout and pinned the widening as innocent. Regression sweep all unchanged: ops_gdn 1825, gdn_metadata_builder 483, gdn_prefill_conv 28, gdn_spec_routing 12, gguf 103, gguf_qwen36_loader 99, gguf_keep_quant 5958, gguf_dequant 215, llm_engine 196, input_batch 163, runner 257, capi 232 **GPU CLOSE-OUT + DEVICE-DELTA ATTRIBUTION 2026-07-28 (`G5`-`G7`), ledger [parity-ledger.md#L800](parity-ledger.md#L800).** The GPU end-to-end gate re-run on a from-scratch RELEASE-TARGET build (`-DVLLM_CPP_CUDA_ARCHITECTURES=121a`, build dir DELETED first; arch VERIFIED by `build-cuda/CMakeFiles/vllm.dir/flags.make` `--generate-code=arch=compute_121a,code=[compute_121a,sm_121a]` and by `cuobjdump -lelf` 20 cubins ALL `sm_121a` zero sm_75, NOT by `CMakeCache.txt`, whose `CMAKE_CUDA_ARCHITECTURES:STRING=75` is the `enable_language(CUDA)` compiler-probe default shadowed by the normal variable at `CMakeLists.txt:186` - the prior wrong-arch conclusion was that decoy): dgx.casa GB10 under `flock $HOME/gpu.lock`, 35B A3B NVFP4 GGUF, **2/2 cases, 10/10 assertions, exit 0**, spec-ON token-identical to spec-OFF, 13 proposed / 11 accepted, 90.2 GiB peak RSS, 8m01s; re-run on the EXACT committed source **3/3 cases, 10/10 assertions, exit 0**, 7m25s, the new probe case SKIPping and adding zero assertions. **The CPU-vs-GPU token delta is a MEASURED near-tie, not a defect** (it was never this row's bar - spec-ON == spec-OFF WITHIN a device is): NEW double-gated spec-OFF-only probe `tests/parity/test_qwen35_gguf_spec_decode.cpp:217` (asset + `VLLM_MTP_GGUF_PROBE=1`, 20 alternatives per position, 484/484 assertions per arm, GPU then `CUDA_VISIBLE_DEVICES=` in one `flock` series) shows both arms picking `11751` at position 0 and forking at position 1 on a BIT-IDENTICAL prefix: GPU rank1 `13` -0.773180 over rank2 `11` -0.847055 (margin 0.0739 nats), CPU rank1 `11` -0.765499 over rank2 `13` -0.830374 (margin 0.0649 nats). Each device's pick is the other's rank 2, both ~7x inside the ratified 0.5-nat band, and the cross-device disagreement on the SAME token (0.057 and 0.082 nats) EXCEEDS the margin being decided, so rounding settles it; the 24 texts look unrelated only because positions 2+ cascade off that one coin flip. Margin sweep over all 24 positions: **GGUF GPU and GGUF CPU carry ZERO exact ties**, minimum margins 0.0482 and 0.0649 nats, and both arms reproduced their sequence across every run. **Gate 4 MET on the safetensors sibling of the same quantization run** (`FromModelDir` takes it unchanged): acceptance 12 proposed / 11 accepted vs the GGUF's 13 / 11. That arm, however, FAILS spec-ON == spec-OFF at concurrency 1 and does not reproduce its own spec-OFF sequence run to run, and the probe attributes both to THREE EXACT ties (positions 7, 10, 16, bit-identical logprobs) produced by its 1/16-grid quantized-GEMM logits - which EXONERATES the GGUF arm and opens a recorded, not-root-caused `SPEC-MTP` item on the safetensors NVFP4 path, not on this row. Gate 3 is NOT APPLICABLE twice over: no F16/F32 head-carrying export exists, and the only same-weights sibling is not token-stable against itself. **EVIDENCE RE-ANCHORED 2026-07-29 to a PRODUCTION-CONFIGURED build, because every GPU number above came from a build configured WITHOUT `-DVLLM_CPP_CUTLASS_DIR` and WITHOUT `-DVLLM_CPP_TRITON=ON`** (the defect `CLAIM-27B-GATE-RCA` proved, which runs the emulation fp4 GEMM + hand GDN kernels). Re-run from a clean `git archive` tree of `main` `3f34534d`, build proven correct three ways (configure log has ZERO `CUTLASS not found` and prints `CUTLASS found ... sm120a NVFP4 cutlass GEMM` + `FlashAttention-2 ... ENABLED for arch(es) [121a]` + the vendored `sm_121a` Triton-AOT lines with `MANIFEST hashes OK`; `cuobjdump -lelf` 40 cubins ALL `sm_121a`, zero `sm_75`; SACRED `test_qwen27_paged_engine` **235/235 exit 0**, and the build precondition proven to FIRE by recompiling only that TU without the two defines against the same `libvllm.a`, which throws and exits 1 with 0 assertions). **The row PASSES UNCHANGED:** `tests/parity/test_qwen35_gguf_spec_decode` **3/3 cases, 10/10 assertions, exit 0**, spec-ON token-identical to spec-OFF, **13 proposed / 11 accepted (identical to the recorded number)**, 90.26 GiB, 7m13.59s; loader gate 19 assertions on the Qwen3.5-2B and 18 on the 35B A3B, unchanged. **ONE recorded finding is RETRACTED by the re-measurement: the CPU-vs-GPU token delta was a BUILD artifact, not a device near-tie cascade.** On the production build both devices emit the SAME 24 tokens; the probe shows GPU rank1 `11` -0.763897 over rank2 `13` -0.824083 where the defective build had rank1 `13` -0.773180 over rank2 `11` -0.847055, while the CPU arm is bit-identical to the earlier measurement (CUTLASS and Triton are CUDA-only). Zero exact ties in either arm, min margins 0.060186 GPU / 0.064875 CPU, 484/484 assertions per arm. Evidence: [docs/BENCHMARKS.md](../docs/BENCHMARKS.md) top section, [parity-ledger.md](parity-ledger.md) | [specs/gguf-mtp-spec-decode.md](specs/gguf-mtp-spec-decode.md) | `DONE` | `edf91449` | -| `SPEC-DFLASH-GGUF` | DFlash speculative decoding from GGUF, two axes: (A) GGUF DRAFT + safetensors target, (B) GGUF target too. llama.cpp master carries a full `dflash` GGUF contract (arch string `dflash`, tensors `fc`/`enc.output_norm`/`output_norm`/`blk.N.*`, KVs `dflash.target_layers` + `dflash.target_hidden_size`); the arch is ABSENT from checkouts older than ~2026-07, so a stale tree reads as "no contract exists". The GGUF tensor set omits `token_embd`/`output` because the draft SHARES the target's embed+lm_head, which is exactly what `LoadDflashDraft` already does. Blockers are in the loader, not the model: `MakeDflashDraftConfig` reads `draft_dir/config.json` (a GGUF has none), `ResolveDflashDraftDir` probes for `config.json` so it cannot see a `.gguf`, and `LoadDflashDraft` is typed on `std::vector` for the shared bf16 head (the axis-B blocker). Axis A independently shippable. NO ABI change | T2 | llama.cpp `origin/master` @ 2026-07-28 (tag era `b10158`): `gguf-py/gguf/constants.py:547,1151,4350`; `gguf-py/gguf/tensor_mapping.py:1297-1305` (`ENC_OUTPUT_NORM`<-`model.hidden_norm`, `FC`<-`model.fc`); `conversion/qwen.py:351` (mask token via the standard tokenizer KV); `convert_hf_to_gguf.py --target-model-dir` | **GD1-GD7 LANDED 2026-07-28 (BOTH AXES COMPLETE and PROVEN end to end on GB10)**: `MakeDflashGgufConfig` + `LoadQwen3DFlashFromGguf` `src/vllm/model_executor/models/qwen3_dflash_gguf.cpp:88,227` (+ header), `IsDflashGgufDraft` + the `.gguf` branch in `ResolveDflashDraftDir`/`LoadDflashDraft` `src/vllm/entrypoints/model_loader.cpp:121,222`. Goes through the `TensorResolver` seam (unlike `SPEC-MTP-GGUF`) because dflash norms are RAW, so the existing `LoadQwen3DFlash` qkv/gate_up concatenation is reused unchanged. **`GD4` defect FIXED** (`model_loader.cpp:238-249`): the GGUF branch left `config.vocab_size` 0 - correct for `MakeDflashGgufConfig` (the DFLASH arch has no vocab KV and no `token_embd`) but fatal for the forward, which sizes the shared embedding view as `{config.vocab_size, H}`, so the first propose threw `cuda embedding: empty table (vocab 0)`. Now back-filled from the target's `embed_tokens` rows (the condition is on the VALUE, not the draft source, so it generalizes to a GGUF target). Load-level green had hidden it; only GENERATING found it. **GD5-GD7 = axis B**: `SharedHeadSource` `src/vllm/entrypoints/model_loader.cpp` re-expresses the shared bf16 `embed_tokens`+`lm_head` seam as a SOURCE and re-types `LoadDflashDraft`'s second parameter - THAT TYPE was the whole axis-B blocker - with the GGUF arm `LoadGgufSharedEmbedAndHeadBf16` `src/vllm/model_executor/models/qwen3_5_gguf_weights.cpp:1044` reusing the trunk loader's tied-embedding rule and sidecar-aware dequant instead of restating them; the shared-head load moved into ONE common tail so all four (draft format x target container) combinations run identical code; the `dflash` half of the GGUF-branch rejection `model_loader.cpp` is deleted (the `mtp` half untouched) and the draft load is wired into the GGUF branch | `tests/vllm/models/test_qwen3_dflash_gguf.cpp:36,84` 2 cases / 47 assertions against the REAL published Qwen3.6-27B DFlash draft (env-gated `VLLM_DFLASH_GGUF_MODEL`, CI asset-free): the +1 target-layer offset undone against the KV read back from the same file, block_size/mask_token present, vocab_size left 0, layer_types cover every block, fc `[H, H*num_taps]` with `nk` SET, qkv/gate_up row-concat shapes, embed/lm_head left EMPTY for the target. **RED-first BEHAVIOURAL** (dropping the `-1` fails the offset checks). **`GD4` e2e gate** `tests/parity/test_qwen27_dflash_spec_decode.cpp:343` (second case, draft source env-driven via `VLLM_DFLASH_DRAFT`/`_B`; asset-gated, CI-inert): on dgx GB10 sm_121a against the Qwen3.6-27B NVFP4 safetensors target, the Q4_K_M GGUF draft and the bf16 z-lab safetensors draft produce **token-for-token IDENTICAL** DFlash-ON continuations with **IDENTICAL** accepted/proposed (20/80 on a 24-token prompt, 42/96 on a 48-token prompt), spec-OFF self-reproducible 3/3 and 0 exact ties (min margin 0.197/0.400 nats). Regression: gguf_mtp 19, qwen35_gguf_spec_decode 10, gguf 103, gguf_qwen36_loader 99, gguf_keep_quant 5958, ops_gdn 1825, llm_engine 196, capi 232, runner 257 all unchanged. **`GD5` unit gate** `tests/vllm/test_gguf_qwen36_loader.cpp` 3 new synthetic-GGUF cases (6 cases / 286 assertions total, CPU and the dgx CUDA build): the untied head really comes from `output.weight` and not the embedding (distinct fill values), the tied fallback aliases it onto `token_embd`, the `nk` flags separate the gather table from the MatmulBT weight, a file with no `token_embd` is refused. 3-mutant battery, 3 caught (`nk` flipped, head forced to the embedding, tied forced false). **`GD7` e2e gate** `tests/parity/test_qwen27_dflash_spec_decode.cpp` third case (targets env-driven via `VLLM_DFLASH_TARGET_B`; asset-gated, CI-inert): on dgx GB10 sm_121a the Qwen3.6-27B NVFP4 **GGUF** target + `Q4_K_M` GGUF draft loads, takes the shared head from the GGUF, generates, and its DFlash-ON continuation is **token-for-token IDENTICAL to that same target's spec-OFF** (24/24, the STRICT form) with acceptance ALIVE at 14/160; 1 case / 15 assertions, exit 0. **The spike's highest risk is EMPTY on this asset, proven not assumed**: the 27B NVFP4 GGUF stores `token_embd`/`output` as ggml BF16, byte-identical to the safetensors sibling (2,542,796,800 bytes each, ZERO differing), so B1's shared-head read is verbatim, not a dequant. Acceptance IS lower than the safetensors-target arm and is NOT chargeable to the head: the two containers diverge at index 4 with NO speculation, because `QUANT-GGUF-NVFP4` is dequant-only so the GGUF target computes in bf16 while the safetensors target runs the true W4A4 kernels. **RE-MEASURED 2026-07-29 on a PRODUCTION-CONFIGURED build (`CLAIM-GGUF-SPEC-REVERIFY`), because every GD4/GD7 GPU number above came from a build configured WITHOUT `-DVLLM_CPP_CUTLASS_DIR` and WITHOUT `-DVLLM_CPP_TRITON=ON`.** Build proven correct three ways (see the `SPEC-MTP-GGUF` row; SACRED 27B **235/235**, `cuobjdump` 40 cubins all `sm_121a`). **AXIS B HOLDS EXACTLY**: `test_qwen27_dflash_spec_decode -tc="dflash axis-B*"` **15/15 assertions, exit 0**, GGUF-target DFlash-ON token-identical to that target's own spec-OFF 24/24, acceptance **14/160 unchanged**, cross-target spec-OFF divergence still at index 4, 81.01 GiB peak RSS, 6m53.08s. **AXIS A WAS RED ON THE 48-TOKEN PROMPT (reproducibly, 3 of 3 runs) AND IS NOW CLOSED.** The RED was real: cross-format TOKEN identity held on both prompts, but the exact accept-count half of bar (a) failed (`arm_a.proposed == arm_b.proposed` / `arm_a.accepted == arm_b.accepted`) because the Q4_K_M draft measured **46/112** against the bf16 z-lab draft's **47/96** (one extra 16-wide propose block, one fewer acceptance, zero token difference), 15/17, exit 1; the 24-token prompt stayed green at 17/17 with both drafts at 15/144. **`GD9` 2026-07-29 root-caused it IN WEIGHT SPACE as ordinary `Q4_K_M` cost, category (a), not a defect in our GGUF draft path - and the bar's own premise ("Same weights, two containers") was false for the asset it was pointed at.** The publishing repo also carries an UNQUANTIZED `BF16` GGUF (3,471,497,440 B) beside `Q8_0`/`Q6_K`/`Q5_K`/`Q4_K_M`, which the spec had recorded as nonexistent; that retired the `NOT APPLICABLE` on gate 2. CPU gate `tests/vllm/models/test_qwen3_dflash_gguf.cpp` third case (asset-gated `VLLM_DFLASH_GGUF_BF16_MODEL` + `VLLM_DFLASH_ST_DIR`): `LoadQwen3DFlashFromGguf(BF16)` is **BYTE-IDENTICAL to `LoadQwen3DFlash(z-lab shards)` on all 58 tensors, 302/302 assertions, exit 0**, and FUNCTIONALLY RED against the `Q4_K_M` file (21/302 red, exactly the 21 quantized matmul tensors), so not a vacuous pass. Supporting: our `DequantGgufRowToBf16` is bit-equal to `gguf-py`'s `gguf.quants.dequantize` on the real `fc.weight` (Q4_K), `blk.0.attn_q.weight` (Q4_K) and `blk.2.ffn_down.weight` (Q6_K), zero differing bf16 values; the ladder's mean relative weight error is monotone and uniform with NO outlier tensor (BF16 0, Q8_0 5.6e-3, Q6_K 1.85e-2, Q5_K 3.85e-2, Q4_K_M 7.6e-2); the only numeric config delta is `rms_norm_eps` at 2.5e-9 relative. Also landed: an off-by-default `VT_SPEC_TRACE=1` per-block propose/accept trace in `GPUModelRunner::sample_tokens_with_rejection` (`src/vllm/v1/worker/gpu/runner.cpp`). **`GD10` 2026-07-29 CONFIRMED IT END TO END ON GB10 and closed gates 3 and 5.** Build proven production-configured three ways (configure log 0 `CUTLASS not found`; `cuobjdump -lelf` 40 cubins ALL `sm_121a` zero `sm_75` on both binaries; SACRED `test_qwen27_paged_engine` **235/235, exit 0**, 31.34s, 23.67 GiB). The **`BF16` GGUF draft reads EXACTLY 47/96**, the safetensors draft's own number, at 48 tokens on the discriminating prompt - reproduced 2 of 2 - plus 27/64 = 27/64 at 24 tokens and 15/144 = 15/144 on the second prompt, tokens IDENTICAL throughout, 17/17 exit 0 each time; the `Q4_K_M` arm reads 46/112 on the SAME binary in the SAME `flock` series. Restoring only the draft's numeric precision restores the count, so quantization is the whole cause and nothing structural survives. Bar (a) is consequently SPLIT rather than relaxed (`tests/parity/test_qwen27_dflash_spec_decode.cpp`): tokens stay EXACT unconditionally; accept counts are EXACT on a cross-FORMAT arm and BANDED (`abs(d_accepted) <= 2`, `abs(d_proposed) <= k*2`) on a cross-QUANTIZATION one, with the arm chosen by `IsQuantizedGgufDraft` reading the draft file's ggml types (`GgmlTraits().block_elems > 1`) rather than by a flag. The band is derived, not picked: measured `d_accepted` is 0, 0, -1, so the bound is that maximum plus one quantum; and `d_proposed = -k * d_accepted` EXACTLY once the token streams match (confirmed at -1 / +16), so the proposed bound follows. **Mutation-proved non-vacuous**: rebuilt at band 0 the `Q4_K_M` arm is 15/17 exit 1 while the `BF16` arm stays 17/17 exit 0 on the exact branch. **AXIS B BROADENED from ONE prompt to THREE**, strict form green on all: "The capital of France is" IDENTICAL 14/160 (15/15), "Write a Python function that reverses a string:" IDENTICAL 24/64 (15/15), "Photosynthesis is the process by which" IDENTICAL 15/128 (9/9), all exit 0, ~6m30-6m52 and ~81 GiB peak RSS each. The second prompt REFINES the recorded acceptance claim: the safetensors-target arm is ALSO 24/64 there with the two containers' DFlash-ON streams IDENTICAL, so the GGUF target's lower acceptance is prompt-dependent (their spec-OFF streams diverge at index 4 on the first prompt, index 16 on the second) and not a standing penalty; the cause remains `QUANT-GGUF-NVFP4` being dequant-only, with the shared head excluded by a byte comparison. Gates 1-5 and 7 MET; gate 6 (speed) `PENDING` BY DESIGN and not owed - a DFlash-ON throughput A/B between the two target containers is not a fair comparison until a native NVFP4 GGUF GEMM exists. Evidence: [docs/BENCHMARKS.md](../docs/BENCHMARKS.md) top section, [parity-ledger.md](parity-ledger.md#L845) | [specs/gguf-dflash-draft.md](specs/gguf-dflash-draft.md) | `DONE` | `c62f2fa3` | +| `SPEC-DFLASH-GGUF` | DFlash speculative decoding from GGUF, two axes: (A) GGUF DRAFT + safetensors target, (B) GGUF target too. llama.cpp master carries a full `dflash` GGUF contract (arch string `dflash`, tensors `fc`/`enc.output_norm`/`output_norm`/`blk.N.*`, KVs `dflash.target_layers` + `dflash.target_hidden_size`); the arch is ABSENT from checkouts older than ~2026-07, so a stale tree reads as "no contract exists". The GGUF tensor set omits `token_embd`/`output` because the draft SHARES the target's embed+lm_head, which is exactly what `LoadDflashDraft` already does. Blockers are in the loader, not the model: `MakeDflashDraftConfig` reads `draft_dir/config.json` (a GGUF has none), `ResolveDflashDraftDir` probes for `config.json` so it cannot see a `.gguf`, and `LoadDflashDraft` is typed on `std::vector` for the shared bf16 head (the axis-B blocker). Axis A independently shippable. NO ABI change | T2 | llama.cpp `origin/master` @ 2026-07-28 (tag era `b10158`): `gguf-py/gguf/constants.py:547,1151,4350`; `gguf-py/gguf/tensor_mapping.py:1297-1305` (`ENC_OUTPUT_NORM`<-`model.hidden_norm`, `FC`<-`model.fc`); `conversion/qwen.py:351` (mask token via the standard tokenizer KV); `convert_hf_to_gguf.py --target-model-dir` | **GD1-GD7 LANDED 2026-07-28 (BOTH AXES COMPLETE and PROVEN end to end on GB10)**: `MakeDflashGgufConfig` + `LoadQwen3DFlashFromGguf` `src/vllm/model_executor/models/qwen3_dflash_gguf.cpp:88,227` (+ header), `IsDflashGgufDraft` + the `.gguf` branch in `ResolveDflashDraftDir`/`LoadDflashDraft` `src/vllm/entrypoints/model_loader.cpp:121,222`. Goes through the `TensorResolver` seam (unlike `SPEC-MTP-GGUF`) because dflash norms are RAW, so the existing `LoadQwen3DFlash` qkv/gate_up concatenation is reused unchanged. **`GD4` defect FIXED** (`model_loader.cpp:238-249`): the GGUF branch left `config.vocab_size` 0 - correct for `MakeDflashGgufConfig` (the DFLASH arch has no vocab KV and no `token_embd`) but fatal for the forward, which sizes the shared embedding view as `{config.vocab_size, H}`, so the first propose threw `cuda embedding: empty table (vocab 0)`. Now back-filled from the target's `embed_tokens` rows (the condition is on the VALUE, not the draft source, so it generalizes to a GGUF target). Load-level green had hidden it; only GENERATING found it. **GD5-GD7 = axis B**: `SharedHeadSource` `src/vllm/entrypoints/model_loader.cpp` re-expresses the shared bf16 `embed_tokens`+`lm_head` seam as a SOURCE and re-types `LoadDflashDraft`'s second parameter - THAT TYPE was the whole axis-B blocker - with the GGUF arm `LoadGgufSharedEmbedAndHeadBf16` `src/vllm/model_executor/models/qwen3_5_gguf_weights.cpp:1065` reusing the trunk loader's tied-embedding rule and sidecar-aware dequant instead of restating them; the shared-head load moved into ONE common tail so all four (draft format x target container) combinations run identical code; the `dflash` half of the GGUF-branch rejection `model_loader.cpp` is deleted (the `mtp` half untouched) and the draft load is wired into the GGUF branch | `tests/vllm/models/test_qwen3_dflash_gguf.cpp:36,84` 2 cases / 47 assertions against the REAL published Qwen3.6-27B DFlash draft (env-gated `VLLM_DFLASH_GGUF_MODEL`, CI asset-free): the +1 target-layer offset undone against the KV read back from the same file, block_size/mask_token present, vocab_size left 0, layer_types cover every block, fc `[H, H*num_taps]` with `nk` SET, qkv/gate_up row-concat shapes, embed/lm_head left EMPTY for the target. **RED-first BEHAVIOURAL** (dropping the `-1` fails the offset checks). **`GD4` e2e gate** `tests/parity/test_qwen27_dflash_spec_decode.cpp:343` (second case, draft source env-driven via `VLLM_DFLASH_DRAFT`/`_B`; asset-gated, CI-inert): on dgx GB10 sm_121a against the Qwen3.6-27B NVFP4 safetensors target, the Q4_K_M GGUF draft and the bf16 z-lab safetensors draft produce **token-for-token IDENTICAL** DFlash-ON continuations with **IDENTICAL** accepted/proposed (20/80 on a 24-token prompt, 42/96 on a 48-token prompt), spec-OFF self-reproducible 3/3 and 0 exact ties (min margin 0.197/0.400 nats). Regression: gguf_mtp 19, qwen35_gguf_spec_decode 10, gguf 103, gguf_qwen36_loader 99, gguf_keep_quant 5958, ops_gdn 1825, llm_engine 196, capi 232, runner 257 all unchanged. **`GD5` unit gate** `tests/vllm/test_gguf_qwen36_loader.cpp` 3 new synthetic-GGUF cases (6 cases / 286 assertions total, CPU and the dgx CUDA build): the untied head really comes from `output.weight` and not the embedding (distinct fill values), the tied fallback aliases it onto `token_embd`, the `nk` flags separate the gather table from the MatmulBT weight, a file with no `token_embd` is refused. 3-mutant battery, 3 caught (`nk` flipped, head forced to the embedding, tied forced false). **`GD7` e2e gate** `tests/parity/test_qwen27_dflash_spec_decode.cpp` third case (targets env-driven via `VLLM_DFLASH_TARGET_B`; asset-gated, CI-inert): on dgx GB10 sm_121a the Qwen3.6-27B NVFP4 **GGUF** target + `Q4_K_M` GGUF draft loads, takes the shared head from the GGUF, generates, and its DFlash-ON continuation is **token-for-token IDENTICAL to that same target's spec-OFF** (24/24, the STRICT form) with acceptance ALIVE at 14/160; 1 case / 15 assertions, exit 0. **The spike's highest risk is EMPTY on this asset, proven not assumed**: the 27B NVFP4 GGUF stores `token_embd`/`output` as ggml BF16, byte-identical to the safetensors sibling (2,542,796,800 bytes each, ZERO differing), so B1's shared-head read is verbatim, not a dequant. Acceptance IS lower than the safetensors-target arm and is NOT chargeable to the head: the two containers diverge at index 4 with NO speculation, because `QUANT-GGUF-NVFP4` is dequant-only so the GGUF target computes in bf16 while the safetensors target runs the true W4A4 kernels. **RE-MEASURED 2026-07-29 on a PRODUCTION-CONFIGURED build (`CLAIM-GGUF-SPEC-REVERIFY`), because every GD4/GD7 GPU number above came from a build configured WITHOUT `-DVLLM_CPP_CUTLASS_DIR` and WITHOUT `-DVLLM_CPP_TRITON=ON`.** Build proven correct three ways (see the `SPEC-MTP-GGUF` row; SACRED 27B **235/235**, `cuobjdump` 40 cubins all `sm_121a`). **AXIS B HOLDS EXACTLY**: `test_qwen27_dflash_spec_decode -tc="dflash axis-B*"` **15/15 assertions, exit 0**, GGUF-target DFlash-ON token-identical to that target's own spec-OFF 24/24, acceptance **14/160 unchanged**, cross-target spec-OFF divergence still at index 4, 81.01 GiB peak RSS, 6m53.08s. **AXIS A WAS RED ON THE 48-TOKEN PROMPT (reproducibly, 3 of 3 runs) AND IS NOW CLOSED.** The RED was real: cross-format TOKEN identity held on both prompts, but the exact accept-count half of bar (a) failed (`arm_a.proposed == arm_b.proposed` / `arm_a.accepted == arm_b.accepted`) because the Q4_K_M draft measured **46/112** against the bf16 z-lab draft's **47/96** (one extra 16-wide propose block, one fewer acceptance, zero token difference), 15/17, exit 1; the 24-token prompt stayed green at 17/17 with both drafts at 15/144. **`GD9` 2026-07-29 root-caused it IN WEIGHT SPACE as ordinary `Q4_K_M` cost, category (a), not a defect in our GGUF draft path - and the bar's own premise ("Same weights, two containers") was false for the asset it was pointed at.** The publishing repo also carries an UNQUANTIZED `BF16` GGUF (3,471,497,440 B) beside `Q8_0`/`Q6_K`/`Q5_K`/`Q4_K_M`, which the spec had recorded as nonexistent; that retired the `NOT APPLICABLE` on gate 2. CPU gate `tests/vllm/models/test_qwen3_dflash_gguf.cpp` third case (asset-gated `VLLM_DFLASH_GGUF_BF16_MODEL` + `VLLM_DFLASH_ST_DIR`): `LoadQwen3DFlashFromGguf(BF16)` is **BYTE-IDENTICAL to `LoadQwen3DFlash(z-lab shards)` on all 58 tensors, 302/302 assertions, exit 0**, and FUNCTIONALLY RED against the `Q4_K_M` file (21/302 red, exactly the 21 quantized matmul tensors), so not a vacuous pass. Supporting: our `DequantGgufRowToBf16` is bit-equal to `gguf-py`'s `gguf.quants.dequantize` on the real `fc.weight` (Q4_K), `blk.0.attn_q.weight` (Q4_K) and `blk.2.ffn_down.weight` (Q6_K), zero differing bf16 values; the ladder's mean relative weight error is monotone and uniform with NO outlier tensor (BF16 0, Q8_0 5.6e-3, Q6_K 1.85e-2, Q5_K 3.85e-2, Q4_K_M 7.6e-2); the only numeric config delta is `rms_norm_eps` at 2.5e-9 relative. Also landed: an off-by-default `VT_SPEC_TRACE=1` per-block propose/accept trace in `GPUModelRunner::sample_tokens_with_rejection` (`src/vllm/v1/worker/gpu/runner.cpp`). **`GD10` 2026-07-29 CONFIRMED IT END TO END ON GB10 and closed gates 3 and 5.** Build proven production-configured three ways (configure log 0 `CUTLASS not found`; `cuobjdump -lelf` 40 cubins ALL `sm_121a` zero `sm_75` on both binaries; SACRED `test_qwen27_paged_engine` **235/235, exit 0**, 31.34s, 23.67 GiB). The **`BF16` GGUF draft reads EXACTLY 47/96**, the safetensors draft's own number, at 48 tokens on the discriminating prompt - reproduced 2 of 2 - plus 27/64 = 27/64 at 24 tokens and 15/144 = 15/144 on the second prompt, tokens IDENTICAL throughout, 17/17 exit 0 each time; the `Q4_K_M` arm reads 46/112 on the SAME binary in the SAME `flock` series. Restoring only the draft's numeric precision restores the count, so quantization is the whole cause and nothing structural survives. Bar (a) is consequently SPLIT rather than relaxed (`tests/parity/test_qwen27_dflash_spec_decode.cpp`): tokens stay EXACT unconditionally; accept counts are EXACT on a cross-FORMAT arm and BANDED (`abs(d_accepted) <= 2`, `abs(d_proposed) <= k*2`) on a cross-QUANTIZATION one, with the arm chosen by `IsQuantizedGgufDraft` reading the draft file's ggml types (`GgmlTraits().block_elems > 1`) rather than by a flag. The band is derived, not picked: measured `d_accepted` is 0, 0, -1, so the bound is that maximum plus one quantum; and `d_proposed = -k * d_accepted` EXACTLY once the token streams match (confirmed at -1 / +16), so the proposed bound follows. **Mutation-proved non-vacuous**: rebuilt at band 0 the `Q4_K_M` arm is 15/17 exit 1 while the `BF16` arm stays 17/17 exit 0 on the exact branch. **AXIS B BROADENED from ONE prompt to THREE**, strict form green on all: "The capital of France is" IDENTICAL 14/160 (15/15), "Write a Python function that reverses a string:" IDENTICAL 24/64 (15/15), "Photosynthesis is the process by which" IDENTICAL 15/128 (9/9), all exit 0, ~6m30-6m52 and ~81 GiB peak RSS each. The second prompt REFINES the recorded acceptance claim: the safetensors-target arm is ALSO 24/64 there with the two containers' DFlash-ON streams IDENTICAL, so the GGUF target's lower acceptance is prompt-dependent (their spec-OFF streams diverge at index 4 on the first prompt, index 16 on the second) and not a standing penalty; the cause remains `QUANT-GGUF-NVFP4` being dequant-only, with the shared head excluded by a byte comparison. Gates 1-5 and 7 MET; gate 6 (speed) `PENDING` BY DESIGN and not owed - a DFlash-ON throughput A/B between the two target containers is not a fair comparison until a native NVFP4 GGUF GEMM exists. Evidence: [docs/BENCHMARKS.md](../docs/BENCHMARKS.md) top section, [parity-ledger.md](parity-ledger.md#L845) | [specs/gguf-dflash-draft.md](specs/gguf-dflash-draft.md) | `DONE` | `c62f2fa3` | | `SPEC-REJECTION` | Rejection sampler. **I3 verify half LANDED (2026-07-24)**: per-request logits EXPANSION to `1 + k_i` rows (`StepInputs::cu_num_logits` / `num_draft_tokens_per_req` / expanded `logits_indices`) plus the GREEDY rejection sampler — accept a draft iff it equals the target argmax at its own position, emit the target argmax on the FIRST mismatch and stop, emit the bonus argmax when all `k_i` accept, `num_sampled = accepted + 1`, `num_rejected = k_i - accepted` (feeds I2's `num_computed_tokens` rollback and `InputBatch::num_accepted_tokens`). One additive vt op (`kGreedyRejectionSample`) with a CPU reference and a CUDA two-phase mirror of upstream's row-argmax + one-thread-per-request accept walk. DEFAULT-OFF and INERT: with no `SpeculativeConfig` no drafts are ever scheduled, `cu_num_logits` is `arange(num_reqs+1)`, `logits_indices` is the pre-change array and the runner never enters the rejection branch. STOCHASTIC/Gumbel, block verification, `apply_sampling_params` over the expanded batch, and the spec grammar bitmask stay DEFERRED (M-mtp-3). **I5b DRAFTER PREFILL INPUT-PREP LANDED (2026-07-24, `CLAIM-SPEC-MTP-I5B`)**: the draft-token input splice this row's I3 note deferred to I5 — `vllm::v1::prepare_prefill_inputs` + its `SpecPrefillInputs` output struct shift each request's `input_ids` left one within its query span, splice the just-sampled next token (`num_sampled>0 ? last_sampled[idx_mapping[r]] : next_prefill_tokens[...]`) into the freed slot, `query_len -= num_rejected`, and emit last-token index / query_start_loc / seq_lens + CG padding (mirror `speculator.py:469-588`, k=1 early-exit :236-238). A HOST routine in a NEW spec_decode-tree TU (no new CUDA kernel; mirrors the DEVICE-NEUTRAL `prepare_inputs`/`combine_sampled_and_draft_tokens` family — the DGX runner leaf ports the loop to the Triton kernel at I5d), unit-gated `test_prepare_prefill_inputs` 7 cases / 27 assertions RED-first, DEFAULT-OFF INERT (nothing calls it until I5d), additive by construction. Row stays `ACTIVE` — the e2e greedy token gate (M-mtp-1) is owed before `DONE` | T1 | `vllm/v1/worker/gpu/spec_decode/rejection_sampler.py:43,101-160`; `rejection_sampler_utils.py:524,564-585,628,828-841,846-849,863-1125`; `vllm/v1/worker/gpu/model_runner.py:866-898,1065-1077`; `vllm/v1/worker/gpu/input_batch.py:303-397,408-453`; **I5b** `vllm/v1/worker/gpu/spec_decode/autoregressive/speculator.py:469-588,236-238` | `include/vllm/v1/spec_decode/rejection_sampler.h`; `src/vllm/v1/spec_decode/rejection_sampler.cpp`; `include/vt/ops.h` (`kGreedyRejectionSample`, `vt::GreedyRejectionSample`); `src/vt/cpu/cpu_sample.cpp` (CPU reference); `src/vt/cuda/cuda_sample.cu` (`RejectionRowArgmaxKernel` + `GreedyRejectAcceptKernel`); `src/vt/ops.cpp`; `include/vllm/v1/worker/gpu/prepare_inputs.h` + `src/vllm/v1/worker/gpu/prepare_inputs.cpp` (the expansion); `include/vllm/v1/worker/gpu/runner.h` + `src/vllm/v1/worker/gpu/runner.cpp` (`step_num_logits`, `sample_tokens_with_rejection`); **I5b** `include/vllm/v1/worker/gpu/spec_decode/autoregressive/prepare_prefill_inputs.h` + `src/vllm/v1/worker/gpu/spec_decode/autoregressive/prepare_prefill_inputs.cpp` — anchor `include/vllm/v1/spec_decode/rejection_sampler.h:96` | `tests/vllm/v1/spec_decode/test_rejection_sampler.cpp`; `tests/vllm/v1/worker/test_prepare_inputs.cpp` (expansion + no-draft byte-identity); `tests/vt/test_cuda_ops.cpp` (CUDA==CPU bit-exact at vocab 248320); **I5b** `tests/vllm/v1/spec_decode/test_prepare_prefill_inputs.cpp` (7 cases / 27 assertions, RED-first) — anchor `tests/vllm/v1/spec_decode/test_rejection_sampler.cpp:128` | [mtp-spec-decode.md §2.4,§5](specs/mtp-spec-decode.md) | `ACTIVE` | `CLAIM-SPEC-REJECTION-I3`, `CLAIM-SPEC-MTP-I5B` | | `SPEC-GDN-SEGMENTS` | GDN speculative metadata and slot-snapshot rollback. **I4 LANDED (2026-07-24):** the spec/non-spec metadata split with decode→prefill reclassification (the #34845 case), the `T>1`/`IS_SPEC` GDN recurrence with per-timestep state snapshots, the conv sliding window advancing by the ACCEPTED count, and the k+1 state-slot allocation. DEFAULT-OFF and INERT (`num_spec==0` ⇒ `num_spec_decodes==0`, no shipped kernel branched — both spec kernels are NEW op ids). ROLLBACK PROVEN bit-exact: for every rejection point j the surviving SSM state and conv window are memcmp-identical to running only the accepted prefix through the shipped `vt::GdnDecode`/`CausalConv1dUpdate`, at the real 27B (Hv=48) and 35B (Hv=32) GDN dims on CPU and CUDA. MEASURED state cost: one f32 SSM slot = Hv·Dv·Dk·4B ⇒ 144 MiB/req (27B, 48 layers) / 60 MiB/req (35B, 30 layers) per extra slot; k=1 doubles the GDN SSM state. **I5a GDN LAYER ROUTING WIRED (2026-07-24, `CLAIM-SPEC-MTP-I5A`):** `GdnBlockPaged`'s `num_spec_decodes>0` branch now routes a PURE-spec batch through `vt::CausalConv1dSpecUpdate` + `vt::GdnSpecDecode` (mirror `qwen_gdn_linear_attn.py:1344-1357,1455-1475`), and the runner per-step upload (`StepDevInputs`/`BuildStepDevInputs` + the two decode-graph `Refresh` copies) now carries I4's six spec device tensors, gated by the extended `ValidateGdnAttentionMetadata` spec contract. DEFAULT-OFF INERT (`num_spec_decodes==0` ⇒ stub uploads + the identical non-spec branch). BIT-EXACT vs the I4 ops applied as a token-sequential decode chain, at the real 27B/35B GDN dims, via `GdnBlockPagedForTest` (`tests/vllm/models/test_qwen3_5_gdn_spec_routing.cpp`, CPU bit-exact + CUDA on-device); RED-first by a reverted stub (spec recurrence zeroed ⇒ 4/8 fail, maxΔ 1.3-1.6). MIXED spec+non-spec batch refused loudly — lands with I5d's runner loop. Row advances to `ACTIVE`: the M-mtp-1 e2e greedy token gate (verify/propose runner wiring) is owed before `DONE`, and `SPEC-MTP` STAYS `GATING` | T1 | `vllm/v1/attention/backends/gdn_attn.py:189-326,413-462`; `fla/ops/fused_sigmoid_gating.py:66-72,103-116,156-166`; `mamba/ops/causal_conv1d.py:818-1067,1181-1184`; `qwen_gdn_linear_attn.py:1329-1576`; `mamba_utils.py:213-234`; `mamba/abstract.py:55-59` | `include/vllm/v1/attention/backends/gdn_attn.h`; `src/vllm/v1/attention/backends/gdn_attn.cpp`; `include/vt/ops.h` (`kGdnSpecDecode`, `kCausalConv1dSpecUpdate`); `src/vt/ops.cpp`; `src/vt/cpu/cpu_ops.cpp`; `src/vt/cuda/cuda_gdn.cu`; `src/vllm/model_executor/models/qwen3_5_common.{h,cpp}` (`MakeQwen3_5KVCacheSpec`); **I5a:** `src/vllm/model_executor/models/qwen3_5.cpp` (`GdnBlockPaged` spec branch, `StepDevInputs`/`BuildStepDevInputs`, `ValidateGdnAttentionMetadata`), `src/vllm/model_executor/models/qwen3_5_internal.h` (`GdnBlockPagedForTest`) | `tests/vllm/v1/attention/test_gdn_metadata_builder.cpp` (20 cases / 483 assertions incl. the full upstream `GDN_BUILD_TEST_CASES` + default-off byte-identity); `tests/vt/test_ops_gdn.cpp` (reject-at-every-j rollback, CPU + CUDA, real dims); `tests/vllm/models/test_model_registry.cpp` (k+1 slot / widened-conv sizing + `num_spec==0` identity); **I5a** `tests/vllm/models/test_qwen3_5_gdn_spec_routing.cpp` (spec-routing bit-exact, RED-first) — anchor `tests/vllm/v1/attention/test_gdn_metadata_builder.cpp:83` | [mtp-spec-decode.md §3,§5](specs/mtp-spec-decode.md) | `ACTIVE` | `CLAIM-SPEC-GDN-I4`, `CLAIM-SPEC-MTP-I5A` | | `SPEC-DFLASH` | Block-diffusion drafter. **READINESS RE-ASSESSED 2026-07-25 (`CLAIM-SPEC-DFLASH-READINESS`, design-only, DONE) against the LANDED MTP machinery (`SPEC-MTP` I1..I7).** Verdict **GREEN, dispatch-ready, NO hardware/oracle/download blocker** (spec [§0](specs/dflash-spec-decode.md)). Refreshed reuse-vs-new map: DFlash gets FREE from landed MTP — the frozen spec-metadata ABI, the greedy rejection sampler (k-general, I3 tested k∈{1,3}), the GDN spec slot path + rollback + mixed spec/non-spec batch (`GdnBlockPagedMixedSpec`/`IndexSelect`/`IndexCopy`, general `num_spec`), the widened-cache-aware conv ops (I5e), the draft-KV layer pattern (`fa_draft`), the I5d/I7 runner verify/propose loop, and **`num_lookahead_tokens=k+1` ALREADY coded** (`speculative.h:91-108` `use_dflash()`); EXTENDS the single I5d-pre `hidden_tap` seam to multi-tap `[T,H×taps]`; builds NEW the `qwen3_dflash` drafter, the project's FIRST non-causal in-block attention primitive, context-KV precompute, `prepare_dflash_inputs`, and the uniform-1+k FULL CG. **k>1 verdict:** the landed rejection + GDN machinery is MECHANICALLY k-general (no `k==1` hardwiring) — DFlash's k=15 blocks need NO mechanism extension, only exercise/validation at scale (D4) + the k+1-slot memory measurement (~2.3 GiB/req 27B GDN state at block-16, the #1 risk, §5). **Checkpoint-fit:** both z-lab drafts EXIST on HF (27B 1.73 GB / 35B 368 MB bf16, DFlashDraftModel) and FIT the 119 GiB pool trivially (drafts NOT yet on dgx — D0 downloads ≤1.73 GB); the active dgx oracle `vllm-oracle-v0.25.0-stage` CONSTRUCTS DFlash (registry `DFlashDraftModel→qwen3_dflash`, speculator dir present) — soft D0 risk = confirm it SERVES DFlash+NVFP4 on sm_121 (non-causal backend; community `AEON-7/vllm-dflash` container proves the combination runs on GB10). W-plan D0-D6 in the spec. **D0+D1 LANDED 2026-07-26 (`CLAIM-DFLASH-D0D1`) on the ADVANCED pin `555967922`/vLLM 0.26.0.dev0 — `SPEC-DFLASH` → `ACTIVE`.** D0 UNBLOCKED (vllm#40898 resolved under `VLLM_USE_V2_MODEL_RUNNER=1`): the mixed-attn z-lab 27B draft CONSTRUCTS + the drafter is ALIVE (acceptance 2.21/8.80/4.75/4.57 > 1, `num_spec=16`, flashinfer-native fp8-KV, goldens committed); gate FORM measured STRICT MODE-MATCHED (vLLM-ON run-deterministic K>=3 but != vLLM-OFF — the k=16 block verify diverges at bf16 near-ties, so NOT the MTP three-way identity). D1 `DF-AUX-TAPS` DONE: `Qwen3_5AuxTaps` + `ModelForwardInput::aux_tap` route to `Qwen3_5{,Dense}Model::ForwardDeviceMultiTap` capturing `(hidden+res)` at `target_layer_ids` into `[T,H×taps]` (eagle3 `_maybe_add_hidden_state`, aux key L+1); config-gated byte-identical off. Unit gate 598 assertions (independent truncated-model reference, RED-first reversed-concat 384 fail); CUDA 697/697 + compute-sanitizer 0; INERTNESS PROVEN — 27B MTP e2e 9/9 + 27B text SACRED 235/235 byte-identical on the new oracle. **D2 `DF-DRAFT-MODEL` CODE LANDED + CPU-GATED 2026-07-26 (`CLAIM-DFLASH-D2`, kernel row `KERNEL-ATTN-DFLASH-BLOCK`):** the `qwen3_dflash` draft model (plain 5-layer Qwen3-dense reusing `dense_attn_block.h` ops), the project's FIRST non-causal / bidirectional attention primitive `vt::DFlashBlockAttention` (a SEPARATE op — causal `kAttention`/`kPagedAttention` byte-identical), the fc aux-combine, mask-embed, per-layer SWA/full resolution, and the z-lab loader. CPU gate GREEN (op 12/12 incl. RED non-causal; model forward 95/95 incl. RED full-layer-causal-flip + block isolation + fc RED); existing causal `test_ops_attention` 9/9 + `test_qwen3_forward` 1028 UNCHANGED. **D2 GPU PROMOTION GREEN on dgx (`CLAIM-DFLASH-D2`):** CUDA `-Werror` clean, CUDA==CPU 198412/198412 + compute-sanitizer 0, draft-forward parity vs the REAL vLLM draft (fc rel-L2 0.46%, hidden ≤1.3%, 11 STRICT + 5 near-tie ids), 27B SACRED 235/235 + MTP 9/9 byte-identical — **D2 DONE.** **D3 `DF-DRAFT-KV-PREP` DONE 2026-07-26 (`CLAIM-DFLASH-D3`):** `PrecomputeContextKV` + `PrepareDflashInputs` + `ForwardBlockLogitsWithContext` (reuse the UNCHANGED D2 kernel via [context;block]); GPU numeric-parity `test_qwen3_dflash_kvprep_parity` 61/61 (prepare INTEGER bit-exact vs vLLM's Triton kernel, context-KV K/V rel-L2 0.31%/0.26%, 13 STRICT + 3 near-tie = 16/16), CPU 114/114 RED-proven, inertness 235/235 + 9/9 + D2 37/37 byte-identical. **D4 `DF-ENGINE-INTEGRATION` propose brick + `dflash` config-select CODE LANDED + CPU-GATED 2026-07-26 (`CLAIM-DFLASH-D4D5`):** `DflashProposeBlock`/`SampleDflashBlockDrafts` (the non-autoregressive whole-block propose composing D3 `ForwardBlockLogitsWithContext` + greedy per-mask argmax, anchor not sampled, `dflash/speculator.py:300-413`) + `ParseSpeculativeConfigJson`/`ResolveDflash` accept `method:"dflash"`. CPU gate `test_dflash_propose` 5/19 GREEN (RED-first anchor-read fails 4/5; brick composes forward+sampler; empty-ctx degenerates to D2; config lookahead k+1). Additive + config-gated ⇒ MTP + non-spec byte-identical BY CONSTRUCTION (`git diff --stat` = new speculator TU + config accept-list + CMake + test, NO runner/model/loader/scheduler edit). **D5 `DF-ENGINE-INTEGRATION` runner-loop LANDED + e2e RUNS on dgx 2026-07-26 (`CLAIM-DFLASH-D5`):** full verify/propose loop wired — loader loads the SEPARATE z-lab draft (`LoadDflashDraft`, host bf16 + target-SHARED bf16 embed/lm_head) via a `--speculative-config` `model` key + `ResolveSpecConfig` dflash branch + `runner.set_dflash_draft`; the verify forward captures the D1 multi-tap (`aux_tap`→`ForwardDeviceMultiTap`) instead of the MTP single tap; `propose_drafts_dflash` ACCUMULATES the per-request combined-feature context (`CombineAuxFeatures(aux_tap)`) across steps and honors the `num_rejected` rollback by appending only the `(T_req−num_rejected)` accepted-prefix features, then runs `DflashProposeBlock` (k=16 GDN-spec exercised first time). **e2e (`test_qwen27_dflash_spec_decode`, 4 prompts×32 tok, our-DFlash-ON vs the committed vLLM-DFlash-ON golden): 2/4 STRICT token-exact (fibonacci, three-laws) + acceptance ~ vLLM on ALL 4 (accepted 19/39/29/25 vs golden 17/39/30/25, deltas +2/0/−1/0 — the MANDATORY dead-drafter-trap condition MET).** The 2 divergences (France tok11 `2972`↔`11751`, 17*23 tok12 `567`↔`488`) are SINGLE bf16 near-tie flips (17*23 RE-CONVERGES after one token = proven near-tie; France cascades from one flip) — the ratified near-tie ROOT the D0 gate-form anticipated, rooted in the D3-documented inline bf16 context-KV recompute envelope (~0.3-1.3% rel-L2), NOT a wiring bug (proven by the 2 exact prompts + near-exact acceptance + a non-trivial shared prefix). Inertness GREEN on this build: SACRED `test_qwen27_paged_engine` 235/235 + MTP `test_qwen27_spec_decode` 9/9 byte-identical; CUDA `-Werror` clean; NO new CUDA kernel (host orchestration reusing D1/D2/D3-sanitized ops). **NOT a clean strict-4/4 pass; STRICT 4/4 token-identity + the speed A/B = D6 (the persistent paged draft-KV bit-matching vLLM's fused context-KV projections + the uniform-1+k FULL CG).** Row STAYS `ACTIVE` (correctness at the ratified near-tie envelope; D6 remains) **D6 2026-07-27 (`CLAIM-DFLASH-D6`) — c1 SPEED A/B DONE + STRICT-irreducibility RCA + CG feasibility (records-only, NO source code):** (1) **c1 speed A/B** (`examples/vllm-bench` at `361189a7`, 8 prose+code prompts×256 tok greedy c1, 2 reps): our DFlash-ON = **2.50x TPOT (40.4 vs 101.2 ms) / 2.48x output-tput (24.4 vs 9.86 tok/s)** over our OFF, acceptance 0.22 (3.56/16), rep-stable <1.5%; `benchmark_binding=true`. vs vLLM-DFlash-ON graphed (same workload): vLLM-DFlash-ON graphed = 28.5 tok/s / 35.1 ms TPOT / acceptance_len 4.30 (same 8 prompts, `VLLM_USE_V2_MODEL_RUNNER=1`, mm-off, gpu_util 0.30), so OURS IS ~14% BELOW vLLM-DFlash-ON on output throughput (24.4 vs 28.5 tok/s) - both ~on-par at spec-OFF (9.86 vs 9.83 tok/s), but vLLM extracts a larger DFlash speedup (2.90x vs our 2.47x) because its draft step is fully device-resident + CUDA-graphed (ours host-orchestrates 13 downloads/step) + slightly higher acceptance (~4.3 vs ~3.6 draft tokens/step). The DONE speed bar (ours >= vLLM) is NOT met; closing it = the device-resident draft rewrite + FULL CG (D6 part 2). (2) **STRICT-4/4 proven bf16-IRREDUCIBLE** — the draft KV cache is bf16 not fp8 (`torch_utils.py:398` `auto`→model dtype; the D0 "fp8-KV" was the backend name, not the KV storage dtype), the D3 golden already compares pre-storage bf16 (residual K 0.31%/V 0.26% = sub-ULP kernel noise), and a fused multi-layer KV GEMM is per-element invariant to our per-layer GEMMs ⇒ bit-exact needs vLLM's exact kernels ⇒ the ratified near-tie gate is the FINAL correctness form (no fused-KV code landed). (3) **FULL CG BLOCKED** on a device-resident draft-path rewrite (the D5 path does 13 device→host downloads/step + host `[context;block]` interleaving) — the remaining throughput-parity increment (the perf form of persistent-paged-KV + the graph). Inertness by construction (the gated binary is the D5 binary; SACRED 235/235 + MTP 9/9 stand). Evidence tool `scripts/spec/vllm_dflash_timing.py`. **D7 2026-07-27 (`CLAIM-DFLASH-D7`) — within-step draft forward made DEVICE-RESIDENT (source-owning): `PrecomputeContextKVDevice` keeps per-layer K/V on device; `ForwardBlockLogitsWithContext` builds [context;block] with `vt::IndexCopy`/`IndexSelect` (removes ~30 D→H `Download`s/step). BIT-IDENTICAL (identity bf16↔f32 round-trips replaced) — e2e `test_qwen27_dflash_spec_decode` 27/27 SAME tokens (2/4 STRICT + 2/4 near-tie, acceptance 19/39/29/25), SACRED 235/235 + MTP 9/9, CUDA `-Werror` clean, compute-sanitizer 0 (198412). But the direct old-vs-new A/B = +2.0% output-tput (IN-NOISE) ⇒ D6's "downloads = the ~14% gap" REFUTED by measurement; ours 19.68 tok/s STILL ~33% BELOW vLLM-DFlash-ON 29.2 tok/s (reconstructed 8-prompt set, more prose-heavy); OFF parity our 9.97 ≥ vLLM 9.66. Residual re-attributed: acceptance (ours 2.49 vs vLLM ~3.13 accepted draft-tok/step, bf16-irreducible) + per-step context-KV RECOMPUTE (O(context²), needs the cross-step persistent paged draft-KV store) + eager-vs-graphed. SPEED BAR NOT met; SPEC-DFLASH stays `ACTIVE`; next = persistent paged draft-KV store → then FULL CG. **D9 2026-07-27 (`CLAIM-DFLASH-D9`) — PERSISTENT PAGED DRAFT-KV LANDED (bit-identical, +22.7% throughput, 0.69×→0.917×); D8 acceptance-ceiling REFUTED; residual = FULL CG ONLY:** `qwen3_dflash.cpp` `AppendContextKVHost` (project ONLY newly-accepted rows → per-layer bf16 K/V, append to `PrecomputedContextKV`) + `ForwardBlockLogitsWithPrecomputedKV` (upload the persistent store, NO re-projection) share the core `ForwardWithCtxKVDev` with the old recompute; `runner.cpp::propose_drafts_dflash` swaps the O(context²) per-step recompute (`dflash_ctx_feats_`) for an append-only per-request `dflash_kv_store_` (rollback=don't-append). NO new CUDA kernel; config-gated. BIT-IDENTICAL: CPU `test_dflash_propose` two new D9 cases = exact float equality vs full recompute; GPU e2e `test_qwen27_dflash_spec_decode` **27/27 SAME tokens** (acceptance 19/39/29/25, same divergences France@11/17×23@12); SACRED 235/235 + MTP 9/9 byte-identical; CUDA `-Werror` clean. **A/B (c1, 8 prose+code×256 tok input-len 512, 2 reps <0.1%, `benchmark_binding=true`):** ours-ON **25.75 tok/s** (was D8 20.99, +22.7%) / 38.40 ms TPOT / acc **3.68/step** vs vLLM-ON graphed **28.09** / 35.60 / acc 3.31 = **0.917×** (~8% below, was 0.69×). **Part 1 same-trajectory:** on the 2 token-identical-trajectory prompts ours per-step acceptance == vLLM's EXACTLY (fibonacci 7.80/7.80, three-laws 3.571/3.571, ratio 1.00) AND on the A/B ours acceptance (3.68) is HIGHER than vLLM's (3.31) ⇒ D8's 0.80–0.85× "bf16 acceptance ceiling" is a trajectory-divergence CONFOUND, REFUTED. Residual (~8%) = eager-vs-graphed ONLY (ours ON/OFF 2.60× vs vLLM 2.91×, OFF at parity, recompute eliminated, acceptance higher) — NOT an irreducible ceiling; the FULL uniform-(1+k) CG (device paged-KV store + paged attn, new-CUDA multi-file) is the SOLE un-landed increment. SPEC-DFLASH stays `ACTIVE` (speed not yet ≥ vLLM; residual isolated to FULL CG). **D12 2026-07-27 (`CLAIM-DFLASH-D12`) — A-wire + Part B LANDED + GPU-gated; Part C (capture) remaining; 0.917×:** A-wire makes the D11 Part-A device store the PRODUCTION path (`runner.{h,cpp}` `dflash_kv_store_`→`shared_ptr`, `MakeDeviceKVStore`/`AppendContextKVDevice`/`ForwardBlockLogitsWithDeviceKV`; GPU-gated e2e `test_qwen27_dflash_spec_decode` 27/27 all-exact acceptance 19/39/29/25 + SACRED 235/235 + MTP 9/9 byte-identical, `-Werror` clean). Part B adds `vt::DFlashPagedBlockAttention` (`OpId::kDFlashPagedBlockAttention`), the capture-safe paged kernel with EVERY metadata input a persistent DEVICE tensor and NO function-local host `cu_seqlens` upload (fixes the `cuda_ops.cu:1277-1280` capture-UAF class), gated CPU==CUDA + cross-check vs materialized `DFlashBlockAttention` `test_ops_dflash_paged_block_attn` 795648/795648 + compute-sanitizer 0. Speed 0.917× (A-wire eager + Part B not yet wired into the forward); `benchmark_binding=false`. Part C (static-shape capture + device mask-scatter + `BeginCapture`/replay + the ≥vLLM c1 A/B) is the SOLE remaining piece; if ours-ON-graphed ≥ vLLM-ON → SPEC-DFLASH DONE. Stays `ACTIVE`. **D13 2026-07-27 (`CLAIM-DFLASH-D13`) — Part C LANDED + GPU-GATED; capture-correctness PROVEN; c1 throughput NEAR-PARITY (ours 0.978x, ~2% below vLLM); gap CLOSED 0.917x→0.978x; STAYS `ACTIVE` (≥vLLM bar not yet met):** single-file additive change (`qwen3_dflash.cpp` +368/-58). (C.1) `DflashDeviceKVStore` → fixed-capacity PAGED cache (per-layer pool `[max_pages,16,Hkv,Dh]` + identity `block_table` + `seq_lens`; append = `vt::IndexCopy` scatter at slot==abs-pos, bit-identical to the D9/D11 store). (C.2) `ForwardPagedBody` runs the (1+k) block through the D12 `vt::DFlashPagedBlockAttention` reading the paged store (no `[context;block]` materialization, no function-local host uploads); runner P==1 propose routes through it, P>1 bit-identical materialized fallback. (C.3) per-request CUDA GRAPH over the paged draft step (warm-in-step repopulates the shared pool free-list right before `BeginCapture` — the fix for a `cudaMalloc`-in-capture `Get` miss from the intervening 27B target forward — then `BeginCapture → ForwardPagedBody → EndCaptureGraph`, replay with growing context entering only via in-place `seq_lens`). **Capture-correctness (MANDATORY): `test_qwen27_dflash_spec_decode` 27/27 with the graph (VT_DFLASH_GRAPH=1) BIT-IDENTICAL to eager (=0)** — same divergence tokens (France@11 got[…2972…], 17×23@12 got[…567…]), same acceptance 19/39/29/25 as D5/D7/D9/D12; graph ENGAGED (5 captures C=2048/5/4/15/6, 32+ replays); the token-diff is the capture-safety proof ([[cudagraph-capture-bakes-stack-addresses]]). **c1 A/B (one flock series, cold rep discarded, 8 prompts×256 tok):** our OFF 10.24 / our ON eager-paged 28.65 (28.69,28.61) / **our ON GRAPHED 28.70 (28.70,28.70), TPOT 34.40** / vLLM-ON graphed steady-state 29.35 (tight 3-rep 29.33/29.37/29.33, TPOT 34.07, acc_len 4.44); D9's 28.09 was a colder cross-session outlier — **NEAR-PARITY: ours 0.978× (~2% below) on the rigorous same-session band** (across sessions ours 28.70 falls inside vLLM's observed 28.09–29.37 range). ON/OFF 2.80× (vLLM ~2.98×), our OFF ≥ vLLM OFF. Per the acceptance rule ("below on any axis = an open gap; near-parity is NOT met"), the ≥vLLM bar is NOT met; STAYS `ACTIVE`. Residual (data-grounded): NOT acceptance (ours realized ~3.68 accepted draft-tok/step > vLLM's 3.44) and NOT launch/graph (both graphed, CG neutral) — per-step COMPUTE (~2% slower target-step); next lever = nsys both draft steps (`--cuda-graph-trace=node`), no premature ceiling. **ATTRIBUTION (supersedes D9):** the CUDA graph is perf-NEUTRAL (+0.3%); the ACTUAL lever was the paged context read (C.1/C.2) removing the D9/D12 per-layer `[context;block]` `IndexCopy` materialization of the whole growing context (25.75 D9 → 28.65 eager-paged, +11%) — the roadmap's "the full CG closes the gap" premise is corrected by measurement. Inertness VERIFIED on the capture binary: SACRED 235/235 + MTP 9/9 byte-identical, CUDA `-Werror` clean, no new kernel (D12 paged kernel already memcheck-0 795648), `check-device-leakage` not increased (paged path REMOVES the materialized-buffer allocs + host uploads). `benchmark_binding=true`. Correctness-complete (ratified near-tie); throughput NEAR-PARITY (0.978×, ~2% residual) ⇒ STAYS `ACTIVE` (the capture-correctness gate is MET; the ≥vLLM speed bar is the sole remaining item, a ~2% per-step-compute residual for an nsys). Anchors: `src/vllm/model_executor/models/qwen3_dflash.cpp` (`DflashDeviceKVStore` paged store, `ForwardPagedBody`, the per-request graph in `ForwardBlockLogitsWithDeviceKV`). **D14 2026-07-27 (`CLAIM-DFLASH-D14`) — SPEED GATE MET → SPEC-DFLASH `DONE`:** an nsys (`--cuda-graph-trace=node`) of the graphed spec-on step attributed the D13 ~2% residual to the from-scratch `DFlashPagedBlockAttentionKernel` draft attention (242.9 ms = 1.8% of GPU time, median ~460 us/call over context C~500-640, vs vLLM's fused flash draft-attn ~0.15%; BOTH engines run identical `cutlass_80_wmma` for the draft bf16 GEMMs, so the GEMMs were NOT the gap). Ported it to a WARP-scoped online-softmax variant `DFlashPagedBlockAttentionWarpKernel` (mirrors the shipped `AttentionWarpKernel`: one warp per (block-query,head), `__shfl_xor` butterfly reduction, register accumulator, NO `__syncthreads` storm; SAME paged/block combined-index read + causal/SWA mask + GQA; default ON, `VT_DFLASH_ATTN_BLOCK=1` keeps the bit-identical D12/D13 block kernel for A/B). Draft attn 242.9 → 77.9 ms (3.1x); our-ON c1 28.60 → 29.32 tok/s (+2.5%). **FINAL same-session 3-rep A/B (8 prompts×256 tok, cold leg discarded): our-ON graphed 29.42/29.27/29.32 (med 29.32) vs vLLM-ON graphed 29.240/29.247/29.233 (med 29.240) — our WORST rep (29.27) > vLLM's BEST (29.247), NON-OVERLAPPING bands, 1.003× ⇒ the ≥vLLM speed gate is MET.** Correctness UNCHANGED (output is exact by spec-decode construction — the target verify is untouched, only which draft proposals are accepted can shift): e2e `test_qwen27_dflash_spec_decode` 27/27 with graph==eager BIT-IDENTICAL, acceptance 19/39/29/25 unchanged (draft accepted 1629 identical warp-vs-block across the whole A/B set), 2/4 STRICT (France@11, 17×23@12 unchanged); CUDA==CPU `test_ops_dflash_paged_block_attn` 795648/795648 (warp within the f32 1e-4 / bf16 3e-2 envelope) + compute-sanitizer 0. Inertness SACRED 235/235 + MTP 9/9 byte-identical; CUDA `-Werror` clean; `check-device-leakage` not increased. `benchmark_binding=true`. Block-diffusion drafting is now correctness-complete (ratified near-tie) AND at/above vLLM throughput — this was the roadmap's FINAL open speed item. Anchors: `src/vt/cuda/cuda_ops.cu` (`DFlashPagedBlockAttentionWarpKernel` + `UseDflashAttnBlockKernel`; the D12 block kernel retained as the `VT_DFLASH_ATTN_BLOCK=1` reference). | T1 | `vllm/v1/worker/gpu/spec_decode/dflash/speculator.py`; `vllm/model_executor/models/qwen3_dflash.py`; `vllm/model_executor/models/interfaces.py:1382` (aux value); `eagle3_utils.py:41-56` (+1 shift) | `include/vllm/model_executor/models/qwen3_5.h` (`Qwen3_5AuxTaps`, `ForwardDeviceMultiTap`); `qwen3_5_dense.h`; `model_registry.h` (`aux_tap`); `src/vllm/model_executor/models/qwen3_5.cpp` (`MaybeCaptureAuxTap`/`ValidateAuxTapLayerIds`/`ForwardDeviceMultiTap`); `qwen3_5_moe.cpp`+`qwen3_5_dense.cpp` (routing); D2/D3 `include/vllm/model_executor/models/qwen3_dflash.h` + `src/vllm/model_executor/models/qwen3_dflash{,_weights}.cpp`; D4 `include/vllm/v1/worker/gpu/spec_decode/dflash/speculator.h` + `src/vllm/v1/worker/gpu/spec_decode/dflash/speculator.cpp` (`DflashProposeBlock`/`SampleDflashBlockDrafts`); D5 `src/vllm/entrypoints/model_loader.cpp` (`LoadDflashDraft`/`DflashDraft`) + `include/vllm/entrypoints/model_loader.h`; D5 `src/vllm/v1/worker/gpu/runner.cpp` (`set_dflash_draft`/`propose_drafts_dflash`/aux-tap capture) + `include/vllm/v1/worker/gpu/runner.h`; `src/vllm/config/speculative.cpp` + `include/vllm/config/speculative.h` (`ResolveDflash` + `dflash`/`model` parse); D14 warp kernel [cuda_ops.cu](../src/vt/cuda/cuda_ops.cu#L1433) | `tests/vllm/models/test_qwen27_paged_forward.cpp` (multi-tap 598); `tests/vt/test_ops_dflash_block_attn.cpp`; `tests/vllm/models/test_qwen3_dflash_forward.cpp`; `tests/vllm/v1/spec_decode/test_dflash_kvprep.cpp`; `tests/parity/test_qwen3_dflash_{draft,kvprep}_parity.cpp`; D4 `tests/vllm/v1/spec_decode/test_dflash_propose.cpp` (5/19, RED-first); D5 `tests/parity/test_qwen27_dflash_spec_decode.cpp` (e2e 27/27, 2/4 strict + acceptance~vLLM); `scripts/spec/d{0,2,3}_dflash_*.py`; `tests/parity/goldens/dflash_27b{,_draft,_kvprep}/`; D6 `scripts/spec/vllm_dflash_timing.py` (vLLM-DFlash c1 timing); D7 device-resident `src/vllm/model_executor/models/qwen3_dflash.cpp` (`PrecomputeContextKVDevice` + `ForwardBlockLogitsWithContext` via `vt::IndexCopy`/`IndexSelect`); D9 persistent paged draft-KV `qwen3_dflash.{h,cpp}` (`AppendContextKVHost`/`ForwardBlockLogitsWithPrecomputedKV`/`ForwardWithCtxKVDev`/`PrecomputedContextKV`) + `runner.{h,cpp}` (`dflash_kv_store_`/`propose_drafts_dflash`) + `tests/vllm/v1/spec_decode/test_dflash_propose.cpp` (2 D9 bit-identity cases); D12 A-wire `runner.{h,cpp}` (device store as production path) + D12 Part B `include/vt/ops.h`/`src/vt/ops.cpp`/`src/vt/cpu/cpu_ops.cpp`/`src/vt/cuda/cuda_ops.cu` (`kDFlashPagedBlockAttention`) + `tests/vt/test_ops_dflash_paged_block_attn.cpp` (CPU==CUDA + cross-check, 795648/795648 + sanitizer-0); D13 `src/vllm/model_executor/models/qwen3_dflash.cpp` (fixed-capacity paged `DflashDeviceKVStore` + `ForwardPagedBody` + the per-request draft-step CUDA graph in `ForwardBlockLogitsWithDeviceKV`); D14 [test_ops_dflash_paged_block_attn](../tests/vt/test_ops_dflash_paged_block_attn.cpp#L79) + [ledger](parity-ledger.md#L738) | [dflash-spec-decode.md](specs/dflash-spec-decode.md) | `DONE` | `489a7544` | From df45fa89faf472973969d0aa891dd8fb3471adf3 Mon Sep 17 00:00:00 2001 From: Luca Barbato Date: Sun, 13 Sep 2026 19:31:54 +0200 Subject: [PATCH 9/9] docs: document VT_TT_RELEASE_WARM_ROWS (#3042) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The warm-slot release knob was read from production code but not documented — the env-doc gate refuses exactly that. It is a user-facing memory lever (frees the warm pass's full-batch-shape activation slots after the cold pre-warm), so it documents rather than allowlists. FOLLOWING_AGENTS_PROTOCOL Following-Agents-Protocol: true AI-Assisted: true Assisted-by: AGENT:zai/glm-5.3-flash [maki] --- docs/ENVIRONMENT.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/ENVIRONMENT.md b/docs/ENVIRONMENT.md index 2f949a298..193823386 100644 --- a/docs/ENVIRONMENT.md +++ b/docs/ENVIRONMENT.md @@ -111,6 +111,8 @@ These change how the engine runs and have no CLI flag (or complement one). | `VT_SERVER_MAX_NEW_TOKENS` | `4096` | Caps a chat request's `max_tokens` value. Set `0` to disable the cap | | `VT_TT_HOST_FREE_DECODE` | unset (host-free eager decode) | Tenstorrent-only decode mode: residual-RMS + RoPE stay on device at decode shapes and `cur_pos` advances on-device, with no per-step host readback. Default since #1604 (2.1x over the host-hybrid path: 10.94-11.06 vs 5.34 tok/s warm, Qwen3-0.6B batch 1). Exact `0` restores the pre-flip host-hybrid path for same-binary A/B; any other spelling keeps the default | | `VT_TT_DECODE_CAPTURE` | unset (capture ARMED on the gated families) | Tenstorrent decode-graph capture on top of host-free decode. Default since [#1625](https://github.com/mudler/vllm.cpp/issues/1625)'s flip landed in [#2910](https://github.com/mudler/vllm.cpp/pull/2910), and exact `0` opts out and keeps gating the eager host-free arm. The default is EVIDENCE-SCOPED rather than global: it applies to the families with a committed captured-arm gate pair, while every other decode driver keeps the explicit opt-in until its own pair exists -- which is what `Platform::static_graph_requires_opt_in()` carves. This row said "capture declined" and "NOT the default" until 2026-09-05, on the strength of a deterministic multi-request hang; that hang was root-caused to [#2669](https://github.com/mudler/vllm.cpp/issues/2669)'s short-chunk device KV push clobber, where prefill chunks below `kPagedFillMinTokens` ran the batched device update per token over one shared physical block and the last writer won, leaving captured decode attending a dead request's rows. Repaired, the captured multi-request battery is 16/16 and hang-free, and the captured arm is the fastest measured arm at 28.61 tok/s warm median against 12.21 for the eager host-free arm (same-binary A/B) | +| `VT_TT_DECODE_CAPTURE` | unset (capture ARMED on the gated families) | Tenstorrent decode-graph capture on top of host-free decode. Default since [#1625](https://github.com/mudler/vllm.cpp/issues/1625)'s flip landed in [#2910](https://github.com/mudler/vllm.cpp/pull/2910), and exact `0` opts out and keeps gating the eager host-free arm. The default is EVIDENCE-SCOPED rather than global: it applies to the families with a committed captured-arm gate pair, while every other decode driver keeps the explicit opt-in until its own pair exists -- which is what `Platform::static_graph_requires_opt_in()` carves. This row said "capture declined" and "NOT the default" until 2026-09-05, on the strength of a deterministic multi-request hang; that hang was root-caused to [#2669](https://github.com/mudler/vllm.cpp/issues/2669)'s short-chunk device KV push clobber, where prefill chunks below `kPagedFillMinTokens` ran the batched device update per token over one shared physical block and the last writer won, leaving captured decode attending a dead request's rows. Repaired, the captured multi-request battery is 16/16 and hang-free, and the captured arm is the fastest measured arm at 28.61 tok/s warm median against 12.21 for the eager host-free arm (same-binary A/B) | +| `VT_TT_RELEASE_WARM_ROWS` | unset (keep all slots) | Tenstorrent-only decode-memory lever: after the engine's cold pre-warm, releases every slot whose consumer shadow has exactly this many rows — the warm pass's full-batch-shape activations, which the captured decode never reads. At 27B this frees ~14 GB of slot residency that otherwise filled the banks to 99 percent during generation. Row count must match the warm batch (e.g. `13104`); unset keeps every slot | | `VT_BENCH_PRETOKENIZE` | `1` (on) | Makes `vllm-bench` encode every prompt before its benchmark clock and admit token IDs, matching the pinned vLLM comparison frontend. Exact `0` restores timed string admission for same-binary A/B; unset, `1`, and invalid spellings keep the safe default-on behavior | | `VT_VULKAN_DEVICE` | first suitable device | Forces the Vulkan physical device index. Required on a multi-GPU host to pin the intended device | | `VT_KV_CACHE_F32` | off (native KV dtype) | Forces the KV cache to fp32. A precision/diagnostic lever, at the cost of double the KV memory |