From 62ef882ee1668e7de88c7e6fbf6000663188ddc6 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 30 Aug 2026 23:21:03 +0800
Subject: [PATCH 01/14] CollectiveX: publish wire-basis bytes so LL bandwidth
 stops mixing byte definitions

DeepEP/UCCL/NCCL low-latency kernels move one copy per (token, expert)
assignment; normal mode and MoRI LL move rank-deduplicated (token, dest-rank)
copies. byte_provenance always carried the smaller deduplicated numerator, so
any bandwidth divided from it on a token-expert backend was a lower bound
published as the wire rate (34.1% low on nccl-ep LL EP8 at T=128: 77.4 MB vs
the 117.4 MB the kernels move) and not comparable across backends.

Rows now carry wire_byte_provenance beside byte_provenance: identical for
token-rank receives, per-assignment for token-expert ones, basis declared by
the existing logical_copies.wire discriminator. bandwidth.py divides from the
wire basis (falling back to the deduplicated one for pre-wire artifacts, which
only ever understates).
---
 experimental/CollectiveX/bandwidth.py         | 38 ++++++++++++-------
 experimental/CollectiveX/bench/ep_harness.py  | 30 ++++++++++++++-
 .../CollectiveX/tests/test_measurement.py     | 17 +++++++++
 3 files changed, 71 insertions(+), 14 deletions(-)

diff --git a/experimental/CollectiveX/bandwidth.py b/experimental/CollectiveX/bandwidth.py
index c0b13e54e0..b927a580ca 100644
--- a/experimental/CollectiveX/bandwidth.py
+++ b/experimental/CollectiveX/bandwidth.py
@@ -8,15 +8,16 @@
   B. alpha/beta fit -- OLS of p50 latency vs bytes, latency ~= alpha + bytes/beta, which
      separates the bandwidth term from the per-call floor that dominates small T.
 
-Bytes are LOGICAL payload (one copy per unique (token, dest-rank) pair), excluding
-protocol/padding. Reading them against a link's peak needs two adjustments: logical bytes
-include the copies a rank routes to itself, which never cross the interconnect, so wire
-traffic is (1 - routing.locality.local_rank_fraction) x these figures; and beta is a
-MARGINAL rate, so it legitimately sits above every measured point.
-
-This basis is goodput; low-latency layouts send one copy per (token, expert), so their wire
-traffic is topk/fanout higher (~1.5x at EP8) and their GB/s is not comparable to a normal row
-or to a vendor table counting every expert copy.
+Bytes are the WIRE basis (`wire_byte_provenance`): one copy per unique (token, dest-rank)
+pair for rank-deduplicating layouts, one copy per (token, expert) for the low-latency
+kernels that do not deduplicate — whichever the row's `logical_copies.wire` declares. That
+makes GB/s comparable across backends and modes. Two adjustments still apply when reading
+against a link's peak: these bytes include the copies a rank routes to itself, which never
+cross the interconnect, so fabric traffic is (1 - routing.locality.local_rank_fraction) x
+these figures; and beta is a MARGINAL rate, so it legitimately sits above every measured
+point. Pre-wire artifacts fall back to the rank-deduplicated `byte_provenance`, which for
+per-assignment LL rows understates the wire rate (a lower bound, ~34% low on nccl-ep LL
+EP8 at T=128), never overstates it.
 
 A line through a ladder is a MODEL and a positive slope alone is not evidence, so every Fit
 carries its own quality and beta is withheld unless it clears both gates below. Deliberately
@@ -108,8 +109,7 @@ def fit_alpha_beta(document: dict, component: str, pct: str = "p50") -> Fit | No
         if not row.get("correctness", {}).get("passed", True):
             excluded += 1
             continue
-        usable.append((row["byte_provenance"][component]["total_logical_bytes"],
-                       percentiles[pct]))
+        usable.append((_wire_bytes(row, component), percentiles[pct]))
     if len(usable) < MIN_FIT_POINTS or len({x for x, _ in usable}) < 2:
         return None  # too few points, or zero-variance x (e.g. a degenerate ladder)
     xs, ys = [x for x, _ in usable], [y for _, y in usable]
@@ -127,6 +127,18 @@ def fit_alpha_beta(document: dict, component: str, pct: str = "p50") -> Fit | No
     )
 
 
+def _wire_bytes(row: dict, component: str) -> int:
+    """Bytes the kernels actually move for this component.
+
+    Prefers `wire_byte_provenance` (per-assignment for token-expert LL receives); artifacts
+    written before that field carry only the rank-deduplicated `byte_provenance`, which for
+    those LL rows is a LOWER BOUND on wire traffic (~topk/fanout below it), so a bandwidth
+    derived from the fallback understates the wire rate rather than overstating it.
+    """
+    provenance = row.get("wire_byte_provenance") or row["byte_provenance"]
+    return provenance[component]["total_logical_bytes"]
+
+
 def _format_fit(component: str, fit: Fit) -> str:
     """One component's fit, withholding what the data does not support."""
     if not fit.beta_is_reliable:
@@ -142,7 +154,7 @@ def _cell(row: dict, component: str, ep: int) -> str:
     percentiles = row["components"][component]["percentiles_us"]
     if not percentiles:
         return f"{component}=n/a"
-    nbytes = row["byte_provenance"][component]["total_logical_bytes"]
+    nbytes = _wire_bytes(row, component)
     p50 = _algbw_per_gpu(nbytes, percentiles["p50"], ep)
     p99 = _algbw_per_gpu(nbytes, percentiles["p99"], ep)
     return f"{component}=n/a" if p50 is None or p99 is None \
@@ -170,7 +182,7 @@ def _provenance(document: dict) -> str:
 
 def render(documents: list[dict]) -> str:
     lines = [
-        "## CollectiveX EP bandwidth (per-GPU, logical payload)",
+        "## CollectiveX EP bandwidth (per-GPU, wire-basis payload)",
         "",
         "GB/s at p50/p99 *latency* -- the p99-latency figure is worst-case bandwidth, not a "
         f"'p99 bandwidth'. `fit`: latency ~= alpha + bytes/beta over the ladder; beta is "
diff --git a/experimental/CollectiveX/bench/ep_harness.py b/experimental/CollectiveX/bench/ep_harness.py
index 53462cc15c..a6144e740b 100644
--- a/experimental/CollectiveX/bench/ep_harness.py
+++ b/experimental/CollectiveX/bench/ep_harness.py
@@ -1344,7 +1344,7 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) ->
         )
         combine_bytes = logical_byte_provenance(rstats["routed_copies"], args.hidden)
         # Second byte basis, for backends whose wire carries one copy per (token, expert). Which
-        # applies is a property of the RECEIVE, not the mode -- MoRI's IntraNodeLL deduplicates
+        # applies is a property of the RECEIVE, not the mode -- MoRI's LL kernels deduplicate
         # where the other low-latency kernels do not -- so key it on the declared
         # receive layout. `routed_copies` stays the canonical comparable basis.
         assignment_copies = int(sum(rstats["expert_assignments_per_rank"]))
@@ -1357,6 +1357,25 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) ->
             field: dispatch_bytes[field] + combine_bytes[field] for field in dispatch_bytes
         }
         stage_bytes = dict.fromkeys(dispatch_bytes, 0)
+        # WIRE bytes: what the kernels actually move, on the basis `wire_basis` declares.
+        # For token-expert receives this is the per-assignment count (topk/fanout above the
+        # rank-deduplicated basis, +34% observed on nccl-ep LL EP8 at T=128); for token-rank
+        # receives it equals the canonical figures. A bandwidth divided from `byte_provenance`
+        # on a token-expert backend is a LOWER BOUND, not the wire rate, and is not comparable
+        # across backends -- consumers computing GB/s must divide from THESE bytes.
+        wire_copies = (
+            assignment_copies if wire_basis == "per-assignment"
+            else int(rstats["routed_copies"])
+        )
+        wire_dispatch_bytes = logical_byte_provenance(
+            wire_copies, args.hidden,
+            backend.dispatch_value_bytes, backend.dispatch_scale_bytes_per_copy,
+        )
+        wire_combine_bytes = logical_byte_provenance(wire_copies, args.hidden)
+        wire_roundtrip_bytes = {
+            field: wire_dispatch_bytes[field] + wire_combine_bytes[field]
+            for field in wire_dispatch_bytes
+        }
         spread = samples[T].spread
         chain = samples[T].chain
         chain_spread = samples[T].chain_spread
@@ -1441,6 +1460,15 @@ def run_sweep(args, backend, torch, dist, device, rank: int, world_size: int) ->
                 "roundtrip": roundtrip_bytes,
                 "stage": stage_bytes,
             },
+            # Same fields on the wire basis (`logical_copies.wire`). Identical to
+            # `byte_provenance` for token-rank receives; per-assignment for the LL kernels
+            # that move one copy per (token, expert). Bandwidth = wire bytes / latency.
+            "wire_byte_provenance": {
+                "combine": wire_combine_bytes,
+                "dispatch": wire_dispatch_bytes,
+                "roundtrip": wire_roundtrip_bytes,
+                "stage": stage_bytes,
+            },
             # Copy counts behind the byte figures above, so a reader can rebase them: `routed` is
             # the basis they use, `assignments` the per-(token, expert) count, `wire` which the
             # kernels move. Kept out of `byte_provenance`, whose values are all per-component.
diff --git a/experimental/CollectiveX/tests/test_measurement.py b/experimental/CollectiveX/tests/test_measurement.py
index f3daeadc26..95d5941ff2 100644
--- a/experimental/CollectiveX/tests/test_measurement.py
+++ b/experimental/CollectiveX/tests/test_measurement.py
@@ -179,5 +179,22 @@ def test_gate_failed_rung_excluded_from_fit_and_marked(self):
         self.assertIn("[correctness FAILED]", out)
         self.assertIn("excluded 1 gate-failed rung", out)
 
+    def test_wire_basis_bytes_are_preferred_over_the_deduplicated_basis(self):
+        # A token-expert LL row moves one copy per (token, expert): dividing GB/s from the
+        # rank-deduplicated `byte_provenance` published a LOWER BOUND as if it were the wire
+        # rate (34% low on nccl-ep LL EP8 at T=128). With `wire_byte_provenance` present the
+        # fit must use it; the pre-wire fixture rows above pin the fallback path.
+        rows = _linear(LADDER)
+        for row in rows:
+            row["wire_byte_provenance"] = {
+                c: {"total_logical_bytes":
+                    row["byte_provenance"][c]["total_logical_bytes"] * 3 // 2}
+                for c in COMPONENTS
+            }
+        baseline = bandwidth.fit_alpha_beta(_doc(_linear(LADDER)), "dispatch")
+        fit = bandwidth.fit_alpha_beta(_doc(rows), "dispatch")
+        self.assertAlmostEqual(fit.beta_gbps, baseline.beta_gbps * 1.5, places=4)
+
+
 if __name__ == "__main__":
     unittest.main()

From d05d8e6b14235bec83810c2cadacc201bc32b01b Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 30 Aug 2026 23:21:03 +0800
Subject: [PATCH 02/14] CollectiveX: hold nccl-ep low-latency fleet-wide until
 the combine fence ships
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The T<=128 ladder clamp reduces exposure to the un-fenced shared-memory combine
race (DeepEP #642's fence, absent at our pin and at NVIDIA master); the source
has said all along it is not a safety boundary — lower rungs are less likely to
hit the race, not immune. Green clamped rows are therefore not
publication-valid, and describing T=256 as a capacity limit was wrong. Rows
held in the registry, normal mode unaffected.
---
 .../CollectiveX/configs/platform_config.json         | 12 ++++++------
 1 file changed, 6 insertions(+), 6 deletions(-)

diff --git a/experimental/CollectiveX/configs/platform_config.json b/experimental/CollectiveX/configs/platform_config.json
index 4281cbbee6..4ed76b4114 100644
--- a/experimental/CollectiveX/configs/platform_config.json
+++ b/experimental/CollectiveX/configs/platform_config.json
@@ -11,7 +11,7 @@
       "scale_up_transport": "nvlink",
       "launcher": "single-slurm",
       "backends": {"deepep-v2": [8, 16], "uccl-ep": [8], "nccl-ep": [8]},
-      "ll_backends": {"deepep-v2": [8], "uccl-ep": [8], "nccl-ep": [8]},
+      "ll_backends": {"deepep-v2": [8], "uccl-ep": [8]},
       "fabric": {"nic": "ConnectX-7 2x200GbE", "switch": "Arista 7060DX5-64S (Tomahawk4, 25.6T)"},
       "operator": {
         "partition": "hpc-gpu-1",
@@ -35,7 +35,7 @@
       "scale_up_transport": "nvlink",
       "launcher": "single-slurm",
       "backends": {"deepep-v2": [8, 16], "uccl-ep": [8], "nccl-ep": [8]},
-      "ll_backends": {"deepep-v2": [8], "uccl-ep": [8], "nccl-ep": [8]},
+      "ll_backends": {"deepep-v2": [8], "uccl-ep": [8]},
       "fabric": {"nic": "ConnectX-7 400G", "switch": "NVIDIA Quantum-2 QM9790 (25.6T, InfiniBand)"},
       "operator": {
         "partition": "main",
@@ -55,7 +55,7 @@
       "scale_up_transport": "nvlink",
       "launcher": "single-slurm",
       "backends": {"deepep-v2": [8, 16], "uccl-ep": [8, 16], "nccl-ep": [8]},
-      "ll_backends": {"deepep-v2": [8, 16], "uccl-ep": [8], "nccl-ep": [8]},
+      "ll_backends": {"deepep-v2": [8, 16], "uccl-ep": [8]},
       "fabric": {"nic": "8x ConnectX-7-class 400G InfiniBand rails (bare metal, gdrdrv)", "switch": "InfiniBand (model not surveyed)"},
       "operator": {
         "partition": "batch_1",
@@ -77,7 +77,7 @@
       "scale_up_transport": "nvlink",
       "launcher": "single-slurm",
       "backends": {"deepep-v2": [8, 16], "nccl-ep": [8]},
-      "ll_backends": {"deepep-v2": [8], "nccl-ep": [8]},
+      "ll_backends": {"deepep-v2": [8]},
       "fabric": {"nic": "ConnectX-8 2x400GbE", "switch": "NVIDIA Spectrum-X SN5600 (51.2T)"},
       "operator": {
         "partition": "batch_1",
@@ -104,7 +104,7 @@
       "scale_up_transport": "mnnvl",
       "launcher": "gb-nv",
       "backends": {"deepep-v2": [8, 16], "nccl-ep": [8, 16], "flashinfer-ep": [8, 16]},
-      "ll_backends": {"deepep-v2": [8, 16], "nccl-ep": [8]},
+      "ll_backends": {"deepep-v2": [8, 16]},
       "fabric": {"nic": "MNNVL (scale-out not used)", "switch": "NVLink NVL72"},
       "operator": {
         "partition": "batch",
@@ -124,7 +124,7 @@
       "scale_up_transport": "mnnvl",
       "launcher": "gb-nv",
       "backends": {"deepep-v2": [8, 16], "nccl-ep": [8, 16], "flashinfer-ep": [8, 16]},
-      "ll_backends": {"deepep-v2": [8, 16], "nccl-ep": [8]},
+      "ll_backends": {"deepep-v2": [8, 16]},
       "fabric": {"nic": "MNNVL (scale-out not used)", "switch": "NVLink NVL72"},
       "operator": {
         "partition": "batch_1",

From 52c8b625aa166cc321b526562588b6eba47640d7 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 30 Aug 2026 23:21:14 +0800
Subject: [PATCH 03/14] CollectiveX: charge nccl-ep's per-step routing
 collective to the timed HT dispatch
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

ncclEpUpdateHandle is documented as a per-step collective and production
routing changes every MoE layer, so a serving step pays it before every HT
dispatch at the handle's full token capacity. Excluding it (as NVIDIA's
ep_bench does) made HT dispatch the one window that omitted its routing work
while deepep-v2, uccl-ep, MoRI and FlashInfer all carry theirs per call.
kernel_generation bumps to nccl-ep-ht-routed — the per-row discriminator the
earlier NCCL measurement changes lacked.
---
 experimental/CollectiveX/bench/ep_nccl.py | 17 ++++++++++++++++-
 1 file changed, 16 insertions(+), 1 deletion(-)

diff --git a/experimental/CollectiveX/bench/ep_nccl.py b/experimental/CollectiveX/bench/ep_nccl.py
index 8d8519321e..be425fdf90 100644
--- a/experimental/CollectiveX/bench/ep_nccl.py
+++ b/experimental/CollectiveX/bench/ep_nccl.py
@@ -95,7 +95,11 @@ class NCCLEPBackend(EPBackend):
     # semantics are switched to their LL values in __init__ (mirrors ep_deepep_v2).
     #   normal      -> HT / FLAT layout / unweighted-rank-sum combine.
     #   low-latency -> LL / EXPERT_MAJOR layout / source-side weighted-kernel-sum combine.
-    kernel_generation = "nccl-ep-ht"
+    # "-routed" marks the generation whose timed HT dispatch charges the per-step
+    # ncclEpUpdateHandle (see dispatch()); earlier "nccl-ep-ht" rows excluded it and the
+    # docs record that those cannot be separated by any other field — this suffix is the
+    # per-row discriminator that change lacked.
+    kernel_generation = "nccl-ep-ht-routed"
     SUPPORTED_MODES = ("normal", "low-latency")
     SUPPORTED_PRECISIONS = ("bf16",)
     stage_device_work = False
@@ -385,6 +389,17 @@ def _rebind(self, h):
     def dispatch(self, p):
         h = self._ensure_handle(p)
         stream = self._stream()
+        if not self._ll:
+            # Charge the per-step routing collective to the window. `ncclEpUpdateHandle` is
+            # documented as a "per-step collective: prepare the handle for the given top-k
+            # routing decisions", and production routing changes every MoE layer — a serving
+            # step pays this update before every HT dispatch, at the handle's full token
+            # capacity, exactly as here. Excluding it (as NVIDIA's ep_bench does) made HT
+            # dispatch the one window that omitted its routing/layout work while deepep-v2,
+            # uccl-ep, MoRI and FlashInfer all carry theirs per call. No sync or counter
+            # read here: the bound problem's counters are deterministic and already read
+            # (_bind_ht_recv_count) in the untimed rebind.
+            h.handle.update(h.topk_idx_t, layout_info=h.layout_info, stream=stream)
         if self._ll:
             # LL EXPERT_MAJOR: tokens in, 3D per-expert padded tokens out, per-expert recv
             # counts written into expert_counters. No weights on the dispatch (the gate is

From a98a2c87b81b814fe4f35689b7c21f46d4f19373 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 30 Aug 2026 23:21:14 +0800
Subject: [PATCH 04/14] CollectiveX: measure MoRI low-latency on AsyncLL, the
 kernel production deploys
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

SGLang's moriep dispatcher maps low-latency to AsyncLL split-phase
(dispatch_send/recv + combine_send/recv, block_num 64 rdma 32 warps 8) and
asserts that kernel type; IntraNodeLL is not selected by any engine, so the
previous rows measured an off-production path (kernel_generation intranode-ll
discriminates them). The timed windows now run AsyncLL send+recv back-to-back
and the adapter fails closed on wheels whose AsyncLL lacks the recv API —
single-call AsyncLL returns before any payload lands. Pending on-metal
validation on mi355x.
---
 experimental/CollectiveX/bench/ep_mori.py | 55 ++++++++++++++++-------
 1 file changed, 38 insertions(+), 17 deletions(-)

diff --git a/experimental/CollectiveX/bench/ep_mori.py b/experimental/CollectiveX/bench/ep_mori.py
index f588a29c11..85089971b1 100644
--- a/experimental/CollectiveX/bench/ep_mori.py
+++ b/experimental/CollectiveX/bench/ep_mori.py
@@ -86,29 +86,30 @@ def __init__(self, args, rank, world_size, local_rank, device):
             else ("IntraNode", "intranode", (80, 0, 16, 16))
         )
         if self.mode == "low-latency":
-            # LOW-LATENCY (decode) mode: IntraNodeLL, the scale-up low-latency kernel. It is
-            # single-phase (plain dispatch()/combine(), no dispatch_recv/combine_recv split),
-            # pure-intranode (shares the no-RDMA ShmemBufsIntraNode staging with IntraNode, so
-            # no symmetric-heap registration), and returns the same compact [max_recv, hidden]
-            # layout. Its combine keeps the plain rank-deduplicated additive sum (combine is
-            # called with weights=None -> weight_ptr 0 in mori.ops, so the gate is NOT applied
-            # in-kernel), identical in semantics to IntraNode/normal mode ("unweighted-rank-sum",
-            # the base default the harness admits for low-latency) over the same compact
-            # rank-deduplicated receive (the base "token-rank" receive_layout, so the
-            # artifact's wire basis stays rank-deduplicated). So LL differs from the normal
-            # IntraNode path ONLY by kernel_type (set here vs omitted) and timing; every transport
-            # method (dispatch/stage/combine/inspect_dispatch/combine_transformed) is reused as-is.
-            # AsyncLL (enum 4) is deliberately NOT used: it is split-phase (dispatch_recv/
-            # combine_recv) and RDMA-staged, which does not fit the single-call dispatch/stage/
-            # combine contract. Scale-up EP8 decode only; scale-out EP16 LL is out of scope (kept
-            # out of ll_backends). Reuse the IntraNode launch tuning under MANUAL launch mode.
+            # LOW-LATENCY (decode) mode: AsyncLL, because that is what production selects —
+            # SGLang's moriep dispatcher maps EpMode.LOW_LATENCY to AsyncLL (split-phase
+            # dispatch_send/dispatch_recv + combine_send/combine_recv) with block_num 64,
+            # rdma_block_num 32, warp_num_per_block 8, and its low-latency impl asserts the
+            # kernel IS AsyncLL. The previous adapter measured IntraNodeLL, a kernel no engine
+            # deploys for LL, so those rows described an off-production path (their
+            # kernel_generation "intranode-ll" is the discriminator). AsyncLL must be driven
+            # split-phase: mori.ops' dispatch()/combine() under this kernel_type launch only
+            # the SEND side and return without the payload landing (the earlier "fails
+            # silently single-call" finding), so the timed dispatch here is send+recv
+            # back-to-back — the full transport the serving step pays, merely without the
+            # expert GEMM interleaved between the phases.
+            # Combine keeps weights=None (gate not applied in-kernel), and the receive is the
+            # same compact rank-deduplicated [max_recv, hidden] layout, so semantics stay
+            # "unweighted-rank-sum" over "token-rank" and the wire basis stays
+            # rank-deduplicated. Scale-up EP8 decode only; scale-out EP16 LL stays out of
+            # ll_backends.
             if scale_out:
                 raise RuntimeError(
                     "MoRI low-latency is scale-up EP8 only (scale-out EP16 low-latency "
                     "is out of scope; see platform_config ll_backends)"
                 )
             kernel_name, self.kernel_generation, blocks = (
-                "IntraNodeLL", "intranode-ll", (80, 0, 16, 16)
+                "AsyncLL", "async-ll", (64, 32, 8, 8)
             )
         self._kernel_type = None
         if kernel_name != "IntraNode":
@@ -118,6 +119,9 @@ def __init__(self, args, rank, world_size, local_rank, device):
                     f"this MoRI image lacks EpDispatchCombineKernelType.{kernel_name}"
                 )
             self._kernel_type = getattr(kernel_enum, kernel_name)
+        # AsyncLL's dispatch()/combine() launch only the SEND kernels; the payload lands in
+        # the recv phase, so both timed components must run send+recv back-to-back.
+        self._split_phase = kernel_name == "AsyncLL"
         self._inter_node = kernel_name == "InterNodeV1"
         self.num_qps = 1
         self.block_num, self.rdma_block_num, self.dispatch_warps, self.combine_warps = blocks
@@ -227,6 +231,16 @@ def create_buffer(self, spec):
         self.op = mori.ops.EpDispatchCombineOp(self.config)
         if getattr(self.op, "launch_config_mode", None) != "MANUAL":
             raise RuntimeError("MoRI explicit launch configuration was not applied")
+        # Fail closed rather than measure a silent send-only AsyncLL: without the recv
+        # methods the single-call dispatch returns before any payload lands.
+        if self._split_phase and not (
+            callable(getattr(self.op, "dispatch_recv", None))
+            and callable(getattr(self.op, "combine_recv", None))
+        ):
+            raise RuntimeError(
+                "this MoRI image's AsyncLL lacks the split-phase "
+                "dispatch_recv/combine_recv API the deployed low-latency path drives"
+            )
 
     def semantic_payload(self, x):
         if not self._fp8:
@@ -265,6 +279,10 @@ def dispatch(self, p):
                 warp_per_block=self.dispatch_warps,
             )
         )
+        if self._split_phase:
+            # AsyncLL: the call above launched only the send kernels; production pays the
+            # recv phase in the same step (SGLang's dispatch_b), so the timed window does too.
+            self.op.dispatch_recv()
         return types.SimpleNamespace(
             dispatch_output=dispatch_output,
             dispatch_weights=dispatch_weights,
@@ -302,6 +320,9 @@ def combine(self, p, h):
             rdma_block_num=self.rdma_block_num,
             warp_per_block=self.combine_warps,
         )
+        if self._split_phase:
+            # AsyncLL combine is send-only until the recv phase lands the reduction.
+            self.op.combine_recv()
         return combined[:p.T]
 
     def inspect_dispatch(self, p, h):

From 211521c937a94a7201010dce9e514ca356f634ee Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 30 Aug 2026 23:21:14 +0800
Subject: [PATCH 05/14] CollectiveX: sync methodology with the wire basis, the
 nccl-ep LL hold and the routed HT window

---
 experimental/CollectiveX/docs/methodology.md | 42 +++++++++++---------
 1 file changed, 24 insertions(+), 18 deletions(-)

diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md
index 6bb8404bf1..c41403cb71 100644
--- a/experimental/CollectiveX/docs/methodology.md
+++ b/experimental/CollectiveX/docs/methodology.md
@@ -210,13 +210,17 @@ EP8 on MI300X/MI325X/MI355X, and UCCL-EP EP8 on H100/H200/B200 only (the legacy
 `Buffer` low-latency kernels, which at EP8 run `cudaIpc` over NVLink, not the CPU-proxy RDMA path,
 because the adapter passes `is_intranode` and UCCL then never starts its proxies. The AMD SKUs drop
 LL: upstream raised `kNumMaxTopK` 9 -> 16 six days before our pin, and the resulting host assert
-cannot hold on AMD's 16 warp groups), and NCCL EP EP8 on all six NVIDIA SKUs. Its
+cannot hold on AMD's 16 warp groups). NCCL EP low-latency is HELD on every SKU. Its
 `LOW_LATENCY` algorithm is the DeepEP-derived decode path, EXPERT_MAJOR receive with a source-side
-weighted-kernel-sum combine. Those rows were dropped while every LL leg wedged on stale peer signals
-([NVIDIA/nccl#2303](https://github.com/NVIDIA/nccl/issues/2303)) and restored once the single-handle
-adapter removed the aliasing that caused it. B300 carries NCCL EP as its only
-low-latency row, and it is a `candidate` transport, so that SKU publishes no production decode
-coverage. Whether a given SKU/backend/EP/mode cell is attempted is a capability
+weighted-kernel-sum combine — but its combine recv pipeline is a port of DeepEP's PRE-FIX code,
+missing the `fence.proxy.async.shared::cta` DeepEP added in #642, and the shared-memory race that
+fence closes is present on every rung, not only the T=256 rung where it was observed (1-in-5,
+bimodal error 0.47 vs 0.0039 on gb300 EP8). The earlier T<=128 ladder clamp reduced exposure; it
+was never a safety boundary, so a green clamped row is not publication-valid and the rows are held
+in the registry until a fenced wheel ships. (Historically those rows also wedged on stale peer
+signals, [NVIDIA/nccl#2303](https://github.com/NVIDIA/nccl/issues/2303), fixed by the single-handle
+adapter.) B300 therefore publishes no low-latency coverage beyond DeepEP V2 EP8.
+Whether a given SKU/backend/EP/mode cell is attempted is a capability
 fact. Whether it succeeded is decided only by the emitted artifact.
 
 ## Workload Identity
@@ -385,18 +389,20 @@ ranks between pairs adds its own ~10µs and removes the cross-pair overlap the m
 capture. That is a differently-defined quantity that must never share a column with the free-running
 period.
 
-One backend's timed window omits a cost the others pay, deliberately. nccl-ep binds routing with
-`ncclEpUpdateHandle`, a collective whose cost scales with the group's token capacity rather than the
-token count, so charging it per iteration would import a ladder-max-proportional term into dispatch
--- the same artifact that sizing HT's combine input to the ladder maximum used to put under combine.
-It is bound during the untimed warm-up, as NVIDIA's own `ep_bench` does (CUDA events around dispatch
-and combine only, handle update outside the loop). Low-latency mode has nothing to exclude:
-`ncclEpUpdateHandle` returns immediately and the kernel reads the cached routing inside the timed
-dispatch. Every other backend's layout cost scales with tokens and belongs in the window -- uccl-ep
-calls `get_dispatch_layout` inside dispatch, while deepep-v2, MoRI and FlashInfer pass routing on every
-call.
-
-The artifact records the mode so a reader can keep distinct measurement contracts separate.
+Every backend's timed HT dispatch now carries its routing work. nccl-ep binds routing with
+`ncclEpUpdateHandle`, documented as a "per-step collective: prepare the handle for the given top-k
+routing decisions" — and production routing changes every MoE layer, so a serving step pays that
+collective before every dispatch, at the handle's full token capacity. Earlier generations excluded
+it (as NVIDIA's own `ep_bench` does: CUDA events around dispatch and combine only, handle update
+outside the loop) on the argument that its capacity-proportional cost would import a ladder-max
+term into dispatch; that argument describes exactly what production pays, since engines size the
+handle to their max token capacity and update it per step. The timed window now includes the
+update; rows carry `kernel_generation` `nccl-ep-ht-routed`, and pre-change `nccl-ep-ht` rows are a
+different measurement contract — the discriminator the earlier NCCL changes lacked. Low-latency
+mode has nothing to include: `ncclEpUpdateHandle` returns immediately there and the kernel reads
+the cached routing inside the timed dispatch. The other backends already carried this cost --
+uccl-ep calls `get_dispatch_layout` inside dispatch, while deepep-v2, MoRI and FlashInfer pass
+routing on every call.
 
 Every measured component uses one fixed timing profile, defined once in `configs/sweep.json`
 and baked into every scheduled case:

From 589041d24c6e6083b930093a2c2ceafafd4ac300 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 30 Aug 2026 23:30:50 +0800
Subject: [PATCH 06/14] CollectiveX: switch the EP workload to DeepSeek-V4-Pro
 routing
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

deepseek-v4-pro: hidden 7168, top-k 6, 384 routed experts — the dsv4 shape the
rest of InferenceX benchmarks (operatorx testlists, HF-config-derived). The
workload name is part of every case_id, so this is a clean identity break in
the durable store: no v3 row can collide with a v4-pro row. Ladders and seed
unchanged.
---
 experimental/CollectiveX/configs/sweep.json  | 6 +++---
 experimental/CollectiveX/docs/methodology.md | 5 +++--
 2 files changed, 6 insertions(+), 5 deletions(-)

diff --git a/experimental/CollectiveX/configs/sweep.json b/experimental/CollectiveX/configs/sweep.json
index 2120902246..2f91873545 100644
--- a/experimental/CollectiveX/configs/sweep.json
+++ b/experimental/CollectiveX/configs/sweep.json
@@ -17,10 +17,10 @@
     "chain_drop": 16
   },
   "workload": {
-    "name": "deepseek-v3",
+    "name": "deepseek-v4-pro",
     "hidden": 7168,
-    "topk": 8,
-    "routed_experts": 256,
+    "topk": 6,
+    "routed_experts": 384,
     "seed": 67,
     "token_ladders": {
       "decode": [1, 2, 4, 8, 16, 32, 64, 128, 256, 512],
diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md
index c41403cb71..2b48df2f95 100644
--- a/experimental/CollectiveX/docs/methodology.md
+++ b/experimental/CollectiveX/docs/methodology.md
@@ -19,7 +19,8 @@ It does not predict serving throughput without a separate correlation study.
 
 ## Matrix
 
-The implemented workload is `deepseek-v3`: hidden 7168, top-k 8, 256 routed experts, packed
+The implemented workload is `deepseek-v4-pro` (DeepSeek-V4-Pro 1.6T, the `dsv4` model the rest of
+InferenceX benchmarks): hidden 7168, top-k 6, 384 routed experts, packed
 placement, and one pinned fixed resource profile per backend/topology. Combine is always BF16.
 Dispatch precision is a swept dimension, with a BF16 control and, on the backends whose FP8 dispatch is
 supported upstream (DeepEP V2, MoRI, UCCL-EP, FlashInfer EP), an FP8 dispatch (`bf16`, `fp8`),
@@ -87,7 +88,7 @@ emits a case for a precision it does not support. `normal`-mode cases use the
 `layout-and-dispatch-v1` semantics. `low-latency` cases use each backend's decode-kernel semantics
 (detailed below).
 
-- `ep-core`: uniform routing over the workload's token ladders, which for `deepseek-v3` include decode
+- `ep-core`: uniform routing over the workload's token ladders, which for `deepseek-v4-pro` include decode
   T=1..512 powers of two and prefill T=1024..8192 powers of two. Ladders are model-specific and
   live with the workload in `configs/sweep.json`.
 

From 5d3b540bcf80ed7ccb9d0581916439470eb6fae2 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Sun, 30 Aug 2026 23:59:05 +0800
Subject: [PATCH 07/14] CollectiveX: sync the README's MoRI low-latency kernel
 description with AsyncLL (review)

---
 experimental/CollectiveX/README.md | 7 ++++---
 1 file changed, 4 insertions(+), 3 deletions(-)

diff --git a/experimental/CollectiveX/README.md b/experimental/CollectiveX/README.md
index 34b25f5132..5be2797487 100644
--- a/experimental/CollectiveX/README.md
+++ b/experimental/CollectiveX/README.md
@@ -26,8 +26,9 @@ in one of two modes:
 - `low-latency` uses each backend's decode-optimized kernel family: on DeepEP the legacy
   `deep_ep.Buffer` IBGDA `low_latency_dispatch`/`low_latency_combine` (a per-expert padded receive
   and a source-side gate-weighted combine). On UCCL-EP the same legacy `Buffer` low-latency kernels,
-  which at the scoped EP8 run `cudaIpc` over NVLink rather than its CPU-proxy transport. On MoRI the `IntraNodeLL` kernel (single-call,
-  pure-intranode, same compact layout and unweighted rank-sum combine as `IntraNode`). It is a
+  which at the scoped EP8 run `cudaIpc` over NVLink rather than its CPU-proxy transport. On MoRI the `AsyncLL` kernel driven split-phase
+  (send+recv inside each timed component — the kernel SGLang deploys for low-latency; same compact
+  layout and unweighted rank-sum combine as `IntraNode`). It is a
   decode-phase-only, per-SKU-capability-gated addition whose runnable set differs from `normal`'s, so
   it is enabled from each SKU's `ll_backends` registry entry (currently DeepEP V2 at EP8 on
   H100/H200 and at EP8 *and EP16* on B200 (the nscale bare-metal pool, whose gdrdrv-backed IBGDA
@@ -128,7 +129,7 @@ scale-up domain.
 | Backend | Engine availability | Current scope |
 |---|---|---|
 | DeepEP V2 | `production`, with vLLM `--all2all-backend deepep_v2`, SGLang `--moe-a2a-backend deepep` | `normal` mode is PR #605 `ElasticBuffer` plus exact upstream #630 and #640 fixes: LSA for scale-up and GIN for x86 EP16 scale-out. FP8 dispatch via `use_fp8_dispatch` (blockwise e4m3fn) alongside BF16. `low-latency` mode is the legacy `deep_ep.Buffer` IBGDA decode kernels (per-expert padded layout, weighted combine, `use_fp8` e4m3fn), decode only, with EP8 wherever enabled, plus EP16 on GB200/GB300 (inside the MNNVL domain) and on B200's nscale bare-metal pool (IBGDA over native IB rails with `/dev/gdrdrv`, although the prior virtualized b200 pool could never run it). B300 is an unsupported coverage row in `low-latency`: the legacy Buffer self-enables NVSHMEM IBGDA even for a single-node EP8 run, and on B300 address-handle creation fails (`ibgda.cpp:2234 Unable to create ah`), rc255 on all eight ranks. `NVSHMEM_DISABLE_IB=1` does not help. The Buffer re-enables IBGDA regardless, and the run fails identically with it set and unset (measured on b300-002 and b300-011) |
-| MoRI | `production`, with vLLM `--all2all-backend mori_*`, SGLang `--moe-a2a-backend mori` | `normal` mode uses the direct `IntraNode` kernel for scale-up EP8 on every CDNA SKU. EP16 remains an unsupported coverage row on all three CDNA SKUs. Part of the old ROCm/mori#475 corruption was this harness passing dispatch's returned recv-slot indices to `combine()` instead of the rank's own routing (root-caused upstream, guarded by ROCm/mori#546, kernels unchanged) — with the corrected call, single-shot InterNodeV1 is clean through T=512 on mi355x — but a residual stochastic corruption remains from T~128 up under repeated execution and is near-certain at prefill sizes (run 33045314017/33050026476; unaffected by per-pair drains, so not a buffer-reuse race; upstream cannot reproduce on ionic driver 26.03 vs our 25.11). The tw pairs additionally have no cross-node GPU fabric, and mi355x EP16 has no publishable transport today: uccl-ep's CPU-proxy RDMA is functional on Pollara but ~13x under its documented bandwidth (~6 GB/s vs 82; unchanged by registration mode or traffic class — ionic-driver suspect, same 25.11-vs-26.03 delta as the mori residual). `low-latency` mode selects the `IntraNodeLL` decode kernel (single-call, pure-intranode, same compact layout and unweighted combine as `IntraNode`), decode/EP8 only. FP8 dispatch is caller-prequantized (per-SKU e4m3fnuz on gfx942, e4m3fn on gfx950). Combine stays BF16 (`quant_type=none`) alongside BF16 dispatch |
+| MoRI | `production`, with vLLM `--all2all-backend mori_*`, SGLang `--moe-a2a-backend mori` | `normal` mode uses the direct `IntraNode` kernel for scale-up EP8 on every CDNA SKU. EP16 remains an unsupported coverage row on all three CDNA SKUs. Part of the old ROCm/mori#475 corruption was this harness passing dispatch's returned recv-slot indices to `combine()` instead of the rank's own routing (root-caused upstream, guarded by ROCm/mori#546, kernels unchanged) — with the corrected call, single-shot InterNodeV1 is clean through T=512 on mi355x — but a residual stochastic corruption remains from T~128 up under repeated execution and is near-certain at prefill sizes (run 33045314017/33050026476; unaffected by per-pair drains, so not a buffer-reuse race; upstream cannot reproduce on ionic driver 26.03 vs our 25.11). The tw pairs additionally have no cross-node GPU fabric, and mi355x EP16 has no publishable transport today: uccl-ep's CPU-proxy RDMA is functional on Pollara but ~13x under its documented bandwidth (~6 GB/s vs 82; unchanged by registration mode or traffic class — ionic-driver suspect, same 25.11-vs-26.03 delta as the mori residual). `low-latency` mode selects the `AsyncLL` decode kernel driven split-phase (dispatch send+recv, combine send+recv — the kernel SGLang deploys; same compact layout and unweighted combine as `IntraNode`; earlier rows measured `IntraNodeLL`, discriminated by `kernel_generation`), decode/EP8 only. FP8 dispatch is caller-prequantized (per-SKU e4m3fnuz on gfx942, e4m3fn on gfx950). Combine stays BF16 (`quant_type=none`) alongside BF16 dispatch |
 | UCCL-EP | `candidate` (no engine exposes a UCCL-EP selector) | [UCCL](https://github.com/uccl-project/uccl) EP: a drop-in, API-identical DeepEP replacement whose CPU proxies issue GPUDirect RDMA over plain `libibverbs` (no NVSHMEM/IBGDA), with software message ordering, atomics, and flow control. Scale-up is single-node `cudaIpc` over NVLink/XGMI (never MNNVL). `normal` mode is the legacy `Buffer` `dispatch`/`combine` (unweighted rank-sum). `low-latency` reuses the legacy `low_latency_dispatch`/`low_latency_combine` decode kernels (weighted combine), decode/EP8 only. FP8 dispatch is caller-prequantized in `normal` mode (blockwise e4m3fn, per-SKU e4m3fnuz on gfx942). In `low-latency` mode the caller sends BF16 and the decode kernel quantizes to e4m3 internally (`use_fp8`). Combine is BF16. Runs on NVIDIA and AMD (H100/H200/B200 + MI300X/MI325X/MI355X), EP8 scale-up. Cross-node EP16 is functional (the internode RDMA path connects and the light case passes correctness) but its CPU-proxy throughput overruns the standardized per-case wall-clock budget on heavy token counts, so EP16 is an unsupported coverage row for now |
 | NCCL EP | `candidate` (NVIDIA's own library, but no engine exposes an NCCL-EP selector) | [NCCL EP](https://github.com/NVIDIA/nccl/tree/master/contrib/nccl_ep): NVIDIA's native MoE dispatch/combine on the NCCL Device API, using LSA (NVLink load/store) intra-node and GIN (GPU-Initiated Networking) inter-node, driven through the `nccl4py` bindings. `normal` mode selects the `HIGH_THROUGHPUT` algorithm (FLAT `[N, hidden]` receive, unweighted rank-sum combine). The `LOW_LATENCY` algorithm carries an EP8 `ll_backends` row on all six NVIDIA SKUs, restored once the single-handle fix removed the NVIDIA/nccl#2303 signal aliasing. That LL decode ladder is clamped to T<=128, below its 256-slot receive: `nccl_ep`'s combine recv pipeline is a port of DeepEP's pre-#642 kernel and is missing the same shared-memory fence before `mbarrier_arrive`, which corrupted T=256 on GB300 in 1 of 5 executions. It was bimodal, with healthy rows at 0.0039 relative error against 0.4704 on the failure. The fence is absent at NVIDIA/nccl master, so it is unfixed upstream. The clamp lowers exposure and is **not** a safety boundary: the fence is missing on every combine recv and T=256 is merely the rung with the most pipeline iterations, so lower rungs are less likely to hit the race rather than immune. Restore when a fixed wheel ships BF16 only: `contrib/nccl_ep/RELEASE.md` says "No FP8 support", so no FP8 case is emitted. That note is worth re-testing rather than trusting, because the C library at our pinned commit does read `inputs->scales` and switch on e4m3/e5m2, the two documented FP8 exclusions are expert-major layouts we do not use, and `NVIDIA/nccl` has not moved since 2026-06-11 while `NVIDIA/nccl-extensions` has replaced that row outright. NVIDIA-only and CUDA 13 only. EP8 scale-up on H100/H200/B200/B300 plus EP8 and EP16 on GB200/GB300, where EP16 stays inside the MNNVL scale-up domain. x86 EP16 scale-out is an unsupported coverage row: the cross-node GIN path faults inside `nccl_ep.cc` identically on RoCE and IB across four SKUs, a GDAKI limit rather than a fabric-selection one |
 | FlashInfer EP | `production`, with vLLM `--all2all-backend flashinfer_nvlink_one_sided` | [FlashInfer](https://github.com/flashinfer-ai/flashinfer) `MoeAlltoAll`: TensorRT-LLM's one-sided MNNVL all-to-all, where each rank writes tokens straight into its peers' workspace windows and combine reads them back, with no send/recv pairing and no NVSHMEM. `normal` mode only (there is one kernel family and no separate decode path), and GB200/GB300 only, since the transport is MNNVL. FP8 dispatch is caller-prequantized blockwise e4m3fn, carried as a fourth dispatch payload alongside its per-128-block FP32 scales, with the combine plane forced to BF16. The C++ `toNvDataType` accepts only fp16/bf16/fp32 for combine, so an FP8 combine buffer would raise rather than corrupt. EP8 and EP16, both inside the scale-up domain. Unlike every other backend here, its combine accumulates in the PAYLOAD dtype rather than FP32: wheels before 0.6.16 reduce the top-k contributions with a pairwise BF16 tree that rounds at every level, so the oracle models that reduction directly (`combine_reduction = "topk-slot-tree"`) instead of widening the tolerance. 0.6.16 moved the accumulator to FP32, and the adapter switches models on the installed version |

From 97944b44e458fab6ca09b954dda13b9a148258ad Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 31 Aug 2026 00:13:19 +0800
Subject: [PATCH 08/14] CollectiveX: rewrite the payload-bandwidth section
 around the wire basis (review)

The Measurement section still described the rank-deduplicated count as the
sole GB/s numerator and cited the retired IntraNodeLL as the dedup exemplar;
bandwidth.py and the app now divide from wire_byte_provenance. The section now
names both accountings, which rows differ, the 34% understatement that
motivated the change, and the fallback behavior for pre-wire artifacts.
---
 experimental/CollectiveX/docs/methodology.md | 34 ++++++++++++--------
 1 file changed, 20 insertions(+), 14 deletions(-)

diff --git a/experimental/CollectiveX/docs/methodology.md b/experimental/CollectiveX/docs/methodology.md
index affb8a2ad3..34ebf39080 100644
--- a/experimental/CollectiveX/docs/methodology.md
+++ b/experimental/CollectiveX/docs/methodology.md
@@ -448,20 +448,26 @@ one-sided kernel within 4% across eight byte-normalized points.
 
 Logical payload bandwidth is:
 
-`logical_payload_bytes / measured_latency_seconds`
-
-Payload bytes use rank-deduplicated token-rank activations and exclude expert metadata,
-padding, and backend buffer capacity. BF16 moves 2 bytes per value with no scale payload. An FP8
-dispatch moves 1 byte per value, plus per-128-block FP32 scales for every blockwise codec here (
-DeepEP V2, UCCL-EP and FlashInfer EP, which carries them as a fourth dispatch payload), and none for
-MoRI's plain e4m3 cast, while combine stays BF16, so the dispatch and combine directions can carry
-different byte counts and the roundtrip is their per-field sum. The rank-deduplicated count is exact
-for the normal-mode layout, and for a low-latency kernel that deduplicates per rank (MoRI's
-`IntraNodeLL`, whose combine is an unweighted rank-sum). The low-latency kernels that apply top-k
-weights inside combine instead send one copy per (token, expert) assignment rather than per
-(token, rank), so for a token whose experts share a destination rank this logical count is a lower
-bound on the bytes those kernels move. Each row states which basis it used in `logical_copies`, so
-the two are never silently mixed. Latency (the headline) is
+`wire_payload_bytes / measured_latency_seconds`
+
+Every row carries two byte accountings and each excludes expert metadata, padding, and backend
+buffer capacity. `byte_provenance` is the canonical comparable basis: rank-deduplicated
+token-rank activations, one copy per unique (token, dest-rank) pair. `wire_byte_provenance` is
+what the kernels actually move: identical to the canonical basis for every layout that
+deduplicates per rank (all normal modes, and MoRI's low-latency kernels, whose combine is an
+unweighted rank-sum), and one copy per (token, expert) assignment for the low-latency kernels
+that apply top-k weights inside combine (DeepEP V2, UCCL-EP, NCCL EP). For a token whose experts
+share a destination rank the deduplicated count is a lower bound on those kernels' traffic —
+34% low on nccl-ep low-latency EP8 at T=128 — which is why every emitted GB/s divides from the
+WIRE basis; a rate derived from `byte_provenance` on such a row is a lower bound, not the wire
+rate, and is not comparable across backends. Artifacts written before `wire_byte_provenance`
+existed fall back to the deduplicated basis, which only ever understates. `logical_copies`
+states each row's wire basis (`routed`, `assignments`, `wire`), so the two are never silently
+mixed. BF16 moves 2 bytes per value with no scale payload. An FP8 dispatch moves 1 byte per
+value, plus per-128-block FP32 scales for every blockwise codec here (DeepEP V2, UCCL-EP and
+FlashInfer EP, which carries them as a fourth dispatch payload), and none for MoRI's plain e4m3
+cast, while combine stays BF16, so the dispatch and combine directions can carry different byte
+counts and the roundtrip is their per-field sum. Latency (the headline) is
 measured directly and is unaffected. Algorithm bandwidth, bus bandwidth,
 wire utilization, and physical-link utilization are not emitted without a defined primitive model or
 transport counters. Logical bandwidth must never be labeled physical bandwidth. Payload and token

From 504601f35b63662388fe8b6ac0ea3751d14027b1 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 31 Aug 2026 11:03:31 +0800
Subject: [PATCH 09/14] CollectiveX: exclude b300-001 and b300-016 (mlx5_4
 DOWN)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The full v4-pro sweep's two b300 EP16 legs each drew one of these nodes and
failed closed at the network-profile gate (rdma-port-5=inactive = the 5th
selector, mlx5_4). Idle-pool survey: exactly these two nodes have mlx5_4 port
state 1: DOWN; the other 12 idle nodes are 4: ACTIVE — a sick node pair
(likely one leaf/cable pair), not a rail-wide outage. SRE handback owed.
---
 experimental/CollectiveX/configs/platform_config.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/experimental/CollectiveX/configs/platform_config.json b/experimental/CollectiveX/configs/platform_config.json
index 4ed76b4114..800b7c6c59 100644
--- a/experimental/CollectiveX/configs/platform_config.json
+++ b/experimental/CollectiveX/configs/platform_config.json
@@ -84,7 +84,7 @@
         "account": "benchmark",
         "qos": "batch_1_qos",
         "squash_dir": "/data/home/sa-shared/sqsh",
-        "exclude_nodes": "b300-005,b300-009,b300-018"
+        "exclude_nodes": "b300-001,b300-005,b300-009,b300-016,b300-018"
       },
       "network": {
         "socket_ifname": "bond0",

From e9ef09940dee106522c1920119fb666ae0623618 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 31 Aug 2026 11:12:48 +0800
Subject: [PATCH 10/14] CollectiveX: hold gb300 low-latency until the
 nvshmem-over-MNNVL init regression is resolved
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

All gb300 deepep-v2 LL shards die at the first CUDA op after Buffer init with
cudaErrorUnknown (context poisoned during nvshmem symmetric-heap init over
MNNVL), on every rank, both degrees and precisions. Discrimination: gb300
normal mode green at n2/n4; gb200 LL green with the same image/pin/workload;
a main-branch control with the old workload fails identically — so neither
this PR nor the v4-pro shape. CI-shaped 2-node hand probe (job 27430 on the
pool) reproduces: bootstrap and buffer creation complete, then every rank's
next CUDA call returns unknown error. Driver is 580.159.03 (installed in the
KV-fix window ~2026-08-26..28, after the last green gb300 LL run). Platform
escalation owed; rows held so sweeps don't burn four known-red shards each.
---
 experimental/CollectiveX/configs/platform_config.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/experimental/CollectiveX/configs/platform_config.json b/experimental/CollectiveX/configs/platform_config.json
index 800b7c6c59..3e2ac373a2 100644
--- a/experimental/CollectiveX/configs/platform_config.json
+++ b/experimental/CollectiveX/configs/platform_config.json
@@ -124,7 +124,7 @@
       "scale_up_transport": "mnnvl",
       "launcher": "gb-nv",
       "backends": {"deepep-v2": [8, 16], "nccl-ep": [8, 16], "flashinfer-ep": [8, 16]},
-      "ll_backends": {"deepep-v2": [8, 16]},
+      "ll_backends": {},
       "fabric": {"nic": "MNNVL (scale-out not used)", "switch": "NVLink NVL72"},
       "operator": {
         "partition": "batch_1",

From af82a766b1f9f4d588462491f30a9b888c347a21 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Mon, 31 Aug 2026 12:14:49 +0800
Subject: [PATCH 11/14] CollectiveX: bump the mi355x image to
 v0.5.18-rocm720-mi35x-20260830 (mori >= #505)
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

AsyncLL under the DSv4-Pro workload (topk 6) dies on every shipped
mi35x-mori image: ROCm/mori#505 (AsyncLL slot assignment double-allocates
when top-k does not divide warpSize, fixed upstream 2026-07-31) postdates
them all. On-metal probe of this date-stamped nightly (job 41563, pure
mori.ops): AsyncLL topk6@384 and topk8@256 both pass, split-phase recv API
present. mi355x only — the -tw SKUs keep their image until separately
validated on their docker path.
---
 experimental/CollectiveX/configs/platform_config.json | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/experimental/CollectiveX/configs/platform_config.json b/experimental/CollectiveX/configs/platform_config.json
index 3e2ac373a2..fddffafb00 100644
--- a/experimental/CollectiveX/configs/platform_config.json
+++ b/experimental/CollectiveX/configs/platform_config.json
@@ -164,7 +164,7 @@
     "mi355x": {
       "arch": "gfx950",
       "product": "mi355x",
-      "image": "rocm/sgl-dev:sglang-0.5.14-rocm720-mi35x-mori-0701",
+      "image": "rocm/sgl-dev:v0.5.18-rocm720-mi35x-20260830",
       "image_platform": "linux/amd64",
       "gpus_per_node": 8,
       "scale_up_domain": 8,

From fe74a8976732d7db343a7b0522428f47c776e6e8 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 1 Sep 2026 10:01:26 +0800
Subject: [PATCH 12/14] =?UTF-8?q?CollectiveX:=20build=20DeepEP=20against?=
 =?UTF-8?q?=20the=20image-matched=20nvshmem=20(cu13)=20=E2=80=94=20un-brea?=
 =?UTF-8?q?ks=20gb300=20LL?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The gb300 LL cudaErrorUnknown was ours, not the platform's. 2x2 on the pool:
stock in-image deep_ep (built on nvidia-nvshmem-cu13==3.4.5) runs LL clean on
the exact r01 nodes and driver (580.159.03) where the CI stack dies; the CI
venv's nvidia-nvshmem-cu12==3.3.9 r12 host library survives sm90/sm100 but
poisons the CUDA context during symmetric-heap init over MNNVL on sm103 —
buffer creation returns, the next CUDA call on every rank fails. The earlier
driver-window attribution was a coincidence of timing (pin bump and driver
install overlapped; hold commit e9ef09940 superseded).

The wheel spec now lives in common.sh (COLLX_DEEPEP_V2_NVSHMEM_SPEC), is
installed by prepare_backend, and keys the venv cache so every NVIDIA deepep
venv rebuilds once against cu13. gb300 ll_backends restored.
---
 .../CollectiveX/configs/platform_config.json       |  2 +-
 experimental/CollectiveX/runtime/common.sh         |  8 ++++++++
 .../CollectiveX/runtime/prepare_backend.sh         | 14 ++++++++++----
 3 files changed, 19 insertions(+), 5 deletions(-)

diff --git a/experimental/CollectiveX/configs/platform_config.json b/experimental/CollectiveX/configs/platform_config.json
index fddffafb00..47a0ffe98c 100644
--- a/experimental/CollectiveX/configs/platform_config.json
+++ b/experimental/CollectiveX/configs/platform_config.json
@@ -124,7 +124,7 @@
       "scale_up_transport": "mnnvl",
       "launcher": "gb-nv",
       "backends": {"deepep-v2": [8, 16], "nccl-ep": [8, 16], "flashinfer-ep": [8, 16]},
-      "ll_backends": {},
+      "ll_backends": {"deepep-v2": [8, 16]},
       "fabric": {"nic": "MNNVL (scale-out not used)", "switch": "NVLink NVL72"},
       "operator": {
         "partition": "batch_1",
diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index 73388776ca..aadbb84758 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -21,6 +21,14 @@ COLLX_DEEPEP_V2_REPO="https://github.com/deepseek-ai/DeepEP"
 # resolution for pip wheels. The backend cache is keyed on this value, so a change forces a rebuild.
 COLLX_DEEPEP_V2_COMMIT="01dc3aaac82068020353dce2c302e38153c0bfaa"
 
+# NVSHMEM wheel for the DeepEP V2 build. Must match the image's CUDA line: the cu12
+# wheel's r12 host library on the cu130 images survives on sm90/sm100 but poisons the
+# CUDA context during symmetric-heap init over MNNVL on sm103 (gb300) — buffer creation
+# returns, then every subsequent CUDA call fails cudaErrorUnknown. Stock in-image deep_ep
+# (built against this exact cu13 wheel) runs clean on the same nodes/driver, which is how
+# the wheel was isolated. Folded into the venv cache key, so a change forces a rebuild.
+COLLX_DEEPEP_V2_NVSHMEM_SPEC="nvidia-nvshmem-cu13==3.4.5"
+
 COLLX_UCCL_REPO="https://github.com/uccl-project/uccl"
 COLLX_UCCL_COMMIT="fc1b582031221645ea9fce58aeb57187713145e3"
 
diff --git a/experimental/CollectiveX/runtime/prepare_backend.sh b/experimental/CollectiveX/runtime/prepare_backend.sh
index fed9b6ec03..d035c90840 100644
--- a/experimental/CollectiveX/runtime/prepare_backend.sh
+++ b/experimental/CollectiveX/runtime/prepare_backend.sh
@@ -129,8 +129,14 @@ deepep_cache_root() {
   base="${COLLX_BACKEND_CACHE_ROOT:-}"
   [[ "$base" = /* ]] || return 1
   image="$(printf '%s' "${COLLECTIVEX_IMAGE:-manual}" | tr -cs 'A-Za-z0-9_.-' '-')"
-  printf '%s/deepep-v2-%s-sm%s-%s-%s' \
-    "$base" "$cpu" "${arch/./}" "${image#-}" "${COLLX_DEEPEP_V2_COMMIT:0:12}"
+  # The NVSHMEM wheel is part of the built venv's identity (see common.sh: the cu12
+  # wheel on cu130 images broke sm103), so it keys the cache and a spec change rebuilds.
+  local nvshmem_key="${COLLX_DEEPEP_V2_NVSHMEM_SPEC#nvidia-}"
+  nvshmem_key="${nvshmem_key//==/-}"
+  [[ "$nvshmem_key" =~ ^[A-Za-z0-9._-]+$ ]] || return 1
+  printf '%s/deepep-v2-%s-sm%s-%s-%s-%s' \
+    "$base" "$cpu" "${arch/./}" "${image#-}" "${COLLX_DEEPEP_V2_COMMIT:0:12}" \
+    "$nvshmem_key"
 }
 
 deepep_activate() {
@@ -145,7 +151,7 @@ deepep_activate() {
   nccl_root="$(nvidia_package_root "$venv/bin/python" nvidia-nccl-cu13 nccl)" \
     || { collx_log "ERROR: DeepEP V2 NCCL package root is unavailable"; return 1; }
   nvshmem_package="$(nvidia_package_root \
-    "$venv/bin/python" nvidia-nvshmem-cu12 nvshmem)" \
+    "$venv/bin/python" "${COLLX_DEEPEP_V2_NVSHMEM_SPEC%%==*}" nvshmem)" \
     || { collx_log "ERROR: DeepEP V2 NVSHMEM package root is unavailable"; return 1; }
   overlay="$(deepep_nvshmem_overlay "$root" "$nvshmem_package")" || return 1
   toolchain="$(cuda_toolchain_paths)" || return 1
@@ -203,7 +209,7 @@ deepep_install() {
   pip=("$venv/bin/python" -m pip install -q --disable-pip-version-check --no-input)
   "${pip[@]}" \
     "pip==26.1.2" "setuptools==82.0.1" "wheel==0.47.0" "ninja==1.13.0" \
-    "numpy==2.2.6" "nvidia-nvshmem-cu12==3.3.9" >&2 2>&1 \
+    "numpy==2.2.6" "$COLLX_DEEPEP_V2_NVSHMEM_SPEC" >&2 2>&1 \
     || { collx_log "ERROR: DeepEP V2 build-tool installation failed"; return 1; }
   "${pip[@]}" --index-url https://download.pytorch.org/whl/cu130 \
     --extra-index-url https://pypi.org/simple "torch==2.10.0" >&2 2>&1 \

From 6f91cf3b0304f66ffdc38f62390a96ce185bab1f Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 1 Sep 2026 13:11:45 +0800
Subject: [PATCH 13/14] =?UTF-8?q?CollectiveX:=20torch=202.11.0=20for=20the?=
 =?UTF-8?q?=20DeepEP=20venv=20=E2=80=94=20the=20actual=20gb300=20LL=20fix?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

Root cause, finally isolated by holding everything else constant: a venv
identical to prepare_backend's recipe (same DeepEP pin, same nccl-cu13 2.30.4,
same nvshmem-cu13 3.4.5) is green with torch==2.11.0+cu130 and red with
torch==2.10.0+cu130 on the same gb300 r01 nodes (jobs 27760 vs 27430/27757).
torch 2.10.0's bundled CUDA-13 userland poisons the CUDA context during
nvshmem symmetric-heap init over MNNVL on sm103 + driver 580.159.03; sm90 and
sm100 are unaffected. Falsified along the way, in order: the v4-pro workload
(main control), the driver alone (stock image green on the failing nodes —
the user's r03 canary forced that 2x2), the nvshmem cu12 wheel (cu13 rebuild
still red), the DeepEP pin (old-pin venv red). 2.11.0 is what the cu130 image
ships. The torch spec joins the nvshmem spec in the venv cache key, so every
deepep venv rebuilds once.
---
 experimental/CollectiveX/runtime/common.sh          | 9 +++++++++
 experimental/CollectiveX/runtime/prepare_backend.sh | 9 +++++----
 2 files changed, 14 insertions(+), 4 deletions(-)

diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index aadbb84758..c270e13305 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -29,6 +29,15 @@ COLLX_DEEPEP_V2_COMMIT="01dc3aaac82068020353dce2c302e38153c0bfaa"
 # the wheel was isolated. Folded into the venv cache key, so a change forces a rebuild.
 COLLX_DEEPEP_V2_NVSHMEM_SPEC="nvidia-nvshmem-cu13==3.4.5"
 
+# Torch for the DeepEP V2 venv. 2.10.0+cu130's bundled CUDA userland poisons the CUDA
+# context during nvshmem symmetric-heap init over MNNVL on sm103/driver 580.159.03 (gb300):
+# buffer creation returns, every rank's next CUDA call fails cudaErrorUnknown. Isolated by
+# a same-recipe venv that differs ONLY in torch (2.11.0 green, 2.10.0 red, on the same
+# nodes; jobs 27760 vs 27430) — the pin, the nvshmem wheel, the rack and the driver were
+# each falsified first. 2.11.0 is also what the cu130 image itself ships. Folded into the
+# venv cache key, so a change forces a rebuild.
+COLLX_DEEPEP_V2_TORCH_SPEC="torch==2.11.0"
+
 COLLX_UCCL_REPO="https://github.com/uccl-project/uccl"
 COLLX_UCCL_COMMIT="fc1b582031221645ea9fce58aeb57187713145e3"
 
diff --git a/experimental/CollectiveX/runtime/prepare_backend.sh b/experimental/CollectiveX/runtime/prepare_backend.sh
index d035c90840..6a34ba0adb 100644
--- a/experimental/CollectiveX/runtime/prepare_backend.sh
+++ b/experimental/CollectiveX/runtime/prepare_backend.sh
@@ -133,10 +133,11 @@ deepep_cache_root() {
   # wheel on cu130 images broke sm103), so it keys the cache and a spec change rebuilds.
   local nvshmem_key="${COLLX_DEEPEP_V2_NVSHMEM_SPEC#nvidia-}"
   nvshmem_key="${nvshmem_key//==/-}"
-  [[ "$nvshmem_key" =~ ^[A-Za-z0-9._-]+$ ]] || return 1
-  printf '%s/deepep-v2-%s-sm%s-%s-%s-%s' \
+  local torch_key="${COLLX_DEEPEP_V2_TORCH_SPEC//==/-}"
+  [[ "$nvshmem_key" =~ ^[A-Za-z0-9._-]+$ && "$torch_key" =~ ^[A-Za-z0-9._-]+$ ]] || return 1
+  printf '%s/deepep-v2-%s-sm%s-%s-%s-%s-%s' \
     "$base" "$cpu" "${arch/./}" "${image#-}" "${COLLX_DEEPEP_V2_COMMIT:0:12}" \
-    "$nvshmem_key"
+    "$torch_key" "$nvshmem_key"
 }
 
 deepep_activate() {
@@ -212,7 +213,7 @@ deepep_install() {
     "numpy==2.2.6" "$COLLX_DEEPEP_V2_NVSHMEM_SPEC" >&2 2>&1 \
     || { collx_log "ERROR: DeepEP V2 build-tool installation failed"; return 1; }
   "${pip[@]}" --index-url https://download.pytorch.org/whl/cu130 \
-    --extra-index-url https://pypi.org/simple "torch==2.10.0" >&2 2>&1 \
+    --extra-index-url https://pypi.org/simple "$COLLX_DEEPEP_V2_TORCH_SPEC" >&2 2>&1 \
     || { collx_log "ERROR: torch 2.10.0+cu130 installation failed"; return 1; }
   # Torch pins NCCL 2.28.9; ElasticBuffer requires 2.30.4.
   "${pip[@]}" --force-reinstall --no-deps "nvidia-nccl-cu13==2.30.4" >&2 2>&1 \

From b8b746740d113b917e5a7159274396bb171fb8a0 Mon Sep 17 00:00:00 2001
From: Oseltamivir <58582368+Oseltamivir@users.noreply.github.com>
Date: Tue, 1 Sep 2026 14:14:03 +0800
Subject: [PATCH 14/14] =?UTF-8?q?CollectiveX:=20pin=20the=20RDC=20device-l?=
 =?UTF-8?q?ink=20arch=20=E2=80=94=20the=20real=20gb300=20LL=20root=20cause?=
MIME-Version: 1.0
Content-Type: text/plain; charset=UTF-8
Content-Transfer-Encoding: 8bit

The DeepEP build compiles every object with the right -gencode (setup.py
'Arch list: 10.3', all compile lines sm_103), but the final nvcc -dlink step
receives NO gencode and falls back to nvcc's CUDA-13 default, sm_75 — the
.so's single linked device image is sm_75 and its kernels can never load on
the target GPU. On gb300/sm103 that surfaced as cudaErrorUnknown at the first
legacy-kernel launch (LL and layout paths; ElasticBuffer JIT paths were
unaffected, which is why normal mode stayed green). Proven by instrumented
runs of prepare_backend's own build (job 27792: compile lines sm_103, step
9/9 -dlink bare, product sm_75) after hand builds of the identical recipe
produced sm_103 and ran LL clean on the failing nodes.

Fix: NVCC_PREPEND_FLAGS carries -gencode=arch=compute_<arch>,code=sm_<arch>
into every nvcc invocation including the dlink. COLLX_DEEPEP_V2_BUILD_GEN
(dlarch1) joins the cache key so the sm_75 venvs with valid .ready markers
rebuild instead of being reused.

The torch 2.10->2.11 bump (previous commit) stays: 2.11 matches the image and
the 2.10-vs-2.11 A/B was run against differently-linked binaries, so its
verdict is voided rather than reversed; the pin question can be revisited
upstream of this fix if anyone cares to re-test.
---
 experimental/CollectiveX/runtime/common.sh       |  7 +++++++
 .../CollectiveX/runtime/prepare_backend.sh       | 16 +++++++++++++---
 2 files changed, 20 insertions(+), 3 deletions(-)

diff --git a/experimental/CollectiveX/runtime/common.sh b/experimental/CollectiveX/runtime/common.sh
index c270e13305..26b88848c9 100644
--- a/experimental/CollectiveX/runtime/common.sh
+++ b/experimental/CollectiveX/runtime/common.sh
@@ -38,6 +38,13 @@ COLLX_DEEPEP_V2_NVSHMEM_SPEC="nvidia-nvshmem-cu13==3.4.5"
 # venv cache key, so a change forces a rebuild.
 COLLX_DEEPEP_V2_TORCH_SPEC="torch==2.11.0"
 
+# Build-recipe generation for the DeepEP venv cache key. Bump when the BUILD FLAGS
+# change without any pin changing — "dlarch1" marks the fix that pins the RDC
+# device-link arch via NVCC_PREPEND_FLAGS (the bare dlink defaulted to sm_75 and
+# produced unloadable kernels; venvs built before this carry .ready and would
+# otherwise be reused broken).
+COLLX_DEEPEP_V2_BUILD_GEN="dlarch1"
+
 COLLX_UCCL_REPO="https://github.com/uccl-project/uccl"
 COLLX_UCCL_COMMIT="fc1b582031221645ea9fce58aeb57187713145e3"
 
diff --git a/experimental/CollectiveX/runtime/prepare_backend.sh b/experimental/CollectiveX/runtime/prepare_backend.sh
index 6a34ba0adb..eaf453cd15 100644
--- a/experimental/CollectiveX/runtime/prepare_backend.sh
+++ b/experimental/CollectiveX/runtime/prepare_backend.sh
@@ -134,10 +134,12 @@ deepep_cache_root() {
   local nvshmem_key="${COLLX_DEEPEP_V2_NVSHMEM_SPEC#nvidia-}"
   nvshmem_key="${nvshmem_key//==/-}"
   local torch_key="${COLLX_DEEPEP_V2_TORCH_SPEC//==/-}"
-  [[ "$nvshmem_key" =~ ^[A-Za-z0-9._-]+$ && "$torch_key" =~ ^[A-Za-z0-9._-]+$ ]] || return 1
-  printf '%s/deepep-v2-%s-sm%s-%s-%s-%s-%s' \
+  local build_gen="${COLLX_DEEPEP_V2_BUILD_GEN:?}"
+  [[ "$nvshmem_key" =~ ^[A-Za-z0-9._-]+$ && "$torch_key" =~ ^[A-Za-z0-9._-]+$ \
+     && "$build_gen" =~ ^[A-Za-z0-9._-]+$ ]] || return 1
+  printf '%s/deepep-v2-%s-sm%s-%s-%s-%s-%s-%s' \
     "$base" "$cpu" "${arch/./}" "${image#-}" "${COLLX_DEEPEP_V2_COMMIT:0:12}" \
-    "$torch_key" "$nvshmem_key"
+    "$torch_key" "$nvshmem_key" "$build_gen"
 }
 
 deepep_activate() {
@@ -222,7 +224,15 @@ deepep_install() {
     || { collx_log "ERROR: DeepEP V2 environment activation failed"; return 1; }
   collx_materialize_deepep_source "$source_dir" \
     || { collx_log "ERROR: DeepEP V2 staged source is invalid"; return 1; }
+  # The RDC device-link step (nvcc -dlink) receives NO -gencode from the extension
+  # build, so nvcc falls back to ITS default arch (sm_75 on CUDA 13) and relinks the
+  # correctly-compiled sm-specific objects into an sm_75 device image — kernels that
+  # can never load on the target GPU (gb300/sm103: cudaErrorUnknown at first launch;
+  # proven by build log: every compile line -gencode sm_103, step 9/9 -dlink bare).
+  # NVCC_PREPEND_FLAGS reaches every nvcc invocation including the dlink.
+  local gencode="-gencode=arch=compute_${arch/./},code=sm_${arch/./}"
   (cd "$source_dir" && TORCH_CUDA_ARCH_LIST="$arch" MAX_JOBS=16 \
+    NVCC_PREPEND_FLAGS="$gencode ${NVCC_PREPEND_FLAGS:-}" \
     "$venv/bin/python" -m pip install -q --no-build-isolation --no-deps \
       --force-reinstall .) >&2 2>&1 \
     || { collx_log "ERROR: DeepEP V2 build failed"; return 1; }