From bf09ba6b6e2e86e33f3fe466a3d08dd74822f771 Mon Sep 17 00:00:00 2001 From: Ruocheng Jia Date: Sat, 26 Sep 2026 03:07:22 -0700 Subject: [PATCH 1/2] [None][test] Skip MoE A2A CFT regression under CUDA forward compatibility Creating a CFT logical endpoint needs kernel-driver support. Under CUDA forward compatibility the newer user-mode driver still exports the cuLogicalEndpoint entry points, so the existing gate lets the test run, and cuLogicalEndpointCreate then fails with CUDA_ERROR_INVALID_VALUE. The worker aborts MPI_COMM_WORLD and takes down the whole test process. Skip the CFT cases when the loaded libcuda is newer than the kernel driver reported by NVML; real CFT failures on a matching driver still surface. Signed-off-by: Ruocheng Jia --- .../moe/multi_gpu/test_moe_a2a_workspace.py | 31 +++++++++++++++++++ 1 file changed, 31 insertions(+) diff --git a/tests/unittest/_torch/moe/multi_gpu/test_moe_a2a_workspace.py b/tests/unittest/_torch/moe/multi_gpu/test_moe_a2a_workspace.py index a857fb587c76..1bc8d6370d9f 100644 --- a/tests/unittest/_torch/moe/multi_gpu/test_moe_a2a_workspace.py +++ b/tests/unittest/_torch/moe/multi_gpu/test_moe_a2a_workspace.py @@ -5,10 +5,12 @@ import ctypes import faulthandler import pickle +import re import sys import traceback import cloudpickle +import pynvml import pytest import torch from mpi4py import MPI @@ -81,6 +83,35 @@ def _cft_skip_reason(): ) if result != 0 or status.value != 0 or not pointer.value: return f"CUDA driver lacks cuLogicalEndpoint{suffix}" + return _forward_compat_reason() + + +def _forward_compat_reason(): + """Skip when the loaded libcuda is newer than the kernel driver. + + Under CUDA forward compatibility the user-mode driver exports the + cuLogicalEndpoint entry points, but creating an endpoint needs + kernel-driver support and fails with CUDA_ERROR_INVALID_VALUE. + """ + user_mode = None + with open("/proc/self/maps") as maps: + for line in maps: + match = re.search(r"libcuda\.so\.(\d+\.\d+(?:\.\d+)?)", line) + if match: + user_mode = match.group(1) + break + pynvml.nvmlInit() + try: + kernel = pynvml.nvmlSystemGetDriverVersion() + finally: + pynvml.nvmlShutdown() + if isinstance(kernel, bytes): + kernel = kernel.decode() + if user_mode is not None and user_mode != kernel: + return ( + f"CFT logical endpoints need kernel-driver support; CUDA forward compatibility " + f"runs user-mode driver {user_mode} on kernel driver {kernel}" + ) return None From 90b54c110f461a39798dfe0e7893958ea647f06e Mon Sep 17 00:00:00 2001 From: Ruocheng Jia Date: Sat, 26 Sep 2026 08:32:25 -0700 Subject: [PATCH 2/2] [None][test] Gate MoE A2A CFT skip on a newer user-mode driver Forward compatibility pairs a newer user-mode libcuda with an older kernel driver; compare versions numerically instead of skipping on any mismatch. Signed-off-by: Ruocheng Jia --- .../unittest/_torch/moe/multi_gpu/test_moe_a2a_workspace.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/tests/unittest/_torch/moe/multi_gpu/test_moe_a2a_workspace.py b/tests/unittest/_torch/moe/multi_gpu/test_moe_a2a_workspace.py index 1bc8d6370d9f..9b4bad7ec9cb 100644 --- a/tests/unittest/_torch/moe/multi_gpu/test_moe_a2a_workspace.py +++ b/tests/unittest/_torch/moe/multi_gpu/test_moe_a2a_workspace.py @@ -107,7 +107,11 @@ def _forward_compat_reason(): pynvml.nvmlShutdown() if isinstance(kernel, bytes): kernel = kernel.decode() - if user_mode is not None and user_mode != kernel: + + def version(text): + return tuple(int(part) for part in text.split(".")) + + if user_mode is not None and version(user_mode) > version(kernel): return ( f"CFT logical endpoints need kernel-driver support; CUDA forward compatibility " f"runs user-mode driver {user_mode} on kernel driver {kernel}"