[None][test] Skip MoE A2A CFT regression under CUDA forward compatibility

tests/unittest/_torch/moe/multi_gpu/test_moe_a2a_workspace.py

@@ -5,10 +5,12 @@
 import ctypes
 import faulthandler
 import pickle
+import re
 import sys
 import traceback
 
 import cloudpickle
+import pynvml
 import pytest
 import torch
 from mpi4py import MPI
@@ -81,6 +83,39 @@ def _cft_skip_reason():
         )
         if result != 0 or status.value != 0 or not pointer.value:
             return f"CUDA driver lacks cuLogicalEndpoint{suffix}"
+    return _forward_compat_reason()
+
+
+def _forward_compat_reason():
+    """Skip when the loaded libcuda is newer than the kernel driver.
+
+    Under CUDA forward compatibility the user-mode driver exports the
+    cuLogicalEndpoint entry points, but creating an endpoint needs
+    kernel-driver support and fails with CUDA_ERROR_INVALID_VALUE.
+    """
+    user_mode = None
+    with open("/proc/self/maps") as maps:
+        for line in maps:
+            match = re.search(r"libcuda\.so\.(\d+\.\d+(?:\.\d+)?)", line)
+            if match:
+                user_mode = match.group(1)
+                break
+    pynvml.nvmlInit()
+    try:
+        kernel = pynvml.nvmlSystemGetDriverVersion()
+    finally:
+        pynvml.nvmlShutdown()
+    if isinstance(kernel, bytes):
+        kernel = kernel.decode()
+
+    def version(text):
+        return tuple(int(part) for part in text.split("."))
+
+    if user_mode is not None and version(user_mode) > version(kernel):
+        return (
+            f"CFT logical endpoints need kernel-driver support; CUDA forward compatibility "
+            f"runs user-mode driver {user_mode} on kernel driver {kernel}"
+        )
     return None