Land immediately verifiable P2P safety fixes

README.md

@@ -1,7 +1,7 @@
 # NVIDIA driver 610.43.03 with P2P for RTX 3090, RTX 4090, and RTX 5090
 
-This enables P2P on consumer GPUs with the 610.43.03 driver version. No kernel parameters
-are needed for the default behavior, just build, install, and go.
+This enables P2P on consumer GPUs with the 610.43.03 driver version. The current branch
+requires the IOMMU passthrough configuration described below.
 
 See the [tinygrad 550.54.15-p2p README](https://github.com/tinygrad/open-gpu-kernel-modules/blob/550.54.15-p2p/README.md)
 for the original description of the approach.
@@ -24,9 +24,10 @@ NVLink where it is. For PCIe pairs, transfers write directly to the other GPU's
 address over DMA.
 
 > [!WARNING]
-> IOMMU must be in passthrough mode (`iommu=pt`), not translating, or DMA will go through
-> IOMMU page tables and transfers will fail. This is very dangerous if you run untrusted
-> software or devices.
+> IOMMU must currently be in passthrough mode (`iommu=pt`), not translating. In particular,
+> the experimental hugetlb registration path does not yet handle scatterlist entries merged
+> by a translated IOMMU. Do not use translated mode until that path is fixed and validated.
+> Passthrough mode weakens DMA isolation and is unsafe with untrusted software or devices.
 
 ## How to use
 
@@ -52,17 +53,19 @@ options nvidia NVreg_RegistryDwords="RMForceP2PType=1"
 
 This branch also includes an experimental path that accelerates `cudaHostRegister` by
 several orders of magnitude when the registered buffer is backed by 1G hugepages, and
-shrinks the device page tables used for such mappings. It is enabled automatically. This
-path skips some of the per-4K-page bookkeeping the stock driver performs, so it may
-misbehave in edge cases the stock driver handles correctly.
+shrinks the device page tables used for such mappings. It is enabled automatically for a
+non-empty registration that is hugepage-aligned, is an exact multiple of the hugepage
+size, and stays within one hugetlb VMA. Other layouts use the normal per-page array path.
+The fast path still skips some base-page bookkeeping and remains experimental.
 
 ## Potential issues
 
 If P2P transfers are slow, make sure your IOMMU is in passthrough (`pt`) mode and that ACS
-is disabled. ACS on root ports forces all GPU-to-GPU traffic through the CPU root complex,
-killing P2P bandwidth. ACS can be disabled in BIOS, with the
-`pcie_acs_override=downstream,multifunction` kernel parameter (if your kernel supports it),
-or with an ACS override patch applied to the kernel.
+redirect is not forcing GPU-to-GPU traffic through the root complex. Prefer a firmware ACS
+control. If the kernel supports the upstream per-device option, use a narrowly scoped
+`pci=disable_acs_redir=<BDF>[;<BDF>...]` setting and verify the resulting IOMMU groups.
+Disabling ACS redirect weakens device isolation; do not use the broad `pcie_acs_override`
+patch or kernel parameter.
 
 ## Sample `p2pBandwidthLatencyTest` output
 

kernel-open/nvidia/os-mlock.c

@@ -264,6 +264,54 @@ static void nv_free_page_array(struct page **pages)
     os_free_mem((NvU8 *)pages - NV_PAGE_ARRAY_HEADER_SIZE);
 }
 
+static NvBool nv_hugetlb_fast_path_eligible(
+    struct vm_area_struct *vma,
+    unsigned long start,
+    NvU64 page_count,
+    unsigned long *hpage_size_out,
+    unsigned int *compound_order_out
+)
+{
+    NvU64 range_size;
+    NvU64 pages_per_hugepage;
+    unsigned long end;
+    unsigned long hpage_size;
+
+    if (!vma || !is_vm_hugetlb_page(vma) || page_count == 0)
+        return NV_FALSE;
+
+    if (page_count > ((NvU64)ULONG_MAX / PAGE_SIZE))
+        return NV_FALSE;
+
+    range_size = page_count * PAGE_SIZE;
+    if (range_size > ULONG_MAX || start > (ULONG_MAX - (unsigned long)range_size))
+        return NV_FALSE;
+
+    end = start + (unsigned long)range_size;
+    hpage_size = vma_kernel_pagesize(vma);
+
+    if (hpage_size < PAGE_SIZE || hpage_size > NV_U32_MAX ||
+        (hpage_size & (hpage_size - 1)) != 0)
+    {
+        return NV_FALSE;
+    }
+
+    if ((start & (hpage_size - 1)) != 0 ||
+        (range_size & (hpage_size - 1)) != 0 ||
+        end > vma->vm_end)
+    {
+        return NV_FALSE;
+    }
+
+    pages_per_hugepage = hpage_size / PAGE_SIZE;
+    if ((pages_per_hugepage & (pages_per_hugepage - 1)) != 0)
+        return NV_FALSE;
+
+    *hpage_size_out = hpage_size;
+    *compound_order_out = ilog2(pages_per_hugepage);
+    return NV_TRUE;
+}
+
 NV_STATUS NV_API_CALL os_lock_user_pages(
     void   *address,
     NvU64   page_count,
@@ -303,47 +351,48 @@ NV_STATUS NV_API_CALL os_lock_user_pages(
      * unfaulted 1GB hugepages (~37ms per fault).
      */
     {
-        struct vm_area_struct *vma = vma_lookup(mm, (unsigned long)address);
-
-        if (vma && is_vm_hugetlb_page(vma))
+        unsigned long start = (unsigned long)address;
+        struct vm_area_struct *vma = vma_lookup(mm, start);
+        unsigned long hpage_size;
+        unsigned int order;
+
+        if (nv_hugetlb_fast_path_eligible(vma,
+                                          start,
+                                          page_count,
+                                          &hpage_size,
+                                          &order))
         {
-            unsigned long hpage_size = vma_kernel_pagesize(vma);
-            unsigned int order = ilog2(hpage_size / PAGE_SIZE);
-            NvU64 pages_per_hp = 1ULL << order;
+            NvU64 num_hugepages = page_count >> order;
 
-            if ((page_count & (pages_per_hp - 1)) == 0)
+            rmStatus = nv_alloc_page_array(num_hugepages, order, &user_pages);
+            if (rmStatus != NV_OK)
             {
-                NvU64 num_hugepages = page_count >> order;
+                nv_mmap_read_unlock(mm);
+                nv_printf(NV_DBG_ERRORS,
+                        "NVRM: failed to allocate hugepage table!\n");
+                return rmStatus;
+            }
 
-                rmStatus = nv_alloc_page_array(num_hugepages, order, &user_pages);
-                if (rmStatus != NV_OK)
+            for (i = 0; i < num_hugepages; i++)
+            {
+                ret = NV_PIN_USER_PAGES(start + i * hpage_size,
+                                        1,
+                                        gup_flags,
+                                        &user_pages[i]);
+                if (ret != 1)
                 {
+                    for (j = 0; j < i; j++)
+                        NV_UNPIN_USER_PAGE(user_pages[j]);
+                    nv_free_page_array(user_pages);
                     nv_mmap_read_unlock(mm);
-                    nv_printf(NV_DBG_ERRORS,
-                            "NVRM: failed to allocate hugepage table!\n");
-                    return rmStatus;
-                }
-
-                for (i = 0; i < num_hugepages; i++)
-                {
-                    ret = NV_PIN_USER_PAGES(
-                        (unsigned long)address + i * hpage_size,
-                        1, gup_flags, &user_pages[i]);
-                    if (ret != 1)
-                    {
-                        for (j = 0; j < i; j++)
-                            NV_UNPIN_USER_PAGE(user_pages[j]);
-                        nv_free_page_array(user_pages);
-                        nv_mmap_read_unlock(mm);
-                        return NV_ERR_INVALID_ADDRESS;
-                    }
+                    return NV_ERR_INVALID_ADDRESS;
                 }
+            }
 
-                nv_mmap_read_unlock(mm);
+            nv_mmap_read_unlock(mm);
 
-                *page_array = user_pages;
-                return NV_OK;
-            }
+            *page_array = user_pages;
+            return NV_OK;
         }
     }
 

src/nvidia/arch/nvalloc/unix/include/nv-reg.h

@@ -1051,7 +1051,7 @@ NV_DEFINE_REG_ENTRY_GLOBAL(__NV_NVLINK_DISABLE, 0);
 NV_DEFINE_REG_ENTRY_GLOBAL(__NV_ENABLE_PCIE_RELAXED_ORDERING_MODE, 0);
 NV_DEFINE_REG_ENTRY_GLOBAL(__NV_REGISTER_PCI_DRIVER, 1);
 NV_DEFINE_REG_ENTRY_GLOBAL(__NV_REGISTER_PLATFORM_DEVICE_DRIVER, 1);
-NV_DEFINE_REG_ENTRY_GLOBAL(__NV_ENABLE_RESIZABLE_BAR, 0);
+NV_DEFINE_REG_ENTRY_GLOBAL(__NV_ENABLE_RESIZABLE_BAR, 1);
 NV_DEFINE_REG_ENTRY_GLOBAL(__NV_ENABLE_DBG_BREAKPOINT, 0);
 NV_DEFINE_REG_ENTRY_GLOBAL(__NV_TEGRA_GPU_PG_MASK, 0);
 NV_DEFINE_REG_ENTRY_GLOBAL(__NV_ENABLE_NONBLOCKING_OPEN, 1);

src/nvidia/arch/nvalloc/unix/src/osmemdesc.c

@@ -1243,7 +1243,7 @@ osDestroyOsDescriptorPageArray
     // Read before nv_unregister_user_pages frees the nv_alloc_t.
     compoundOrder = nv_get_compound_order(pPrivate);
 
-    if (compoundOrder > 0)
+    if (compoundOrder > 0 || IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc))
         osPageCount = pMemDesc->PageCount;
     else
         osPageCount = NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount);

src/nvidia/src/kernel/gpu/bif/kernel_bif.c

@@ -1127,8 +1127,11 @@ _kbifInitRegistryOverrides
 {
     NvU32 data32;
 
-    // P2P Override: default to both reads+writes enabled so BAR1 P2P works out of the box
-    pKernelBif->p2pOverride = 0x11;
+    // Enable BAR1 P2P reads and writes without overriding platform atomic capabilities.
+    pKernelBif->p2pOverride =
+        DRF_DEF(_REG_STR, _CL_FORCE_P2P, _READ, _ENABLE) |
+        DRF_DEF(_REG_STR, _CL_FORCE_P2P, _WRITE, _ENABLE) |
+        DRF_DEF(_REG_STR, _CL_FORCE_P2P, _ATOMICS, _DEFAULT);
     if (osReadRegistryDword(pGpu, NV_REG_STR_CL_FORCE_P2P, &data32) == NV_OK)
     {
         pKernelBif->p2pOverride = data32;
@@ -2069,4 +2072,3 @@ kbifWaitForConfigAccessAfterReset_IMPL
 
     return NV_ERR_GENERIC;
 }
-

src/nvidia/src/kernel/mem_mgr/io_vaspace.c

@@ -59,15 +59,15 @@ iovaspaceConstruct__IMPL
 void
 iovaspaceDestruct_IMPL(OBJIOVASPACE *pIOVAS)
 {
-    // OBJVASPACE *pVAS = staticCast(pIOVAS, OBJVASPACE);
-
-    // TODO: might keep p2p mappings...
-    // if (pIOVAS->mappingCount != 0)
-    // {
-    //     NV_PRINTF(LEVEL_ERROR, "%lld left-over mappings in IOVAS 0x%x\n",
-    //               pIOVAS->mappingCount, pVAS->vaspaceId);
-    //     DBG_BREAKPOINT();
-    // }
+    OBJVASPACE *pVAS = staticCast(pIOVAS, OBJVASPACE);
+
+    if (pIOVAS->mappingCount != 0)
+    {
+        NV_PRINTF(LEVEL_WARNING,
+                  "%" NvU64_fmtu " left-over mappings in IOVAS 0x%x\n",
+                  pIOVAS->mappingCount,
+                  pVAS->vaspaceId);
+    }
 }
 
 NV_STATUS
@@ -600,20 +600,24 @@ OBJIOVASPACE *iovaspaceFromMapping(PIOVAMAPPING pIovaMapping)
     OBJIOVASPACE *pIOVAS = iovaspaceFromId(pIovaMapping->iovaspaceId);
 
     //
-    // The IOVASPACE has to be there as the mapping is referencing it. If it's
-    // not, the mapping has been left dangling outlasting the IOVAS it was
-    // under.
+    // A missing IOVAS means the mapping was left dangling and outlived the
+    // address space it belonged to. The destroy path reports that condition.
     //
-    // NV_ASSERT(pIOVAS != NULL);
-
     return pIOVAS;
 }
 
 void iovaMappingDestroy(PIOVAMAPPING pIovaMapping)
 {
     OBJIOVASPACE *pIOVAS = iovaspaceFromMapping(pIovaMapping);
 
-    if (pIOVAS == NULL) return;
+    if (pIOVAS == NULL)
+    {
+        NV_PRINTF(LEVEL_WARNING,
+                  "IOVA mapping outlived IOVAS 0x%x\n",
+                  pIovaMapping->iovaspaceId);
+        return;
+    }
+
     iovaspaceDestroyMapping(pIOVAS, pIovaMapping);
 }
 

src/nvidia/src/kernel/rmapi/nv_gpu_ops.c

@@ -3967,8 +3967,7 @@ nvGpuOpsBuildExternalAllocPtes
     NvBool         isPeerSupported,
     NvBool         isBar1P2PSupported,
     NvU32          peerId,
-    gpuExternalMappingInfo *pGpuExternalMappingInfo,
-    RmPhysAddr     bar1BusAddr
+    gpuExternalMappingInfo *pGpuExternalMappingInfo
 )
 {
     NV_STATUS               status              = NV_OK;
@@ -4125,14 +4124,7 @@ nvGpuOpsBuildExternalAllocPtes
         NvU32 ptePcfHw  = 0;
 
          nvFieldSetBool(&pPteFmt->fldValid, NV_TRUE, pte.v8);
-         if ((aperture == GMMU_APERTURE_PEER) && isBar1P2PSupported)
-         {
-             gmmuFieldSetAperture(&pPteFmt->fldAperture, GMMU_APERTURE_SYS_COH, pte.v8);
-         }
-         else
-         {
-             gmmuFieldSetAperture(&pPteFmt->fldAperture, aperture, pte.v8);
-         }
+         gmmuFieldSetAperture(&pPteFmt->fldAperture, aperture, pte.v8);
          nvFieldSet32(&pPteFmt->fldKind, kind, pte.v8);
 
          ptePcfSw |= vol         ? (1 << SW_MMU_PCF_UNCACHED_IDX) : 0;
@@ -4176,14 +4168,7 @@ nvGpuOpsBuildExternalAllocPtes
         if (nvFieldIsValid32(&pPteFmt->fldAtomicDisable.desc))
             nvFieldSetBool(&pPteFmt->fldAtomicDisable, !atomic, pte.v8);
 
-        if ((aperture == GMMU_APERTURE_PEER) && isBar1P2PSupported)
-        {
-            gmmuFieldSetAperture(&pPteFmt->fldAperture, GMMU_APERTURE_SYS_NONCOH, pte.v8);
-        }
-        else
-        {
-            gmmuFieldSetAperture(&pPteFmt->fldAperture, aperture, pte.v8);
-        }
+        gmmuFieldSetAperture(&pPteFmt->fldAperture, aperture, pte.v8);
 
         if (!isCompressedKind)
         {
@@ -4194,11 +4179,6 @@ nvGpuOpsBuildExternalAllocPtes
         }
     }
 
-    if ((aperture == GMMU_APERTURE_PEER) && isBar1P2PSupported)
-    {
-        fabricBaseAddress = bar1BusAddr;
-    }
-
     if ((aperture == GMMU_APERTURE_PEER) && !isBar1P2PSupported)
     {
         nvFieldSet32(&pPteFmt->fldPeerIndex, peerId, pte.v8);
@@ -4458,8 +4438,7 @@ nvGpuOpsBuildExternalAllocPhysAddrs
     NvBool      isPeerSupported,
     NvBool      isBar1P2PSupported,
     NvU32       peerId,
-    UvmGpuExternalPhysAddrInfo *pGpuExternalPhysAddrInfo,
-    RmPhysAddr bar1BusAddr
+    UvmGpuExternalPhysAddrInfo *pGpuExternalPhysAddrInfo
 )
 {
     NV_STATUS               status              = NV_OK;
@@ -4537,11 +4516,6 @@ nvGpuOpsBuildExternalAllocPhysAddrs
         return NV_ERR_BUFFER_TOO_SMALL;
 
 
-    if ((aperture == GMMU_APERTURE_PEER) && isBar1P2PSupported)
-    {
-        fabricBaseAddress = bar1BusAddr;
-    }
-
     if ((aperture == GMMU_APERTURE_PEER) && !isBar1P2PSupported)
     {
         //
@@ -4684,7 +4658,6 @@ NV_STATUS nvGpuOpsGetExternalAllocPtesOrPhysAddrs(struct gpuAddressSpace *vaSpac
     Memory *pMemory = NULL;
     PMEMORY_DESCRIPTOR pMemDesc = NULL;
     OBJGPU *pMappingGpu = NULL;
-    RmPhysAddr bar1BusAddr = 0;
     NvU32 peerId = 0;
     NvBool isSliSupported = NV_FALSE;
     NvBool isPeerSupported = NV_FALSE;
@@ -4824,8 +4797,6 @@ NV_STATUS nvGpuOpsGetExternalAllocPtesOrPhysAddrs(struct gpuAddressSpace *vaSpac
                                                        &peerId);
             if (status != NV_OK)
                 goto freeGpaMemdesc;
-
-            bar1BusAddr = gpumgrGetGpuPhysFbAddr(pAdjustedMemDesc->pGpu);
         }
 
         //
@@ -4914,15 +4885,14 @@ NV_STATUS nvGpuOpsGetExternalAllocPtesOrPhysAddrs(struct gpuAddressSpace *vaSpac
                                                 isPeerSupported,
                                                 isBar1P2PSupported,
                                                 peerId,
-                                                pGpuExternalMappingInfo,
-                                                bar1BusAddr);
+                                                pGpuExternalMappingInfo);
     }
 
     if (pGpuExternalPhysAddrInfo != NULL)
     {
         status = nvGpuOpsBuildExternalAllocPhysAddrs(pVAS, vaSpace->device->session, pMappingGpu, pAdjustedMemDesc,
                                                      pMemory, offset, size, isIndirectPeerSupported, isPeerSupported,
-                                                     isBar1P2PSupported, peerId, pGpuExternalPhysAddrInfo, bar1BusAddr);
+                                                     isBar1P2PSupported, peerId, pGpuExternalPhysAddrInfo);
     }
 
 freeGpaMemdesc:
@@ -11076,7 +11046,7 @@ NV_STATUS nvGpuOpsGetChannelResourcePtes(struct gpuAddressSpace *vaSpace,
 
     status = nvGpuOpsBuildExternalAllocPtes(pVAS, vaSpace->device->session, pMappingGpu, pMemDesc, NULL,
                                             offset, size, NV_FALSE, NV_FALSE,
-                                            NV_FALSE, 0, pGpuExternalMappingInfo, 0);
+                                            NV_FALSE, 0, pGpuExternalMappingInfo);
 
     _nvGpuOpsLocksRelease(&acquiredLocks);
     threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);