From 31906f937249a4cd68b0fb7eaef823058edf4a46 Mon Sep 17 00:00:00 2001 From: "Duc P. Tran" <1055172+ducphuc@users.noreply.github.com> Date: Tue, 4 Aug 2026 11:27:50 -0700 Subject: [PATCH 01/21] Remove dead BAR1 P2P address plumbing BAR1 mappings already select SYS_COH or SYS_NONCOH before PTE construction, so the PEER-aperture rewrites and bar1BusAddr base assignment were unreachable. Keep the live BAR1 DMA address encoder unchanged. --- src/nvidia/src/kernel/rmapi/nv_gpu_ops.c | 44 ++++-------------------- 1 file changed, 7 insertions(+), 37 deletions(-) diff --git a/src/nvidia/src/kernel/rmapi/nv_gpu_ops.c b/src/nvidia/src/kernel/rmapi/nv_gpu_ops.c index 3de77d045a..9a005fd148 100644 --- a/src/nvidia/src/kernel/rmapi/nv_gpu_ops.c +++ b/src/nvidia/src/kernel/rmapi/nv_gpu_ops.c @@ -3967,8 +3967,7 @@ nvGpuOpsBuildExternalAllocPtes NvBool isPeerSupported, NvBool isBar1P2PSupported, NvU32 peerId, - gpuExternalMappingInfo *pGpuExternalMappingInfo, - RmPhysAddr bar1BusAddr + gpuExternalMappingInfo *pGpuExternalMappingInfo ) { NV_STATUS status = NV_OK; @@ -4125,14 +4124,7 @@ nvGpuOpsBuildExternalAllocPtes NvU32 ptePcfHw = 0; nvFieldSetBool(&pPteFmt->fldValid, NV_TRUE, pte.v8); - if ((aperture == GMMU_APERTURE_PEER) && isBar1P2PSupported) - { - gmmuFieldSetAperture(&pPteFmt->fldAperture, GMMU_APERTURE_SYS_COH, pte.v8); - } - else - { - gmmuFieldSetAperture(&pPteFmt->fldAperture, aperture, pte.v8); - } + gmmuFieldSetAperture(&pPteFmt->fldAperture, aperture, pte.v8); nvFieldSet32(&pPteFmt->fldKind, kind, pte.v8); ptePcfSw |= vol ? (1 << SW_MMU_PCF_UNCACHED_IDX) : 0; @@ -4176,14 +4168,7 @@ nvGpuOpsBuildExternalAllocPtes if (nvFieldIsValid32(&pPteFmt->fldAtomicDisable.desc)) nvFieldSetBool(&pPteFmt->fldAtomicDisable, !atomic, pte.v8); - if ((aperture == GMMU_APERTURE_PEER) && isBar1P2PSupported) - { - gmmuFieldSetAperture(&pPteFmt->fldAperture, GMMU_APERTURE_SYS_NONCOH, pte.v8); - } - else - { - gmmuFieldSetAperture(&pPteFmt->fldAperture, aperture, pte.v8); - } + gmmuFieldSetAperture(&pPteFmt->fldAperture, aperture, pte.v8); if (!isCompressedKind) { @@ -4194,11 +4179,6 @@ nvGpuOpsBuildExternalAllocPtes } } - if ((aperture == GMMU_APERTURE_PEER) && isBar1P2PSupported) - { - fabricBaseAddress = bar1BusAddr; - } - if ((aperture == GMMU_APERTURE_PEER) && !isBar1P2PSupported) { nvFieldSet32(&pPteFmt->fldPeerIndex, peerId, pte.v8); @@ -4458,8 +4438,7 @@ nvGpuOpsBuildExternalAllocPhysAddrs NvBool isPeerSupported, NvBool isBar1P2PSupported, NvU32 peerId, - UvmGpuExternalPhysAddrInfo *pGpuExternalPhysAddrInfo, - RmPhysAddr bar1BusAddr + UvmGpuExternalPhysAddrInfo *pGpuExternalPhysAddrInfo ) { NV_STATUS status = NV_OK; @@ -4537,11 +4516,6 @@ nvGpuOpsBuildExternalAllocPhysAddrs return NV_ERR_BUFFER_TOO_SMALL; - if ((aperture == GMMU_APERTURE_PEER) && isBar1P2PSupported) - { - fabricBaseAddress = bar1BusAddr; - } - if ((aperture == GMMU_APERTURE_PEER) && !isBar1P2PSupported) { // @@ -4684,7 +4658,6 @@ NV_STATUS nvGpuOpsGetExternalAllocPtesOrPhysAddrs(struct gpuAddressSpace *vaSpac Memory *pMemory = NULL; PMEMORY_DESCRIPTOR pMemDesc = NULL; OBJGPU *pMappingGpu = NULL; - RmPhysAddr bar1BusAddr = 0; NvU32 peerId = 0; NvBool isSliSupported = NV_FALSE; NvBool isPeerSupported = NV_FALSE; @@ -4824,8 +4797,6 @@ NV_STATUS nvGpuOpsGetExternalAllocPtesOrPhysAddrs(struct gpuAddressSpace *vaSpac &peerId); if (status != NV_OK) goto freeGpaMemdesc; - - bar1BusAddr = gpumgrGetGpuPhysFbAddr(pAdjustedMemDesc->pGpu); } // @@ -4914,15 +4885,14 @@ NV_STATUS nvGpuOpsGetExternalAllocPtesOrPhysAddrs(struct gpuAddressSpace *vaSpac isPeerSupported, isBar1P2PSupported, peerId, - pGpuExternalMappingInfo, - bar1BusAddr); + pGpuExternalMappingInfo); } if (pGpuExternalPhysAddrInfo != NULL) { status = nvGpuOpsBuildExternalAllocPhysAddrs(pVAS, vaSpace->device->session, pMappingGpu, pAdjustedMemDesc, pMemory, offset, size, isIndirectPeerSupported, isPeerSupported, - isBar1P2PSupported, peerId, pGpuExternalPhysAddrInfo, bar1BusAddr); + isBar1P2PSupported, peerId, pGpuExternalPhysAddrInfo); } freeGpaMemdesc: @@ -11076,7 +11046,7 @@ NV_STATUS nvGpuOpsGetChannelResourcePtes(struct gpuAddressSpace *vaSpace, status = nvGpuOpsBuildExternalAllocPtes(pVAS, vaSpace->device->session, pMappingGpu, pMemDesc, NULL, offset, size, NV_FALSE, NV_FALSE, - NV_FALSE, 0, pGpuExternalMappingInfo, 0); + NV_FALSE, 0, pGpuExternalMappingInfo); _nvGpuOpsLocksRelease(&acquiredLocks); threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE); From ee4cbbcf7fe43853a8dfb363aea0ffde5512af6a Mon Sep 17 00:00:00 2001 From: "Duc P. Tran" <1055172+ducphuc@users.noreply.github.com> Date: Tue, 4 Aug 2026 11:28:29 -0700 Subject: [PATCH 02/21] Leave P2P atomics at the platform default Keep BAR1 reads and writes enabled, but encode the atomics field as DEFAULT instead of implicitly disabling it with 0x11. Preserve the branch-wide BAR1 protocol policy, including qualifying PCIe-only datacenter GPUs, and retain registry-key precedence. --- src/nvidia/src/kernel/gpu/bif/kernel_bif.c | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/src/nvidia/src/kernel/gpu/bif/kernel_bif.c b/src/nvidia/src/kernel/gpu/bif/kernel_bif.c index 322bbc9730..269b8fea70 100644 --- a/src/nvidia/src/kernel/gpu/bif/kernel_bif.c +++ b/src/nvidia/src/kernel/gpu/bif/kernel_bif.c @@ -1127,8 +1127,11 @@ _kbifInitRegistryOverrides { NvU32 data32; - // P2P Override: default to both reads+writes enabled so BAR1 P2P works out of the box - pKernelBif->p2pOverride = 0x11; + // Enable BAR1 P2P reads and writes without overriding platform atomic capabilities. + pKernelBif->p2pOverride = + DRF_DEF(_REG_STR, _CL_FORCE_P2P, _READ, _ENABLE) | + DRF_DEF(_REG_STR, _CL_FORCE_P2P, _WRITE, _ENABLE) | + DRF_DEF(_REG_STR, _CL_FORCE_P2P, _ATOMICS, _DEFAULT); if (osReadRegistryDword(pGpu, NV_REG_STR_CL_FORCE_P2P, &data32) == NV_OK) { pKernelBif->p2pOverride = data32; @@ -2069,4 +2072,3 @@ kbifWaitForConfigAccessAfterReset_IMPL return NV_ERR_GENERIC; } - From e4020347f6c2ba5300ae06a29f655833783dbfb9 Mon Sep 17 00:00:00 2001 From: "Duc P. Tran" <1055172+ducphuc@users.noreply.github.com> Date: Tue, 4 Aug 2026 11:29:59 -0700 Subject: [PATCH 03/21] Reject unsafe hugetlb fast-path ranges Use the reduced page-array path only for non-empty, overflow-safe ranges that are hugepage aligned, exactly sized in hugepages, and contained in one hugetlb VMA. Fall back to the existing base-page path for every other layout. --- kernel-open/nvidia/os-mlock.c | 113 ++++++++++++++++++++++++---------- 1 file changed, 81 insertions(+), 32 deletions(-) diff --git a/kernel-open/nvidia/os-mlock.c b/kernel-open/nvidia/os-mlock.c index 994c326328..e038aa736f 100644 --- a/kernel-open/nvidia/os-mlock.c +++ b/kernel-open/nvidia/os-mlock.c @@ -264,6 +264,54 @@ static void nv_free_page_array(struct page **pages) os_free_mem((NvU8 *)pages - NV_PAGE_ARRAY_HEADER_SIZE); } +static NvBool nv_hugetlb_fast_path_eligible( + struct vm_area_struct *vma, + unsigned long start, + NvU64 page_count, + unsigned long *hpage_size_out, + unsigned int *compound_order_out +) +{ + NvU64 range_size; + NvU64 pages_per_hugepage; + unsigned long end; + unsigned long hpage_size; + + if (!vma || !is_vm_hugetlb_page(vma) || page_count == 0) + return NV_FALSE; + + if (page_count > ((NvU64)ULONG_MAX / PAGE_SIZE)) + return NV_FALSE; + + range_size = page_count * PAGE_SIZE; + if (range_size > ULONG_MAX || start > (ULONG_MAX - (unsigned long)range_size)) + return NV_FALSE; + + end = start + (unsigned long)range_size; + hpage_size = vma_kernel_pagesize(vma); + + if (hpage_size < PAGE_SIZE || hpage_size > NV_U32_MAX || + (hpage_size & (hpage_size - 1)) != 0) + { + return NV_FALSE; + } + + if ((start & (hpage_size - 1)) != 0 || + (range_size & (hpage_size - 1)) != 0 || + end > vma->vm_end) + { + return NV_FALSE; + } + + pages_per_hugepage = hpage_size / PAGE_SIZE; + if ((pages_per_hugepage & (pages_per_hugepage - 1)) != 0) + return NV_FALSE; + + *hpage_size_out = hpage_size; + *compound_order_out = ilog2(pages_per_hugepage); + return NV_TRUE; +} + NV_STATUS NV_API_CALL os_lock_user_pages( void *address, NvU64 page_count, @@ -303,47 +351,48 @@ NV_STATUS NV_API_CALL os_lock_user_pages( * unfaulted 1GB hugepages (~37ms per fault). */ { - struct vm_area_struct *vma = vma_lookup(mm, (unsigned long)address); - - if (vma && is_vm_hugetlb_page(vma)) + unsigned long start = (unsigned long)address; + struct vm_area_struct *vma = vma_lookup(mm, start); + unsigned long hpage_size; + unsigned int order; + + if (nv_hugetlb_fast_path_eligible(vma, + start, + page_count, + &hpage_size, + &order)) { - unsigned long hpage_size = vma_kernel_pagesize(vma); - unsigned int order = ilog2(hpage_size / PAGE_SIZE); - NvU64 pages_per_hp = 1ULL << order; + NvU64 num_hugepages = page_count >> order; - if ((page_count & (pages_per_hp - 1)) == 0) + rmStatus = nv_alloc_page_array(num_hugepages, order, &user_pages); + if (rmStatus != NV_OK) { - NvU64 num_hugepages = page_count >> order; + nv_mmap_read_unlock(mm); + nv_printf(NV_DBG_ERRORS, + "NVRM: failed to allocate hugepage table!\n"); + return rmStatus; + } - rmStatus = nv_alloc_page_array(num_hugepages, order, &user_pages); - if (rmStatus != NV_OK) + for (i = 0; i < num_hugepages; i++) + { + ret = NV_PIN_USER_PAGES(start + i * hpage_size, + 1, + gup_flags, + &user_pages[i]); + if (ret != 1) { + for (j = 0; j < i; j++) + NV_UNPIN_USER_PAGE(user_pages[j]); + nv_free_page_array(user_pages); nv_mmap_read_unlock(mm); - nv_printf(NV_DBG_ERRORS, - "NVRM: failed to allocate hugepage table!\n"); - return rmStatus; - } - - for (i = 0; i < num_hugepages; i++) - { - ret = NV_PIN_USER_PAGES( - (unsigned long)address + i * hpage_size, - 1, gup_flags, &user_pages[i]); - if (ret != 1) - { - for (j = 0; j < i; j++) - NV_UNPIN_USER_PAGE(user_pages[j]); - nv_free_page_array(user_pages); - nv_mmap_read_unlock(mm); - return NV_ERR_INVALID_ADDRESS; - } + return NV_ERR_INVALID_ADDRESS; } + } - nv_mmap_read_unlock(mm); + nv_mmap_read_unlock(mm); - *page_array = user_pages; - return NV_OK; - } + *page_array = user_pages; + return NV_OK; } } From 11d8868c36d9f0a036ab93bded21add53af258f2 Mon Sep 17 00:00:00 2001 From: "Duc P. Tran" <1055172+ducphuc@users.noreply.github.com> Date: Tue, 4 Aug 2026 11:31:01 -0700 Subject: [PATCH 04/21] Restore IOVA lifetime diagnostics Report leftover mappings and mappings that outlive their I/O VA space without triggering a debug breakpoint. Use the portable NvU64 format for mappingCount and retain null-safe teardown. --- src/nvidia/src/kernel/mem_mgr/io_vaspace.c | 34 ++++++++++++---------- 1 file changed, 19 insertions(+), 15 deletions(-) diff --git a/src/nvidia/src/kernel/mem_mgr/io_vaspace.c b/src/nvidia/src/kernel/mem_mgr/io_vaspace.c index eaa8cecab1..d3256379c8 100644 --- a/src/nvidia/src/kernel/mem_mgr/io_vaspace.c +++ b/src/nvidia/src/kernel/mem_mgr/io_vaspace.c @@ -59,15 +59,15 @@ iovaspaceConstruct__IMPL void iovaspaceDestruct_IMPL(OBJIOVASPACE *pIOVAS) { - // OBJVASPACE *pVAS = staticCast(pIOVAS, OBJVASPACE); - - // TODO: might keep p2p mappings... - // if (pIOVAS->mappingCount != 0) - // { - // NV_PRINTF(LEVEL_ERROR, "%lld left-over mappings in IOVAS 0x%x\n", - // pIOVAS->mappingCount, pVAS->vaspaceId); - // DBG_BREAKPOINT(); - // } + OBJVASPACE *pVAS = staticCast(pIOVAS, OBJVASPACE); + + if (pIOVAS->mappingCount != 0) + { + NV_PRINTF(LEVEL_WARNING, + "%" NvU64_fmtu " left-over mappings in IOVAS 0x%x\n", + pIOVAS->mappingCount, + pVAS->vaspaceId); + } } NV_STATUS @@ -600,12 +600,9 @@ OBJIOVASPACE *iovaspaceFromMapping(PIOVAMAPPING pIovaMapping) OBJIOVASPACE *pIOVAS = iovaspaceFromId(pIovaMapping->iovaspaceId); // - // The IOVASPACE has to be there as the mapping is referencing it. If it's - // not, the mapping has been left dangling outlasting the IOVAS it was - // under. + // A missing IOVAS means the mapping was left dangling and outlived the + // address space it belonged to. The destroy path reports that condition. // - // NV_ASSERT(pIOVAS != NULL); - return pIOVAS; } @@ -613,7 +610,14 @@ void iovaMappingDestroy(PIOVAMAPPING pIovaMapping) { OBJIOVASPACE *pIOVAS = iovaspaceFromMapping(pIovaMapping); - if (pIOVAS == NULL) return; + if (pIOVAS == NULL) + { + NV_PRINTF(LEVEL_WARNING, + "IOVA mapping outlived IOVAS 0x%x\n", + pIovaMapping->iovaspaceId); + return; + } + iovaspaceDestroyMapping(pIOVAS, pIovaMapping); } From 2fb0e5f00ecd4653e0bda22dce92afc19b5fd215 Mon Sep 17 00:00:00 2001 From: "Duc P. Tran" <1055172+ducphuc@users.noreply.github.com> Date: Tue, 4 Aug 2026 11:31:26 -0700 Subject: [PATCH 05/21] Restore dynamic-granularity unregister accounting Dynamic-granularity page arrays already use the OS page size, so pass PageCount directly when unregistering them. The RM-to-OS conversion is only correct for ordinary RM-granularity arrays on 64K-page systems. --- src/nvidia/arch/nvalloc/unix/src/osmemdesc.c | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nvidia/arch/nvalloc/unix/src/osmemdesc.c b/src/nvidia/arch/nvalloc/unix/src/osmemdesc.c index 89ee179c4d..6b7cb20f84 100644 --- a/src/nvidia/arch/nvalloc/unix/src/osmemdesc.c +++ b/src/nvidia/arch/nvalloc/unix/src/osmemdesc.c @@ -1243,7 +1243,7 @@ osDestroyOsDescriptorPageArray // Read before nv_unregister_user_pages frees the nv_alloc_t. compoundOrder = nv_get_compound_order(pPrivate); - if (compoundOrder > 0) + if (compoundOrder > 0 || IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc)) osPageCount = pMemDesc->PageCount; else osPageCount = NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount); From 9fac5a9d0ef4d2765b659f184a93971a79349f7c Mon Sep 17 00:00:00 2001 From: "Duc P. Tran" <1055172+ducphuc@users.noreply.github.com> Date: Tue, 4 Aug 2026 11:31:51 -0700 Subject: [PATCH 06/21] Synchronize the Resizable BAR registry mirror Match the inactive nv-reg.h mirror to the kernel-open definition. Only os-registry.c instantiates the active table, so this is source hygiene and does not change runtime behavior. --- src/nvidia/arch/nvalloc/unix/include/nv-reg.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/nvidia/arch/nvalloc/unix/include/nv-reg.h b/src/nvidia/arch/nvalloc/unix/include/nv-reg.h index 93999d4f3c..decf092752 100644 --- a/src/nvidia/arch/nvalloc/unix/include/nv-reg.h +++ b/src/nvidia/arch/nvalloc/unix/include/nv-reg.h @@ -1051,7 +1051,7 @@ NV_DEFINE_REG_ENTRY_GLOBAL(__NV_NVLINK_DISABLE, 0); NV_DEFINE_REG_ENTRY_GLOBAL(__NV_ENABLE_PCIE_RELAXED_ORDERING_MODE, 0); NV_DEFINE_REG_ENTRY_GLOBAL(__NV_REGISTER_PCI_DRIVER, 1); NV_DEFINE_REG_ENTRY_GLOBAL(__NV_REGISTER_PLATFORM_DEVICE_DRIVER, 1); -NV_DEFINE_REG_ENTRY_GLOBAL(__NV_ENABLE_RESIZABLE_BAR, 0); +NV_DEFINE_REG_ENTRY_GLOBAL(__NV_ENABLE_RESIZABLE_BAR, 1); NV_DEFINE_REG_ENTRY_GLOBAL(__NV_ENABLE_DBG_BREAKPOINT, 0); NV_DEFINE_REG_ENTRY_GLOBAL(__NV_TEGRA_GPU_PG_MASK, 0); NV_DEFINE_REG_ENTRY_GLOBAL(__NV_ENABLE_NONBLOCKING_OPEN, 1); From cc19db2105d7ba773410f88acc73c28ab502169e Mon Sep 17 00:00:00 2001 From: "Duc P. Tran" <1055172+ducphuc@users.noreply.github.com> Date: Tue, 4 Aug 2026 11:32:51 -0700 Subject: [PATCH 07/21] Document current P2P safety constraints Keep IOMMU passthrough required while translated-IOMMU scatterlist merging remains unsafe, document the hugetlb eligibility fallback, and replace the broad ACS override recommendation with scoped upstream controls and an isolation warning. --- README.md | 27 +++++++++++++++------------ 1 file changed, 15 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index 2999d9671c..30cd34449b 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # NVIDIA driver 610.43.03 with P2P for RTX 3090, RTX 4090, and RTX 5090 -This enables P2P on consumer GPUs with the 610.43.03 driver version. No kernel parameters -are needed for the default behavior, just build, install, and go. +This enables P2P on consumer GPUs with the 610.43.03 driver version. The current branch +requires the IOMMU passthrough configuration described below. See the [tinygrad 550.54.15-p2p README](https://github.com/tinygrad/open-gpu-kernel-modules/blob/550.54.15-p2p/README.md) for the original description of the approach. @@ -24,9 +24,10 @@ NVLink where it is. For PCIe pairs, transfers write directly to the other GPU's address over DMA. > [!WARNING] -> IOMMU must be in passthrough mode (`iommu=pt`), not translating, or DMA will go through -> IOMMU page tables and transfers will fail. This is very dangerous if you run untrusted -> software or devices. +> IOMMU must currently be in passthrough mode (`iommu=pt`), not translating. In particular, +> the experimental hugetlb registration path does not yet handle scatterlist entries merged +> by a translated IOMMU. Do not use translated mode until that path is fixed and validated. +> Passthrough mode weakens DMA isolation and is unsafe with untrusted software or devices. ## How to use @@ -52,17 +53,19 @@ options nvidia NVreg_RegistryDwords="RMForceP2PType=1" This branch also includes an experimental path that accelerates `cudaHostRegister` by several orders of magnitude when the registered buffer is backed by 1G hugepages, and -shrinks the device page tables used for such mappings. It is enabled automatically. This -path skips some of the per-4K-page bookkeeping the stock driver performs, so it may -misbehave in edge cases the stock driver handles correctly. +shrinks the device page tables used for such mappings. It is enabled automatically for a +non-empty registration that is hugepage-aligned, is an exact multiple of the hugepage +size, and stays within one hugetlb VMA. Other layouts use the normal per-page array path. +The fast path still skips some base-page bookkeeping and remains experimental. ## Potential issues If P2P transfers are slow, make sure your IOMMU is in passthrough (`pt`) mode and that ACS -is disabled. ACS on root ports forces all GPU-to-GPU traffic through the CPU root complex, -killing P2P bandwidth. ACS can be disabled in BIOS, with the -`pcie_acs_override=downstream,multifunction` kernel parameter (if your kernel supports it), -or with an ACS override patch applied to the kernel. +redirect is not forcing GPU-to-GPU traffic through the root complex. Prefer a firmware ACS +control. If the kernel supports the upstream per-device option, use a narrowly scoped +`pci=disable_acs_redir=[;...]` setting and verify the resulting IOMMU groups. +Disabling ACS redirect weakens device isolation; do not use the broad `pcie_acs_override` +patch or kernel parameter. ## Sample `p2pBandwidthLatencyTest` output From 9c35a0e8471cecd692ff8833431869534245d682 Mon Sep 17 00:00:00 2001 From: "Duc P. Tran" <1055172+ducphuc@users.noreply.github.com> Date: Thu, 9 Jul 2026 09:08:16 -0700 Subject: [PATCH 08/21] Make PCIe mailbox P2P setup transactional Mailbox P2P setup used to assert-and-continue, so a failure while programming the HSHUB peer masks or setting up the mailboxes left half-programmed peer state behind. Convert the setup path to return status and unwind on failure: tear down partially set up mailboxes, invalidate peer masks programmed for a fresh mapping, and restore the peer bookkeeping. When taking an additional reference on an existing mapping, a failure only drops the newly taken references and leaves the mapping intact. --- .../gpu/bus/arch/maxwell/kern_bus_gm200.c | 415 ++++++++++++++---- 1 file changed, 336 insertions(+), 79 deletions(-) diff --git a/src/nvidia/src/kernel/gpu/bus/arch/maxwell/kern_bus_gm200.c b/src/nvidia/src/kernel/gpu/bus/arch/maxwell/kern_bus_gm200.c index 1d56500465..6711be5e07 100644 --- a/src/nvidia/src/kernel/gpu/bus/arch/maxwell/kern_bus_gm200.c +++ b/src/nvidia/src/kernel/gpu/bus/arch/maxwell/kern_bus_gm200.c @@ -37,6 +37,59 @@ ((PCIE_P2P_WRITE_MAILBOX_SIZE << DRF_SIZE(NV_P2P_WMBOX_ADDR_ADDR)) - \ PCIE_P2P_WRITE_MAILBOX_SIZE) +static NV_STATUS +_kbusSetupMailboxes_GM200 +( + OBJGPU *pGpu0, + KernelBus *pKernelBus0, + OBJGPU *pGpu1, + KernelBus *pKernelBus1, + NvU32 local2Remote, + NvU32 remote2Local, + NvBool *pbLocalMailboxTeardownAttempted, + NvBool *pbRemoteMailboxTeardownAttempted +); + +static NV_STATUS +_kbusProgramPciePeerMask_GM200 +( + OBJGPU *pGpu, + NvU32 peerMask +) +{ + RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu); + NV2080_CTRL_INTERNAL_HSHUB_PEER_CONN_CONFIG_PARAMS params = {0}; + + params.programPciePeerMask = peerMask; + + return pRmApi->Control(pRmApi, + pGpu->hInternalClient, + pGpu->hInternalSubdevice, + NV2080_CTRL_CMD_INTERNAL_HSHUB_PEER_CONN_CONFIG, + ¶ms, + sizeof(params)); +} + +static NV_STATUS +_kbusInvalidatePeerMask_GM200 +( + OBJGPU *pGpu, + NvU32 peerMask +) +{ + RM_API *pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu); + NV2080_CTRL_INTERNAL_HSHUB_PEER_CONN_CONFIG_PARAMS params = {0}; + + params.invalidatePeerMask = peerMask; + + return pRmApi->Control(pRmApi, + pGpu->hInternalClient, + pGpu->hInternalSubdevice, + NV2080_CTRL_CMD_INTERNAL_HSHUB_PEER_CONN_CONFIG, + ¶ms, + sizeof(params)); +} + /*! * @brief Setup the mailboxes of 2 GPUs so that the local GPU can access remote GPU. * @@ -59,6 +112,27 @@ kbusSetupMailboxes_GM200 NvU32 local2Remote, NvU32 remote2Local ) +{ + NV_STATUS status = _kbusSetupMailboxes_GM200(pGpu0, pKernelBus0, + pGpu1, pKernelBus1, + local2Remote, remote2Local, + NULL, NULL); + + NV_ASSERT_OK(status); +} + +static NV_STATUS +_kbusSetupMailboxes_GM200 +( + OBJGPU *pGpu0, + KernelBus *pKernelBus0, + OBJGPU *pGpu1, + KernelBus *pKernelBus1, + NvU32 local2Remote, + NvU32 remote2Local, + NvBool *pbLocalMailboxTeardownAttempted, + NvBool *pbRemoteMailboxTeardownAttempted +) { PMEMORY_DESCRIPTOR *ppMemDesc = NULL; RmPhysAddr localP2PDomainRemoteAddr; @@ -72,35 +146,65 @@ kbusSetupMailboxes_GM200 NV2080_CTRL_CMD_INTERNAL_BUS_SETUP_P2P_MAILBOX_LOCAL_PARAMS params0 = {0}; NV2080_CTRL_CMD_INTERNAL_BUS_SETUP_P2P_MAILBOX_REMOTE_PARAMS params1 = {0}; NV_STATUS status; + NvBool bRemoteWMBoxMapped = NV_FALSE; + NvBool bLocalP2PDomainMapped = NV_FALSE; + NvBool bRemoteP2PDomainMapped = NV_FALSE; + NvBool bLocalMailboxControl = NV_FALSE; + NvBool bRemoteMailboxControl = NV_FALSE; + NvBool bMailboxTagWritten = NV_FALSE; + + if (pbLocalMailboxTeardownAttempted != NULL) + { + *pbLocalMailboxTeardownAttempted = NV_FALSE; + } + if (pbRemoteMailboxTeardownAttempted != NULL) + { + *pbRemoteMailboxTeardownAttempted = NV_FALSE; + } - NV_ASSERT_OR_RETURN_VOID(local2Remote < P2P_MAX_NUM_PEERS); - NV_ASSERT_OR_RETURN_VOID(remote2Local < P2P_MAX_NUM_PEERS); + NV_ASSERT_OR_RETURN(local2Remote < P2P_MAX_NUM_PEERS, NV_ERR_INVALID_ARGUMENT); + NV_ASSERT_OR_RETURN(remote2Local < P2P_MAX_NUM_PEERS, NV_ERR_INVALID_ARGUMENT); // Ensure we have the correct bidirectional peer mapping - NV_ASSERT_OR_RETURN_VOID(pKernelBus1->p2pPcie.busPeer[remote2Local].remotePeerId == - local2Remote); - NV_ASSERT_OR_RETURN_VOID(pKernelBus0->p2pPcie.busPeer[local2Remote].remotePeerId == - remote2Local); + NV_ASSERT_OR_RETURN(pKernelBus1->p2pPcie.busPeer[remote2Local].remotePeerId == + local2Remote, NV_ERR_INVALID_STATE); + NV_ASSERT_OR_RETURN(pKernelBus0->p2pPcie.busPeer[local2Remote].remotePeerId == + remote2Local, NV_ERR_INVALID_STATE); ppMemDesc = &pKernelBus0->p2pPcie.busPeer[local2Remote].pRemoteWMBoxMemDesc; remoteWMBoxLocalAddr = kbusSetupMailboxAccess_HAL(pGpu1, pKernelBus1, pGpu0, remote2Local, ppMemDesc); - NV_ASSERT_OR_RETURN_VOID(remoteWMBoxLocalAddr != ~0ULL); + if (remoteWMBoxLocalAddr == ~0ULL) + { + status = NV_ERR_INVALID_ADDRESS; + goto kbusSetupMailboxes_cleanup; + } + bRemoteWMBoxMapped = NV_TRUE; ppMemDesc = &pKernelBus1->p2pPcie.busPeer[remote2Local].pRemoteP2PDomMemDesc; localP2PDomainRemoteAddr = kbusSetupP2PDomainAccess_HAL(pGpu0, pKernelBus0, pGpu1, ppMemDesc); - NV_ASSERT_OR_RETURN_VOID(localP2PDomainRemoteAddr != ~0ULL); + if (localP2PDomainRemoteAddr == ~0ULL) + { + status = NV_ERR_INVALID_ADDRESS; + goto kbusSetupMailboxes_cleanup; + } + bLocalP2PDomainMapped = NV_TRUE; ppMemDesc = &pKernelBus0->p2pPcie.busPeer[local2Remote].pRemoteP2PDomMemDesc; remoteP2PDomainLocalAddr = kbusSetupP2PDomainAccess_HAL(pGpu1, pKernelBus1, pGpu0, ppMemDesc); - NV_ASSERT_OR_RETURN_VOID(remoteP2PDomainLocalAddr != ~0ULL); + if (remoteP2PDomainLocalAddr == ~0ULL) + { + status = NV_ERR_INVALID_ADDRESS; + goto kbusSetupMailboxes_cleanup; + } + bRemoteP2PDomainMapped = NV_TRUE; // Setup the local GPU to access remote GPU's FB. @@ -110,7 +214,11 @@ kbusSetupMailboxes_GM200 PCIE_P2P_WRITE_MAILBOX_SIZE * remote2Local; // Write mailbox data window needs to be 64KB aligned. - NV_ASSERT((remoteWMBoxAddrU64 & 0xFFFF) == 0); + if ((remoteWMBoxAddrU64 & 0xFFFF) != 0) + { + status = NV_ERR_INVALID_ADDRESS; + goto kbusSetupMailboxes_cleanup; + } // Setup PCIE P2P Mailbox on local GPU params0.local2Remote = local2Remote; @@ -127,7 +235,11 @@ kbusSetupMailboxes_GM200 NV2080_CTRL_CMD_INTERNAL_BUS_SETUP_P2P_MAILBOX_LOCAL, ¶ms0, sizeof(NV2080_CTRL_CMD_INTERNAL_BUS_SETUP_P2P_MAILBOX_LOCAL_PARAMS)); - NV_ASSERT(status == NV_OK); + if (status != NV_OK) + { + goto kbusSetupMailboxes_cleanup; + } + bLocalMailboxControl = NV_TRUE; // Setup PCIE P2P Mailbox on remote GPU params1.local2Remote = local2Remote; @@ -143,9 +255,62 @@ kbusSetupMailboxes_GM200 NV2080_CTRL_CMD_INTERNAL_BUS_SETUP_P2P_MAILBOX_REMOTE, ¶ms1, sizeof(NV2080_CTRL_CMD_INTERNAL_BUS_SETUP_P2P_MAILBOX_REMOTE_PARAMS)); - NV_ASSERT(status == NV_OK); + if (status != NV_OK) + { + goto kbusSetupMailboxes_cleanup; + } + bRemoteMailboxControl = NV_TRUE; kbusWriteP2PWmbTag_HAL(pGpu1, pKernelBus1, remote2Local, params0.p2pWmbTag); + bMailboxTagWritten = NV_TRUE; + + return NV_OK; + +kbusSetupMailboxes_cleanup: + NV_PRINTF(LEVEL_ERROR, + "P2P_MAILBOX_SETUP_FAIL localGpu=%u remoteGpu=%u localPeer=%u remotePeer=%u " + "status=0x%x wmboxMapped=%u localDomainMapped=%u remoteDomainMapped=%u " + "localCtrl=%u remoteCtrl=%u tagWritten=%u tag=0x%llx\n", + gpuGetInstance(pGpu0), + gpuGetInstance(pGpu1), + local2Remote, + remote2Local, + status, + bRemoteWMBoxMapped, + bLocalP2PDomainMapped, + bRemoteP2PDomainMapped, + bLocalMailboxControl, + bRemoteMailboxControl, + bMailboxTagWritten, + (NvU64)params0.p2pWmbTag); + + if (bLocalMailboxControl) + { + kbusDestroyMailbox(pGpu0, pKernelBus0, pGpu1, local2Remote); + if (pbLocalMailboxTeardownAttempted != NULL) + { + *pbLocalMailboxTeardownAttempted = NV_TRUE; + } + } + else if (bRemoteWMBoxMapped || bRemoteP2PDomainMapped) + { + kbusDestroyPeerAccess_HAL(pGpu0, pKernelBus0, local2Remote); + } + + if (bRemoteMailboxControl || bMailboxTagWritten) + { + kbusDestroyMailbox(pGpu1, pKernelBus1, pGpu0, remote2Local); + if (pbRemoteMailboxTeardownAttempted != NULL) + { + *pbRemoteMailboxTeardownAttempted = NV_TRUE; + } + } + else if (bLocalP2PDomainMapped) + { + kbusDestroyPeerAccess_HAL(pGpu1, pKernelBus1, remote2Local); + } + + return status; } void @@ -198,11 +363,28 @@ kbusSetupMailboxAccess_GM200 PMEMORY_DESCRIPTOR *ppWMBoxMemDesc ) { - return kbusSetupPeerBarAccess(pGpu0, pGpu1, - gpumgrGetGpuPhysFbAddr(pGpu0) + - pKernelBus0->p2pPcie.writeMailboxBar1Addr + - PCIE_P2P_WRITE_MAILBOX_SIZE * local2Remote, - PCIE_P2P_WRITE_MAILBOX_SIZE, ppWMBoxMemDesc); + RmPhysAddr fbBase = gpumgrGetGpuPhysFbAddr(pGpu0); + NvU64 mailboxOffset = pKernelBus0->p2pPcie.writeMailboxBar1Addr; + NvU64 peerOffset = PCIE_P2P_WRITE_MAILBOX_SIZE * local2Remote; + RmPhysAddr base; + + if (pKernelBus0->p2pPcie.writeMailboxBar1Addr == + PCIE_P2P_INVALID_WRITE_MAILBOX_ADDR) + { + NV_PRINTF(LEVEL_ERROR, + "PCIe mailbox P2P requested without an allocated mailbox area " + "ownerGpu=%u accessorGpu=%u peer=%u writeMailboxBar1Addr=0x%llx\n", + gpuGetInstance(pGpu0), + gpuGetInstance(pGpu1), + local2Remote, + pKernelBus0->p2pPcie.writeMailboxBar1Addr); + return ~0ULL; + } + + base = fbBase + mailboxOffset + peerOffset; + + return kbusSetupPeerBarAccess(pGpu0, pGpu1, base, + PCIE_P2P_WRITE_MAILBOX_SIZE, ppWMBoxMemDesc); } void @@ -353,9 +535,20 @@ kbusCreateP2PMappingForMailbox_GM200 NvU32 attributes ) { - RM_API *pRmApi; - NV2080_CTRL_INTERNAL_HSHUB_PEER_CONN_CONFIG_PARAMS params; NvU32 gpuInst0, gpuInst1; + NvBool bPeer0HshubProgrammed = NV_FALSE; + NvBool bPeer1HshubProgrammed = NV_FALSE; + NvBool bPeer0MailboxTeardownNeeded = NV_FALSE; + NvBool bPeer1MailboxTeardownNeeded = NV_FALSE; + NvBool bPeer0MailboxTeardownAttempted = NV_FALSE; + NvBool bPeer1MailboxTeardownAttempted = NV_FALSE; + NvU32 oldPeer0RemotePeerId; + NvU32 oldPeer1RemotePeerId; + NvU32 oldPeer0RefCount; + NvU32 oldPeer1RefCount; + NvU32 oldPeerMask0; + NvU32 oldPeerMask1; + NV_STATUS status; if (IS_VIRTUAL(pGpu0) || IS_VIRTUAL(pGpu1)) { @@ -397,25 +590,26 @@ kbusCreateP2PMappingForMailbox_GM200 NV_ASSERT(pKernelBus0->p2pPcie.busPeer[*peer0].remotePeerId == *peer1); NV_ASSERT(pKernelBus1->p2pPcie.busPeer[*peer1].remotePeerId == *peer0); - pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu0); - portMemSet(¶ms, 0, sizeof(params)); - params.programPciePeerMask = NVBIT32(*peer0); - NV_ASSERT_OK_OR_RETURN(pRmApi->Control(pRmApi, - pGpu0->hInternalClient, - pGpu0->hInternalSubdevice, - NV2080_CTRL_CMD_INTERNAL_HSHUB_PEER_CONN_CONFIG, - ¶ms, - sizeof(params))); - - pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu1); - portMemSet(¶ms, 0, sizeof(params)); - params.programPciePeerMask = NVBIT32(*peer1); - NV_ASSERT_OK_OR_RETURN(pRmApi->Control(pRmApi, - pGpu1->hInternalClient, - pGpu1->hInternalSubdevice, - NV2080_CTRL_CMD_INTERNAL_HSHUB_PEER_CONN_CONFIG, - ¶ms, - sizeof(params))); + status = _kbusProgramPciePeerMask_GM200(pGpu0, NVBIT32(*peer0)); + if (status != NV_OK) + { + pKernelBus0->p2pPcie.busPeer[*peer0].refCount--; + pKernelBus1->p2pPcie.busPeer[*peer1].refCount--; + return status; + } + + status = _kbusProgramPciePeerMask_GM200(pGpu1, NVBIT32(*peer1)); + if (status != NV_OK) + { + // + // The mapping pre-exists and its HSHUB peer masks are + // still needed by the existing references, so only drop + // the references taken above. + // + pKernelBus0->p2pPcie.busPeer[*peer0].refCount--; + pKernelBus1->p2pPcie.busPeer[*peer1].refCount--; + return status; + } return NV_OK; } @@ -449,25 +643,26 @@ kbusCreateP2PMappingForMailbox_GM200 NV_ASSERT(!pKernelBus0->p2pPcie.busPeer[*peer0].bReserved); NV_ASSERT(!pKernelBus1->p2pPcie.busPeer[*peer1].bReserved); - pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu0); - portMemSet(¶ms, 0, sizeof(params)); - params.programPciePeerMask = NVBIT32(*peer0); - NV_ASSERT_OK_OR_RETURN(pRmApi->Control(pRmApi, - pGpu0->hInternalClient, - pGpu0->hInternalSubdevice, - NV2080_CTRL_CMD_INTERNAL_HSHUB_PEER_CONN_CONFIG, - ¶ms, - sizeof(params))); - - pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu1); - portMemSet(¶ms, 0, sizeof(params)); - params.programPciePeerMask = NVBIT32(*peer1); - NV_ASSERT_OK_OR_RETURN(pRmApi->Control(pRmApi, - pGpu1->hInternalClient, - pGpu1->hInternalSubdevice, - NV2080_CTRL_CMD_INTERNAL_HSHUB_PEER_CONN_CONFIG, - ¶ms, - sizeof(params))); + status = _kbusProgramPciePeerMask_GM200(pGpu0, NVBIT32(*peer0)); + if (status != NV_OK) + { + pKernelBus0->p2pPcie.busPeer[*peer0].refCount--; + pKernelBus1->p2pPcie.busPeer[*peer1].refCount--; + return status; + } + + status = _kbusProgramPciePeerMask_GM200(pGpu1, NVBIT32(*peer1)); + if (status != NV_OK) + { + // + // The mapping pre-exists and its HSHUB peer masks are still + // needed by the existing references, so only drop the + // references taken above. + // + pKernelBus0->p2pPcie.busPeer[*peer0].refCount--; + pKernelBus1->p2pPcie.busPeer[*peer1].refCount--; + return status; + } return NV_OK; } @@ -521,6 +716,13 @@ kbusCreateP2PMappingForMailbox_GM200 } busCreateP2PMapping_setupMapping: + oldPeer0RemotePeerId = pKernelBus0->p2pPcie.busPeer[*peer0].remotePeerId; + oldPeer1RemotePeerId = pKernelBus1->p2pPcie.busPeer[*peer1].remotePeerId; + oldPeer0RefCount = pKernelBus0->p2pPcie.busPeer[*peer0].refCount; + oldPeer1RefCount = pKernelBus1->p2pPcie.busPeer[*peer1].refCount; + oldPeerMask0 = pKernelBus0->p2pPcie.peerNumberMask[gpuInst1]; + oldPeerMask1 = pKernelBus1->p2pPcie.peerNumberMask[gpuInst0]; + pKernelBus0->p2pPcie.busPeer[*peer0].remotePeerId = *peer1; pKernelBus0->p2pPcie.peerNumberMask[gpuInst1] |= NVBIT(*peer0); pKernelBus1->p2pPcie.busPeer[*peer1].remotePeerId = *peer0; @@ -538,34 +740,80 @@ kbusCreateP2PMappingForMailbox_GM200 pKernelBus0->p2pPcie.busPeer[*peer0].refCount++; pKernelBus1->p2pPcie.busPeer[*peer1].refCount++; - pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu0); - portMemSet(¶ms, 0, sizeof(params)); - params.programPciePeerMask = NVBIT32(*peer0); - NV_ASSERT_OK_OR_RETURN(pRmApi->Control(pRmApi, - pGpu0->hInternalClient, - pGpu0->hInternalSubdevice, - NV2080_CTRL_CMD_INTERNAL_HSHUB_PEER_CONN_CONFIG, - ¶ms, - sizeof(params))); - - pRmApi = GPU_GET_PHYSICAL_RMAPI(pGpu1); - portMemSet(¶ms, 0, sizeof(params)); - params.programPciePeerMask = NVBIT32(*peer1); - NV_ASSERT_OK_OR_RETURN(pRmApi->Control(pRmApi, - pGpu1->hInternalClient, - pGpu1->hInternalSubdevice, - NV2080_CTRL_CMD_INTERNAL_HSHUB_PEER_CONN_CONFIG, - ¶ms, - sizeof(params))); + status = _kbusProgramPciePeerMask_GM200(pGpu0, NVBIT32(*peer0)); + if (status != NV_OK) + { + goto busCreateP2PMapping_rollback; + } + bPeer0HshubProgrammed = NV_TRUE; + + status = _kbusProgramPciePeerMask_GM200(pGpu1, NVBIT32(*peer1)); + if (status != NV_OK) + { + goto busCreateP2PMapping_rollback; + } + bPeer1HshubProgrammed = NV_TRUE; + + status = _kbusSetupMailboxes_GM200(pGpu0, pKernelBus0, pGpu1, pKernelBus1, + *peer0, *peer1, + &bPeer0MailboxTeardownAttempted, + &bPeer1MailboxTeardownAttempted); + if (status != NV_OK) + { + goto busCreateP2PMapping_rollback; + } + bPeer0MailboxTeardownNeeded = NV_TRUE; + bPeer1MailboxTeardownNeeded = NV_TRUE; + + status = _kbusSetupMailboxes_GM200(pGpu1, pKernelBus1, pGpu0, pKernelBus0, + *peer1, *peer0, + &bPeer1MailboxTeardownAttempted, + &bPeer0MailboxTeardownAttempted); + if (status != NV_OK) + { + goto busCreateP2PMapping_rollback; + } NV_PRINTF(LEVEL_INFO, "added PCIe P2P mapping between GPU%u (peer %u) and GPU%u (peer %u)\n", gpuInst0, *peer0, gpuInst1, *peer1); - kbusSetupMailboxes_HAL(pGpu0, pKernelBus0, pGpu1, pKernelBus1, *peer0, *peer1); - kbusSetupMailboxes_HAL(pGpu1, pKernelBus1, pGpu0, pKernelBus0, *peer1, *peer0); - return NV_OK; + +busCreateP2PMapping_rollback: + if (bPeer0MailboxTeardownNeeded && !bPeer0MailboxTeardownAttempted) + { + kbusDestroyMailbox(pGpu0, pKernelBus0, pGpu1, *peer0); + bPeer0MailboxTeardownAttempted = NV_TRUE; + } + + if (bPeer1MailboxTeardownNeeded && !bPeer1MailboxTeardownAttempted) + { + kbusDestroyMailbox(pGpu1, pKernelBus1, pGpu0, *peer1); + bPeer1MailboxTeardownAttempted = NV_TRUE; + } + + if (bPeer0HshubProgrammed && !bPeer0MailboxTeardownAttempted) + { + NV_ASSERT_OK(_kbusInvalidatePeerMask_GM200(pGpu0, NVBIT32(*peer0))); + } + + if (bPeer1HshubProgrammed && !bPeer1MailboxTeardownAttempted) + { + NV_ASSERT_OK(_kbusInvalidatePeerMask_GM200(pGpu1, NVBIT32(*peer1))); + } + + pKernelBus0->p2pPcie.busPeer[*peer0].remotePeerId = oldPeer0RemotePeerId; + pKernelBus1->p2pPcie.busPeer[*peer1].remotePeerId = oldPeer1RemotePeerId; + pKernelBus0->p2pPcie.busPeer[*peer0].refCount = oldPeer0RefCount; + pKernelBus1->p2pPcie.busPeer[*peer1].refCount = oldPeer1RefCount; + pKernelBus0->p2pPcie.peerNumberMask[gpuInst1] = oldPeerMask0; + pKernelBus1->p2pPcie.peerNumberMask[gpuInst0] = oldPeerMask1; + + *peer0 = BUS_INVALID_PEER; + *peer1 = BUS_INVALID_PEER; + + return status; } /*! @@ -797,6 +1045,15 @@ kbusSetP2PMailboxBar1Area_GM200 if (!kbusIsP2pMailboxClientAllocated(pKernelBus)) { + if (pKernelBus->p2pPcie.writeMailboxBar1Addr == + PCIE_P2P_INVALID_WRITE_MAILBOX_ADDR) + { + NV_PRINTF(LEVEL_ERROR, + "P2P mailbox area expected from RM but no valid address is installed gpu=%u\n", + gpuGetInstance(pGpu)); + return NV_ERR_INVALID_STATE; + } + // P2P mailbox area already allocated by RM. Nothing to do. return NV_OK; } From d682adc57bb69054fd0abc4346830e8cf6713faf Mon Sep 17 00:00:00 2001 From: "Duc P. Tran" <1055172+ducphuc@users.noreply.github.com> Date: Tue, 4 Aug 2026 18:08:40 -0700 Subject: [PATCH 09/21] Enable display-aware BAR1 P2P using runtime coverage Allow default-enabled GPUs to place static BAR1 above fixed console and mailbox mappings when the remaining aperture covers all aligned client framebuffer memory. Preserve the tested GB206 partial-window exception without generalizing partial coverage to other implementations. Validate external BAR1 mapping addresses against the DMA window so spanning and outside allocations fail safely. Document the current CUDA API rejection behavior and add a source-level policy truth-table test. --- README.md | 7 + .../gpu/bus/arch/turing/bar1_p2p_policy.h | 17 +++ .../gpu/bus/arch/turing/kern_bus_tu102.c | 133 +++++++++++++----- src/nvidia/src/kernel/rmapi/nv_gpu_ops.c | 42 +++++- tests/bar1_p2p_policy_test.c | 36 +++++ 5 files changed, 194 insertions(+), 41 deletions(-) create mode 100644 src/nvidia/src/kernel/gpu/bus/arch/turing/bar1_p2p_policy.h create mode 100644 tests/bar1_p2p_policy_test.c diff --git a/README.md b/README.md index 30cd34449b..090286ec15 100644 --- a/README.md +++ b/README.md @@ -13,6 +13,7 @@ for the original description of the approach. | RTX 3090 | Pairwise NVLink where available, PCIe BAR1 otherwise | | RTX 4090 | PCIe BAR1 | | RTX 5090 | PCIe BAR1 | +| RTX 5060 Ti / 5060 (GB206) | PCIe BAR1, including with a display attached | P2P also works between different devices of the same generation, for example RTX 5090 to RTX PRO 6000 Blackwell. @@ -23,6 +24,12 @@ This enables BAR1 P2P on consumer GPUs where NVLink isn't available, and falls b NVLink where it is. For PCIe pairs, transfers write directly to the other GPU's physical address over DMA. +On default-enabled GPUs, display-aware static BAR1 placement is used when runtime geometry +covers all aligned client framebuffer memory. GB206 cards (RTX 5060 Ti / 5060) retain an +experimental partial-window exception so inside-range P2P works with a display attached. +Allocations spanning or outside that partial static window are rejected by the CUDA API; +there is currently no transparent dynamic-mapping fallback for those allocations. + > [!WARNING] > IOMMU must currently be in passthrough mode (`iommu=pt`), not translating. In particular, > the experimental hugetlb registration path does not yet handle scatterlist entries merged diff --git a/src/nvidia/src/kernel/gpu/bus/arch/turing/bar1_p2p_policy.h b/src/nvidia/src/kernel/gpu/bus/arch/turing/bar1_p2p_policy.h new file mode 100644 index 0000000000..e6b7975a1e --- /dev/null +++ b/src/nvidia/src/kernel/gpu/bus/arch/turing/bar1_p2p_policy.h @@ -0,0 +1,17 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 Duc P. Tran + * SPDX-License-Identifier: MIT + */ + +#ifndef KERN_BUS_BAR1_P2P_POLICY_H +#define KERN_BUS_BAR1_P2P_POLICY_H + +/* + * Display-aware placement is additive for default-enabled devices whose + * runtime BAR1 geometry covers all aligned client FB. GB206 keeps its tested + * partial-window exception; other partial windows are not generalized. + */ +#define KBUS_USE_DISPLAY_AWARE_STATIC_BAR1(defaultEnabled, fullCoverage, gb206Exception) \ + ((defaultEnabled) && ((fullCoverage) || (gb206Exception))) + +#endif // KERN_BUS_BAR1_P2P_POLICY_H diff --git a/src/nvidia/src/kernel/gpu/bus/arch/turing/kern_bus_tu102.c b/src/nvidia/src/kernel/gpu/bus/arch/turing/kern_bus_tu102.c index f06fb0953f..3819e59bf1 100644 --- a/src/nvidia/src/kernel/gpu/bus/arch/turing/kern_bus_tu102.c +++ b/src/nvidia/src/kernel/gpu/bus/arch/turing/kern_bus_tu102.c @@ -31,6 +31,7 @@ #include "gpu/mem_mgr/virt_mem_allocator.h" #include "nvrm_registry.h" #include "kernel/virtualization/hypervisor/hypervisor.h" +#include "bar1_p2p_policy.h" #include "published/turing/tu102/dev_bus.h" #include "published/turing/tu102/dev_vm.h" @@ -385,8 +386,35 @@ kbusIsStaticBar1Supported_TU102 // NvU64 fbSize = pMemoryManager->Ram.fbAddrSpaceSizeMb << 20; NvU64 fbSizeAligned = RM_ALIGN_UP(fbSize, RM_PAGE_SIZE_2M); + NvU64 clientFbSize = memmgrGetClientFbAddrSpaceSize(pGpu, pMemoryManager); + NvU64 clientFbSizeAligned = RM_ALIGN_DOWN(clientFbSize, RM_PAGE_SIZE_2M); NvU64 bar1VASize = pKernelBus->bar1[gfid].mappableLength; NvU64 bar1VASizeAligned = RM_ALIGN_DOWN(bar1VASize, RM_PAGE_SIZE_2M); + NvU64 staticBar1Offset = NV_ALIGN_UP(consoleSize + mailboxSize, RM_PAGE_SIZE_512M); + NvBool bBar1P2PDefault = + pKernelBus->getProperty(pKernelBus, PDB_PROP_KBUS_SUPPORT_BAR1_P2P_BY_DEFAULT); + NvU64 maxStaticMapSize = + (bar1VASizeAligned > staticBar1Offset) ? + RM_ALIGN_DOWN(bar1VASizeAligned - staticBar1Offset, RM_PAGE_SIZE_2M) : 0; + NvBool bCompleteStaticCoverage = + ((clientFbSizeAligned != 0) && (maxStaticMapSize >= clientFbSizeAligned)); + NvBool bGb206CoverageException = + ((gpuGetChipImpl(pGpu) == GPU_IMPLEMENTATION_GB206) && + (maxStaticMapSize != 0)); + NvBool bUseDisplayAwareStaticBar1 = + KBUS_USE_DISPLAY_AWARE_STATIC_BAR1(bBar1P2PDefault, + bCompleteStaticCoverage, + bGb206CoverageException); + // + // Default-enabled GPUs may place a complete static mapping after fixed + // console/mailbox mappings when runtime geometry covers all aligned client + // FB. Preserve the tested GB206 partial-window behavior, but do not extend + // partial coverage to other implementations. + // + NvU64 autoStaticMapSize = bUseDisplayAwareStaticBar1 ? + ((clientFbSizeAligned < maxStaticMapSize) ? + clientFbSizeAligned : maxStaticMapSize) : + fbSizeAligned; if (gfid != 0) { @@ -427,14 +455,13 @@ kbusIsStaticBar1Supported_TU102 // really wants to enable static BAR1 regardless of the auto checks // NvU64 bar1MapSize = - RM_ALIGN_DOWN(memmgrGetClientFbAddrSpaceSize(pGpu, pMemoryManager), - RM_PAGE_SIZE_2M); + clientFbSizeAligned; - if (bar1VASizeAligned < bar1MapSize) + if (bar1VASizeAligned < (staticBar1Offset + bar1MapSize)) { NV_PRINTF(LEVEL_ERROR, "BAR1 size %lld is not large enough to map FB size" - "%lld to force static BAR1\n", - bar1VASizeAligned, bar1MapSize); + "%lld at offset %lld to force static BAR1\n", + bar1VASizeAligned, bar1MapSize, staticBar1Offset); DBG_BREAKPOINT(); return NV_ERR_INVALID_REGISTRY_KEY; @@ -469,19 +496,17 @@ kbusIsStaticBar1Supported_TU102 // NvU32 userdSize = 0; NvU32 numChannels = kfifoGetMaxChannelsInSystem(pGpu, pKernelFifo); - NvU64 requiredAutoBar1Size = fbSizeAligned; + NvU64 requiredAutoBar1Size = autoStaticMapSize; NvU64 mmioPrivSize = 16 * RM_PAGE_SIZE; NvU64 doorbellSize = 16 * RM_PAGE_SIZE; + NvU64 alignmentPadding = staticBar1Offset - (consoleSize + mailboxSize); + NvU64 dynamicBar1Size; kfifoGetUserdSizeAlign_HAL(pKernelFifo, &userdSize, NULL); userdSize *= numChannels; - requiredAutoBar1Size += userdSize; - requiredAutoBar1Size += mmioPrivSize; - requiredAutoBar1Size += doorbellSize; - requiredAutoBar1Size += consoleSize; - requiredAutoBar1Size += mailboxSize; + dynamicBar1Size = userdSize + mmioPrivSize + doorbellSize; // // Console mappings are already mapped from the bottom of the BAR1 VASpace, @@ -493,10 +518,30 @@ kbusIsStaticBar1Supported_TU102 // if ((consoleSize != 0) || (mailboxSize != 0)) { - requiredAutoBar1Size += RM_PAGE_SIZE_512M - ((consoleSize + mailboxSize) % RM_PAGE_SIZE_512M); + if (bUseDisplayAwareStaticBar1) + { + requiredAutoBar1Size += staticBar1Offset; + + if (dynamicBar1Size > alignmentPadding) + { + requiredAutoBar1Size += dynamicBar1Size - alignmentPadding; + } + } + else + { + requiredAutoBar1Size += dynamicBar1Size; + requiredAutoBar1Size += consoleSize; + requiredAutoBar1Size += mailboxSize; + requiredAutoBar1Size += alignmentPadding; + } + } + else + { + requiredAutoBar1Size += dynamicBar1Size; } - if (bar1VASizeAligned >= requiredAutoBar1Size) + if ((autoStaticMapSize != 0) && + (bar1VASizeAligned >= requiredAutoBar1Size)) { NV_PRINTF(LEVEL_INFO, "Enabling static BAR1 automatically!\n"); return NV_OK; @@ -535,8 +580,12 @@ kbusEnableStaticBar1Mapping_TU102 MEMORY_DESCRIPTOR *pDmaMemDesc = NULL; NV_STATUS status = NV_OK; NvU64 bar1MapSize; + NvU64 clientFbSizeAligned; + NvU64 bar1VASizeAligned; NvU64 bar1BusAddr; NvU32 mapFlags = BUS_MAP_FB_FLAGS_MAP_UNICAST | BUS_MAP_FB_FLAGS_MAP_OFFSET_FIXED; + NvBool bBar1P2PDefault = + pKernelBus->getProperty(pKernelBus, PDB_PROP_KBUS_SUPPORT_BAR1_P2P_BY_DEFAULT); // // But use memmgrGetClientFbAddrSpaceSize @@ -548,8 +597,33 @@ kbusEnableStaticBar1Mapping_TU102 // The last client FB addresses not aligned to 2MB will // not be mappable to a 2MB mapping. // - bar1MapSize = RM_ALIGN_DOWN(memmgrGetClientFbAddrSpaceSize(pGpu, pMemoryManager), - RM_PAGE_SIZE_2M); + clientFbSizeAligned = RM_ALIGN_DOWN(memmgrGetClientFbAddrSpaceSize(pGpu, pMemoryManager), + RM_PAGE_SIZE_2M); + bar1VASizeAligned = RM_ALIGN_DOWN(pKernelBus->bar1[gfid].mappableLength, + RM_PAGE_SIZE_2M); + bar1MapSize = clientFbSizeAligned; + + { + NvU64 maxStaticMapSize = + (bar1Offset < bar1VASizeAligned) ? + RM_ALIGN_DOWN(bar1VASizeAligned - bar1Offset, RM_PAGE_SIZE_2M) : 0; + NvBool bCompleteStaticCoverage = + ((clientFbSizeAligned != 0) && (maxStaticMapSize >= clientFbSizeAligned)); + NvBool bGb206CoverageException = + ((gpuGetChipImpl(pGpu) == GPU_IMPLEMENTATION_GB206) && + (maxStaticMapSize != 0)); + NvBool bUseDisplayAwareStaticBar1 = + KBUS_USE_DISPLAY_AWARE_STATIC_BAR1(bBar1P2PDefault, + bCompleteStaticCoverage, + bGb206CoverageException); + + if (bUseDisplayAwareStaticBar1 && (bar1MapSize > maxStaticMapSize)) + { + bar1MapSize = maxStaticMapSize; + } + } + + NV_ASSERT_OR_RETURN(bar1MapSize != 0, NV_ERR_NOT_SUPPORTED); // // The static mapping is not backed by an allocated physical FB. @@ -574,7 +648,7 @@ kbusEnableStaticBar1Mapping_TU102 // Deploy the static mapping. The RUSD statistics will read incorrectly // until the subsequent call to kbusUpdateRusdStatistics at the end of // kbusStatePostLoad_GM107 with bStaticBar1Enabled set - // + // NV_ASSERT_OK_OR_GOTO(status, kbusMapFbApertureSingle(pGpu, pKernelBus, pMemDesc, 0, &bar1Offset, &bar1MapSize, @@ -1043,7 +1117,6 @@ kbusGetStaticFbAperture_TU102 NvBool bDiscontigAllowed = !!(busMapFlags & BUS_MAP_FB_FLAGS_ALLOW_DISCONTIG); NvBool bInStaticRegion = NV_FALSE; NvBool bInDynamicRegion = NV_FALSE; - NvBool bInLastPage = NV_TRUE; NV_CHECK_OR_RETURN(LEVEL_SILENT, kbusIsStaticBar1Enabled(pGpu, pKernelBus), NV_ERR_NOT_SUPPORTED); @@ -1076,7 +1149,6 @@ kbusGetStaticFbAperture_TU102 if (curLimit > staticBar1Size) { bInDynamicRegion = NV_TRUE; - bInLastPage = bInLastPage && ((curLimit - staticBar1Size) < RM_PAGE_SIZE_2M); } else { @@ -1090,25 +1162,14 @@ kbusGetStaticFbAperture_TU102 if (bInDynamicRegion && bInStaticRegion) { // - // With rounding down the static region to 2MB, - // we can allocate the last non-2MB aligned region - // but not have a mapping for it + // The static region may not cover all of client FB: it is rounded + // down to 2MB, and on display-attached GB206 it is clipped to the + // BAR1 VA left after the console/mailbox reservation. The static BAR1 + // path cannot represent a range spanning that boundary; current CUDA + // P2P callers receive a predictable API rejection rather than a + // transparent dynamic-mapping fallback. // - if (bInLastPage) - { - return NV_ERR_NOT_SUPPORTED; - } - - NV_PRINTF(LEVEL_ERROR, "MemDesc spans both static and dynamic region," - "which is unsupported.\n"); - NV_PRINTF(LEVEL_ERROR, "static Bar1 map [0, 0x%llx]\n", - pKernelBus->bar1[gfid].staticBar1.size); - NV_PRINTF(LEVEL_ERROR, "Requested map range 0x%llx to 0x%llx, mapGranularity 0x%llx\n", - mapRange.start, mrangeLimit(mapRange) - 1llu, mapRange.size); - - memdescPrintMemdesc(pMemDesc, NV_TRUE, MAKE_NV_PRINTF_STR("Dumping memdesc:")); - - return NV_ERR_INVALID_ARGUMENT; + return NV_ERR_NOT_SUPPORTED; } if (bInDynamicRegion) diff --git a/src/nvidia/src/kernel/rmapi/nv_gpu_ops.c b/src/nvidia/src/kernel/rmapi/nv_gpu_ops.c index 9a005fd148..1db38cbbfd 100644 --- a/src/nvidia/src/kernel/rmapi/nv_gpu_ops.c +++ b/src/nvidia/src/kernel/rmapi/nv_gpu_ops.c @@ -3934,22 +3934,42 @@ nvGpuOpsMemGetPageSize * * @param[in] pAddresses : Array of physical addresses to be encoded. * @param[in] dmaBaseAddress : IOVA base address. + * @param[in] dmaSize : IOVA window size. + * @param[in] pageSize : Size covered by each physical address. * @param[in] count : Count of physical addresses. */ -static void +static NV_STATUS _nvGpuOpsEncodeBar1P2PAddrs ( NvU64 *pAddresses, NvU64 dmaBaseAddress, + NvU64 dmaSize, + NvU64 pageSize, NvU64 count ) { - NvU32 i; + NvU64 i; for (i = 0; i < count; i++) { - pAddresses[i] = dmaBaseAddress + pAddresses[i]; + NvU64 offset = pAddresses[i]; + NvU64 encodedAddress; + + if ((offset >= dmaSize) || + (pageSize > (dmaSize - offset)) || + !portSafeAddU64(dmaBaseAddress, offset, &encodedAddress)) + { + NV_PRINTF(LEVEL_ERROR, + "BAR1 P2P address range exceeds DMA window: " + "offset=0x%llx pageSize=0x%llx dmaBase=0x%llx dmaSize=0x%llx\n", + offset, pageSize, dmaBaseAddress, dmaSize); + return NV_ERR_INVALID_ADDRESS; + } + + pAddresses[i] = encodedAddress; } + + return NV_OK; } static @@ -4289,7 +4309,13 @@ nvGpuOpsBuildExternalAllocPtes status = NV_ERR_INVALID_STATE; goto done; } - _nvGpuOpsEncodeBar1P2PAddrs(physicalAddresses, dmaBaseAddress, pteCount); + NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR, + _nvGpuOpsEncodeBar1P2PAddrs(physicalAddresses, + dmaBaseAddress, + dmaSize, + mappingPageSize, + pteCount), + done); } else { @@ -4623,7 +4649,13 @@ nvGpuOpsBuildExternalAllocPhysAddrs status = NV_ERR_INVALID_STATE; goto done; } - _nvGpuOpsEncodeBar1P2PAddrs(physicalAddresses, dmaBaseAddress, physAddrCount); + NV_CHECK_OK_OR_GOTO(status, LEVEL_ERROR, + _nvGpuOpsEncodeBar1P2PAddrs(physicalAddresses, + dmaBaseAddress, + dmaSize, + mappingPageSize, + physAddrCount), + done); } else { diff --git a/tests/bar1_p2p_policy_test.c b/tests/bar1_p2p_policy_test.c new file mode 100644 index 0000000000..df16d31e29 --- /dev/null +++ b/tests/bar1_p2p_policy_test.c @@ -0,0 +1,36 @@ +/* SPDX-License-Identifier: MIT */ + +#include + +#include "../src/nvidia/src/kernel/gpu/bus/arch/turing/bar1_p2p_policy.h" + +int main(void) +{ + int defaultEnabled; + int fullCoverage; + int gb206Exception; + + for (defaultEnabled = 0; defaultEnabled <= 1; ++defaultEnabled) + { + for (fullCoverage = 0; fullCoverage <= 1; ++fullCoverage) + { + for (gb206Exception = 0; gb206Exception <= 1; ++gb206Exception) + { + int oldAccepted = defaultEnabled && gb206Exception; + int newAccepted = KBUS_USE_DISPLAY_AWARE_STATIC_BAR1( + defaultEnabled, fullCoverage, gb206Exception); + + assert(!oldAccepted || newAccepted); + assert(newAccepted == + (defaultEnabled && (fullCoverage || gb206Exception))); + } + } + } + + assert(KBUS_USE_DISPLAY_AWARE_STATIC_BAR1(1, 1, 0)); + assert(KBUS_USE_DISPLAY_AWARE_STATIC_BAR1(1, 0, 1)); + assert(!KBUS_USE_DISPLAY_AWARE_STATIC_BAR1(1, 0, 0)); + assert(!KBUS_USE_DISPLAY_AWARE_STATIC_BAR1(0, 1, 1)); + + return 0; +} From 3543e68bff1f30313fb0b5b80f77904387cb10cb Mon Sep 17 00:00:00 2001 From: "Duc P. Tran" <1055172+ducphuc@users.noreply.github.com> Date: Tue, 4 Aug 2026 11:34:49 -0700 Subject: [PATCH 10/21] Fix BAR1 diagnostic formatting --- src/nvidia/src/kernel/gpu/bus/arch/turing/kern_bus_tu102.c | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/src/nvidia/src/kernel/gpu/bus/arch/turing/kern_bus_tu102.c b/src/nvidia/src/kernel/gpu/bus/arch/turing/kern_bus_tu102.c index 3819e59bf1..8a04f18e48 100644 --- a/src/nvidia/src/kernel/gpu/bus/arch/turing/kern_bus_tu102.c +++ b/src/nvidia/src/kernel/gpu/bus/arch/turing/kern_bus_tu102.c @@ -459,8 +459,8 @@ kbusIsStaticBar1Supported_TU102 if (bar1VASizeAligned < (staticBar1Offset + bar1MapSize)) { - NV_PRINTF(LEVEL_ERROR, "BAR1 size %lld is not large enough to map FB size" - "%lld at offset %lld to force static BAR1\n", + NV_PRINTF(LEVEL_ERROR, "BAR1 size %" NvU64_fmtu " is not large enough to map FB size " + "%" NvU64_fmtu " at offset %" NvU64_fmtu " to force static BAR1\n", bar1VASizeAligned, bar1MapSize, staticBar1Offset); DBG_BREAKPOINT(); From e9ab701c07c4a56502edd6fec200af93086f1c3f Mon Sep 17 00:00:00 2001 From: "Duc P. Tran" <1055172+ducphuc@users.noreply.github.com> Date: Tue, 4 Aug 2026 11:36:00 -0700 Subject: [PATCH 11/21] Restore UVM coherence without registering Blackwell BAR1 as P2PDMA MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Report coherence from the actual system-memory window again, but keep non-coherent Blackwell devices out of the static-BAR1 PCI P2PDMA registration path. Static BAR1 is also the GPU peer aperture there, and P2PDMA registration replaces its pagemap operations. This intentionally leaves device P2PDMA unavailable on non-coherent Blackwell datacenter GPUs. It matches this branch’s effective behavior and does not remove GPU-to-GPU P2P support. --- kernel-open/nvidia-uvm/uvm_devmem.c | 12 ++++++++++++ kernel-open/nvidia-uvm/uvm_gpu.h | 7 ------- 2 files changed, 12 insertions(+), 7 deletions(-) diff --git a/kernel-open/nvidia-uvm/uvm_devmem.c b/kernel-open/nvidia-uvm/uvm_devmem.c index 38ca9713f5..bfa5b5f006 100644 --- a/kernel-open/nvidia-uvm/uvm_devmem.c +++ b/kernel-open/nvidia-uvm/uvm_devmem.c @@ -613,6 +613,18 @@ void uvm_devmem_device_p2p_init(uvm_parent_gpu_t *parent_gpu) parent_gpu->device_p2p_initialised = false; + if (parent_gpu->rm_info.gpuArch >= NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GB100) { + // Static BAR1 is also the GPU peer aperture on non-coherent Blackwell. + // Registering it as P2PDMA memory would replace its pagemap operations + // and conflict with the BAR1-as-sysmem PTEs used for GPU peer access. + UVM_DBG_PRINT("Skipping PCI P2PDMA static BAR1 registration on non-coherent GPU %s " + "(size 0x%llx, write-combined %u)\n", + uvm_parent_gpu_name(parent_gpu), + parent_gpu->static_bar1_size, + parent_gpu->static_bar1_write_combined); + return; + } + // RM sets static_bar1_size when it has created a contiguous BAR mapping // large enough to cover all of GPU memory that will be allocated to // userspace buffers. This is required to support the P2PDMA feature to diff --git a/kernel-open/nvidia-uvm/uvm_gpu.h b/kernel-open/nvidia-uvm/uvm_gpu.h index 7761f569cb..dedbfcce51 100644 --- a/kernel-open/nvidia-uvm/uvm_gpu.h +++ b/kernel-open/nvidia-uvm/uvm_gpu.h @@ -1825,13 +1825,6 @@ NvU64 uvm_parent_gpu_canonical_address(uvm_parent_gpu_t *parent_gpu, NvU64 addr) static bool uvm_parent_gpu_is_coherent(const uvm_parent_gpu_t *parent_gpu) { - // Blackwell+ consumer GPUs (e.g. 5090) use BAR1 P2P via the SYS_COH - // aperture rewrite in nvGpuOpsBuildExternalAllocPtes. UVM's P2P - // registration path must take the coherent route to match, otherwise - // the ZONE_DEVICE peer DMA setup conflicts with BAR1-as-sysmem PTEs. - if (parent_gpu->rm_info.gpuArch >= NV2080_CTRL_MC_ARCH_INFO_ARCHITECTURE_GB100) - return true; - return parent_gpu->system_bus.memory_window_end > parent_gpu->system_bus.memory_window_start; } From ae8144ced27742c7423391b6e58d14eb29759791 Mon Sep 17 00:00:00 2001 From: "Duc P. Tran" <1055172+ducphuc@users.noreply.github.com> Date: Tue, 4 Aug 2026 18:09:26 -0700 Subject: [PATCH 12/21] Record GB206 BAR1 boundary validation --- validation/gb206-bar1-boundary-2026-08-04.md | 119 +++++++++++++++++++ 1 file changed, 119 insertions(+) create mode 100644 validation/gb206-bar1-boundary-2026-08-04.md diff --git a/validation/gb206-bar1-boundary-2026-08-04.md b/validation/gb206-bar1-boundary-2026-08-04.md new file mode 100644 index 0000000000..6bdeec6ce4 --- /dev/null +++ b/validation/gb206-bar1-boundary-2026-08-04.md @@ -0,0 +1,119 @@ +# GB206 BAR1 boundary validation — 2026-08-04 + +## Revisions + +- Diagnostic branch: `test/gb206-bar1-boundary-pressure` +- Diagnostic tag/commit: `test/gb206-bar1-boundary-d0676cfc` / `d0676cfc` +- Production branch: `production/runtime-bar1-coverage` +- Production module commit: `d682adc5` +- Driver/kernel: `610.43.03` / `7.0.0-28-generic` +- Secure Boot signer: `sugardaddy Secure Boot Module Signature key` + +The production branch does not contain the diagnostic logging, CUDA boundary +harness source, or generated harness binary. + +## Diagnostic module identity + +| Module | SHA-256 | srcversion | +|---|---|---| +| `nvidia.ko` | `04925f26b64b5095705cf98f2e4fefd405ec300220fe340e2c0f07220553d346` | `9020898A6D608A1C767CC72` | +| `nvidia-uvm.ko` | `58ec9925164d3dc39d5a6b54076792fe3b80548c2b0fb3ef93501d9fd01bf4f9` | `1BD7F0E70C0717835738BDB` | + +Both modules used vermagic `7.0.0-28-generic SMP preempt mod_unload modversions`. + +## Natural geometry and deterministic matrix + +The test used natural BAR1 geometry only. The invalidating 4 GiB override was not +used. + +- Full aligned coverage: `selectedStatic=0x3e1000000`, aligned client FB + `0x3e1000000`. +- Display/console partial coverage: `selectedStatic=0x3dfe00000`, raw client FB + `0x3e10a0000`, console reservation `0x260000`, static offset `0x20000000`. +- Inside tests passed in both GPU directions and both peer-enable orderings, + including peer kernels, `cudaMemcpyPeer`, and beginning/middle/end probes. +- Full-coverage pressure reached normal allocation exhaustion after 1,969 passing + tests with no unsafe classification or data error. +- The partial side produced the same 64 MiB spanning range in both orderings: + `minOffset=0x3dcc00000`, `maxEnd=0x3e0c00000`, + `dmaSize=0x3dfe00000`. Peer-before rejected during allocation; peer-after + rejected during peer enablement. Each had 491 prior passing tests. +- Prepared boundary mode reported an outside range + `0x3e0c00000..0x3e1000000`, rejected it with `localHealthy=1`, and passed the + three-iteration recovery. + +## Rejection/recovery cycles + +From `2026-08-04T14:53:04-07:00` through +`2026-08-04T15:02:57-07:00`, 100 independent prepared-boundary cycles completed. +Every cycle required: + +- one spanning candidate rejection; +- one outside rejection with `localHealthy=1`; +- `rejected=2 apiErrors=2 dataErrors=0`; and +- a three-iteration inside recovery. + +Both GPUs returned to zero MiB used at every ten-cycle checkpoint. The cycle +window contained no assertion, Xid, mailbox setup failure, IOMMU/AER fault, +stale state, invalid state, or cleanup warning. + +The individual logs and their checksum manifest were written under `/tmp` and +were cleared by the required reboot. The pass count, checkpoints, and journal +result were captured before reboot; this file records the durable summary. + +## Production policy and routing audit + +The production predicate is additive: property-enabled GPUs use display-aware +placement only when runtime geometry covers all aligned client FB, while GB206 +retains the tested partial-window exception. Other partial geometries are not +generalized. `tests/bar1_p2p_policy_test.c` verifies the policy truth table and +that the existing exception cannot become a rejection. + +Generated HAL dispatch, the global `pcieP2PType` default, registry precedence, +and the GH100 BAR1 routing source were unchanged. Their pre/post hashes matched. +The live BAR1 encoder bounds checks remain present. + +## Signed production modules + +Installed path: `/lib/modules/7.0.0-28-generic/updates/local/`. + +| Module | SHA-256 | srcversion | +|---|---|---| +| `nvidia.ko` | `9bf54a3665eea55e839b93feb44f4c92c994db88baf7e31331341ab1d712a626` | `9020898A6D608A1C767CC72` | +| `nvidia-modeset.ko` | `8e3217b97a4432cc9b84f5d1eb475d8f76f0fc7985565bd4e20a986f1a3a65ef` | `0BCB09E2E1D4422BB162693` | +| `nvidia-drm.ko` | `7402ab6be81127e636d5bd29b9920a9aaa204c6bc5991848781add2a7fa75b9f` | `65769FC23A53EFDFC4A2DB5` | +| `nvidia-uvm.ko` | `b5c2d10652954ff40c566b596ebbf2d56b094d6be722f5123de37bf29da3dfea` | `1BD7F0E70C0717835738BDB` | +| `nvidia-peermem.ko` | `61363a4d58f1f0fccf29b944b833e35502e0a584a64fd5202190b18858b4f9ba` | `05E8CF2F419E46C7D3D974E` | + +## Production validation + +- Full module build and source policy regression: passed. +- `nvidia-smi`: both RTX 5060 Ti GPUs healthy. +- P2P read/write capability: `OK` both directions. +- P2P atomics: `NS`, not `DR` (not disabled by the registry default). +- `simpleP2P`: passed before reload, after reload, after resume, and after reboot; + 13.09 GB/s. +- `p2pBandwidthLatencyTest`: 14.09 GB/s each unidirectional P2P direction and + 27.79–27.80 GB/s bidirectional. +- Both peer-enable orderings passed bidirectional inside correctness. +- The known partial-boundary sequence rejected in both orderings and recovered + immediately. +- Modeset/DRM stack load and full driver unload/reload: passed. +- Deep suspend via the enabled NVIDIA systemd suspend/resume hooks: entered at + 15:17:13 and exited at 15:17:39; post-resume P2P passed. +- Boot from the updated initramfs: candidate hashes/signer matched and post-boot + P2P passed. No GPU-specific boot journal fault was present. + +## Suspend configuration notes + +`NVreg_PreserveVideoMemoryAllocations=1` correctly requires the NVIDIA procfs +suspend hook; a raw `rtcwake -m mem` attempt was rejected, while the supported +systemd path succeeded. `NVreg_TemporaryFilePath=/var` selects the root filesystem +on `/dev/nvme6n1p5` for preservation files, but the journal I/O errors referenced +the separate `/dev/nvme6n1p3` partition. That partition is intentionally inaccessible +while OPAL-locked for BitLocker, so those messages are expected and are unrelated to +the NVIDIA validation. + +The restricted Codex mount namespace exposes `/` with a read-only VFS mount flag +while the ext4 filesystem reports `rw`; this does not indicate that the host root +filesystem was remounted read-only. No storage failure is inferred from this test. From 543ba973840b2689f451806a6c982780c7a1c4e8 Mon Sep 17 00:00:00 2001 From: "Duc P. Tran" <1055172+ducphuc@users.noreply.github.com> Date: Tue, 4 Aug 2026 16:13:17 -0700 Subject: [PATCH 13/21] Make BAR1 policy test reproducible --- tests/.gitignore | 1 + tests/Makefile | 18 ++++++++++++++++++ tests/README.md | 11 +++++++++++ validation/gb206-bar1-boundary-2026-08-04.md | 5 +++-- 4 files changed, 33 insertions(+), 2 deletions(-) create mode 100644 tests/.gitignore create mode 100644 tests/Makefile create mode 100644 tests/README.md diff --git a/tests/.gitignore b/tests/.gitignore new file mode 100644 index 0000000000..95e69555a7 --- /dev/null +++ b/tests/.gitignore @@ -0,0 +1 @@ +/bar1_p2p_policy_test diff --git a/tests/Makefile b/tests/Makefile new file mode 100644 index 0000000000..1589c5525d --- /dev/null +++ b/tests/Makefile @@ -0,0 +1,18 @@ +CC ?= cc + +POLICY_TEST := bar1_p2p_policy_test +POLICY_HEADER := ../src/nvidia/src/kernel/gpu/bus/arch/turing/bar1_p2p_policy.h +POLICY_TEST_CFLAGS := -std=c11 -Wall -Wextra -Werror + +.PHONY: all check clean + +all: $(POLICY_TEST) + +$(POLICY_TEST): $(POLICY_TEST).c $(POLICY_HEADER) + $(CC) $(CPPFLAGS) $(CFLAGS) $(POLICY_TEST_CFLAGS) -o $@ $< + +check: $(POLICY_TEST) + ./$(POLICY_TEST) + +clean: + $(RM) $(POLICY_TEST) diff --git a/tests/README.md b/tests/README.md new file mode 100644 index 0000000000..c0ece98328 --- /dev/null +++ b/tests/README.md @@ -0,0 +1,11 @@ +# BAR1 policy tests + +Run the source-level BAR1 P2P policy regression test with: + +```sh +make -C tests check +``` + +The test verifies the runtime-coverage truth table, including that the GB206 +partial-window exception remains accepted and that full-coverage devices are +added only when BAR1 P2P is enabled by the existing device property. diff --git a/validation/gb206-bar1-boundary-2026-08-04.md b/validation/gb206-bar1-boundary-2026-08-04.md index 6bdeec6ce4..6fb4c05620 100644 --- a/validation/gb206-bar1-boundary-2026-08-04.md +++ b/validation/gb206-bar1-boundary-2026-08-04.md @@ -66,8 +66,9 @@ result were captured before reboot; this file records the durable summary. The production predicate is additive: property-enabled GPUs use display-aware placement only when runtime geometry covers all aligned client FB, while GB206 retains the tested partial-window exception. Other partial geometries are not -generalized. `tests/bar1_p2p_policy_test.c` verifies the policy truth table and -that the existing exception cannot become a rejection. +generalized. `make -C tests check` runs `tests/bar1_p2p_policy_test.c` to verify +the policy truth table and that the existing exception cannot become a +rejection. Generated HAL dispatch, the global `pcieP2PType` default, registry precedence, and the GH100 BAR1 routing source were unchanged. Their pre/post hashes matched. From 39c4f0bf897a81c516ad9f037a3c502cc2455940 Mon Sep 17 00:00:00 2001 From: Duc Tran Date: Wed, 5 Aug 2026 22:15:00 -0700 Subject: [PATCH 14/21] Add full MIT notice to BAR1 policy header Co-authored-by: Copilot Autofix powered by AI <175728472+Copilot@users.noreply.github.com> --- .../gpu/bus/arch/turing/bar1_p2p_policy.h | 18 ++++++++++++++++++ 1 file changed, 18 insertions(+) diff --git a/src/nvidia/src/kernel/gpu/bus/arch/turing/bar1_p2p_policy.h b/src/nvidia/src/kernel/gpu/bus/arch/turing/bar1_p2p_policy.h index e6b7975a1e..031d913ff1 100644 --- a/src/nvidia/src/kernel/gpu/bus/arch/turing/bar1_p2p_policy.h +++ b/src/nvidia/src/kernel/gpu/bus/arch/turing/bar1_p2p_policy.h @@ -1,6 +1,24 @@ /* * SPDX-FileCopyrightText: Copyright (c) 2026 Duc P. Tran * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. */ #ifndef KERN_BUS_BAR1_P2P_POLICY_H From 26e1ff536e6375d74d36a9e16e71e3ea81f61b64 Mon Sep 17 00:00:00 2001 From: "Duc P. Tran" <1055172+ducphuc@users.noreply.github.com> Date: Wed, 5 Aug 2026 22:38:38 -0700 Subject: [PATCH 15/21] Generalize partial BAR1 coverage using runtime geometry Allow every property-enabled GPU to use display-aware static BAR1 when runtime geometry leaves nonzero aligned client-FB and static-window ranges. Remove the GB206 implementation check while retaining the same inside-window behavior and fail-closed spanning/outside rejection. Keep the GB206 record as the validated partial-coverage example without treating it as an eligibility allowlist. --- README.md | 11 +++--- .../gpu/bus/arch/turing/bar1_p2p_policy.h | 11 +++--- .../gpu/bus/arch/turing/kern_bus_tu102.c | 36 +++++++----------- tests/README.md | 8 ++-- tests/bar1_p2p_policy_test.c | 36 +++++++----------- validation/gb206-bar1-boundary-2026-08-04.md | 37 ++++++++++++++++--- 6 files changed, 74 insertions(+), 65 deletions(-) diff --git a/README.md b/README.md index 090286ec15..f9d6f060d0 100644 --- a/README.md +++ b/README.md @@ -24,11 +24,12 @@ This enables BAR1 P2P on consumer GPUs where NVLink isn't available, and falls b NVLink where it is. For PCIe pairs, transfers write directly to the other GPU's physical address over DMA. -On default-enabled GPUs, display-aware static BAR1 placement is used when runtime geometry -covers all aligned client framebuffer memory. GB206 cards (RTX 5060 Ti / 5060) retain an -experimental partial-window exception so inside-range P2P works with a display attached. -Allocations spanning or outside that partial static window are rejected by the CUDA API; -there is currently no transparent dynamic-mapping fallback for those allocations. +On property-enabled GPUs, display-aware static BAR1 placement is used whenever runtime +geometry leaves a non-empty aligned static window after fixed console and mailbox +reservations. Partial windows support allocations wholly inside that window; allocations +spanning or outside it are rejected by the CUDA API because there is currently no +transparent dynamic-mapping fallback. GB206 cards (RTX 5060 Ti / 5060) are the +hardware-validated partial-coverage example, not an implementation allowlist. > [!WARNING] > IOMMU must currently be in passthrough mode (`iommu=pt`), not translating. In particular, diff --git a/src/nvidia/src/kernel/gpu/bus/arch/turing/bar1_p2p_policy.h b/src/nvidia/src/kernel/gpu/bus/arch/turing/bar1_p2p_policy.h index 031d913ff1..4008b21262 100644 --- a/src/nvidia/src/kernel/gpu/bus/arch/turing/bar1_p2p_policy.h +++ b/src/nvidia/src/kernel/gpu/bus/arch/turing/bar1_p2p_policy.h @@ -25,11 +25,12 @@ #define KERN_BUS_BAR1_P2P_POLICY_H /* - * Display-aware placement is additive for default-enabled devices whose - * runtime BAR1 geometry covers all aligned client FB. GB206 keeps its tested - * partial-window exception; other partial windows are not generalized. + * Display-aware placement is additive for default-enabled devices with a + * non-empty aligned client FB range and a non-empty aligned static BAR1 + * window. Partial windows are safe because each external mapping is checked + * against the selected static BAR1 DMA window before its addresses are used. */ -#define KBUS_USE_DISPLAY_AWARE_STATIC_BAR1(defaultEnabled, fullCoverage, gb206Exception) \ - ((defaultEnabled) && ((fullCoverage) || (gb206Exception))) +#define KBUS_USE_DISPLAY_AWARE_STATIC_BAR1(defaultEnabled, clientFbSize, maxStaticMapSize) \ + ((defaultEnabled) && ((clientFbSize) != 0) && ((maxStaticMapSize) != 0)) #endif // KERN_BUS_BAR1_P2P_POLICY_H diff --git a/src/nvidia/src/kernel/gpu/bus/arch/turing/kern_bus_tu102.c b/src/nvidia/src/kernel/gpu/bus/arch/turing/kern_bus_tu102.c index 8a04f18e48..74e28c0c50 100644 --- a/src/nvidia/src/kernel/gpu/bus/arch/turing/kern_bus_tu102.c +++ b/src/nvidia/src/kernel/gpu/bus/arch/turing/kern_bus_tu102.c @@ -396,20 +396,15 @@ kbusIsStaticBar1Supported_TU102 NvU64 maxStaticMapSize = (bar1VASizeAligned > staticBar1Offset) ? RM_ALIGN_DOWN(bar1VASizeAligned - staticBar1Offset, RM_PAGE_SIZE_2M) : 0; - NvBool bCompleteStaticCoverage = - ((clientFbSizeAligned != 0) && (maxStaticMapSize >= clientFbSizeAligned)); - NvBool bGb206CoverageException = - ((gpuGetChipImpl(pGpu) == GPU_IMPLEMENTATION_GB206) && - (maxStaticMapSize != 0)); NvBool bUseDisplayAwareStaticBar1 = KBUS_USE_DISPLAY_AWARE_STATIC_BAR1(bBar1P2PDefault, - bCompleteStaticCoverage, - bGb206CoverageException); + clientFbSizeAligned, + maxStaticMapSize); // - // Default-enabled GPUs may place a complete static mapping after fixed - // console/mailbox mappings when runtime geometry covers all aligned client - // FB. Preserve the tested GB206 partial-window behavior, but do not extend - // partial coverage to other implementations. + // Default-enabled GPUs may place a complete or partial static mapping + // after fixed console/mailbox mappings whenever runtime geometry leaves a + // non-empty aligned window. External mappings are checked against the + // resulting DMA window, so spanning and outside allocations fail safely. // NvU64 autoStaticMapSize = bUseDisplayAwareStaticBar1 ? ((clientFbSizeAligned < maxStaticMapSize) ? @@ -607,15 +602,10 @@ kbusEnableStaticBar1Mapping_TU102 NvU64 maxStaticMapSize = (bar1Offset < bar1VASizeAligned) ? RM_ALIGN_DOWN(bar1VASizeAligned - bar1Offset, RM_PAGE_SIZE_2M) : 0; - NvBool bCompleteStaticCoverage = - ((clientFbSizeAligned != 0) && (maxStaticMapSize >= clientFbSizeAligned)); - NvBool bGb206CoverageException = - ((gpuGetChipImpl(pGpu) == GPU_IMPLEMENTATION_GB206) && - (maxStaticMapSize != 0)); NvBool bUseDisplayAwareStaticBar1 = KBUS_USE_DISPLAY_AWARE_STATIC_BAR1(bBar1P2PDefault, - bCompleteStaticCoverage, - bGb206CoverageException); + clientFbSizeAligned, + maxStaticMapSize); if (bUseDisplayAwareStaticBar1 && (bar1MapSize > maxStaticMapSize)) { @@ -1163,11 +1153,11 @@ kbusGetStaticFbAperture_TU102 { // // The static region may not cover all of client FB: it is rounded - // down to 2MB, and on display-attached GB206 it is clipped to the - // BAR1 VA left after the console/mailbox reservation. The static BAR1 - // path cannot represent a range spanning that boundary; current CUDA - // P2P callers receive a predictable API rejection rather than a - // transparent dynamic-mapping fallback. + // down to 2MB and may be clipped to the BAR1 VA left after the + // console/mailbox reservation. The static BAR1 path cannot represent + // a range spanning that boundary; current CUDA P2P callers receive a + // predictable API rejection rather than a transparent dynamic-mapping + // fallback. // return NV_ERR_NOT_SUPPORTED; } diff --git a/tests/README.md b/tests/README.md index c0ece98328..d991be436d 100644 --- a/tests/README.md +++ b/tests/README.md @@ -6,6 +6,8 @@ Run the source-level BAR1 P2P policy regression test with: make -C tests check ``` -The test verifies the runtime-coverage truth table, including that the GB206 -partial-window exception remains accepted and that full-coverage devices are -added only when BAR1 P2P is enabled by the existing device property. +The test verifies the runtime-coverage truth table. Display-aware placement is +available only when BAR1 P2P is enabled by the existing device property and +both the aligned client framebuffer and available static BAR1 window are +non-empty. Partial, exact, and larger-than-client coverage are accepted without +an implementation-specific exception. diff --git a/tests/bar1_p2p_policy_test.c b/tests/bar1_p2p_policy_test.c index df16d31e29..fda9280762 100644 --- a/tests/bar1_p2p_policy_test.c +++ b/tests/bar1_p2p_policy_test.c @@ -6,31 +6,21 @@ int main(void) { - int defaultEnabled; - int fullCoverage; - int gb206Exception; + const unsigned long long clientFbSize = 16ULL << 30; + const unsigned long long partialStaticSize = 15ULL << 30; - for (defaultEnabled = 0; defaultEnabled <= 1; ++defaultEnabled) - { - for (fullCoverage = 0; fullCoverage <= 1; ++fullCoverage) - { - for (gb206Exception = 0; gb206Exception <= 1; ++gb206Exception) - { - int oldAccepted = defaultEnabled && gb206Exception; - int newAccepted = KBUS_USE_DISPLAY_AWARE_STATIC_BAR1( - defaultEnabled, fullCoverage, gb206Exception); + assert(!KBUS_USE_DISPLAY_AWARE_STATIC_BAR1(0, clientFbSize, + partialStaticSize)); + assert(!KBUS_USE_DISPLAY_AWARE_STATIC_BAR1(1, 0, + partialStaticSize)); + assert(!KBUS_USE_DISPLAY_AWARE_STATIC_BAR1(1, clientFbSize, 0)); - assert(!oldAccepted || newAccepted); - assert(newAccepted == - (defaultEnabled && (fullCoverage || gb206Exception))); - } - } - } - - assert(KBUS_USE_DISPLAY_AWARE_STATIC_BAR1(1, 1, 0)); - assert(KBUS_USE_DISPLAY_AWARE_STATIC_BAR1(1, 0, 1)); - assert(!KBUS_USE_DISPLAY_AWARE_STATIC_BAR1(1, 0, 0)); - assert(!KBUS_USE_DISPLAY_AWARE_STATIC_BAR1(0, 1, 1)); + assert(KBUS_USE_DISPLAY_AWARE_STATIC_BAR1(1, clientFbSize, + partialStaticSize)); + assert(KBUS_USE_DISPLAY_AWARE_STATIC_BAR1(1, clientFbSize, + clientFbSize)); + assert(KBUS_USE_DISPLAY_AWARE_STATIC_BAR1(1, clientFbSize, + clientFbSize + (1ULL << 30))); return 0; } diff --git a/validation/gb206-bar1-boundary-2026-08-04.md b/validation/gb206-bar1-boundary-2026-08-04.md index 6fb4c05620..ffbff3419a 100644 --- a/validation/gb206-bar1-boundary-2026-08-04.md +++ b/validation/gb206-bar1-boundary-2026-08-04.md @@ -63,12 +63,37 @@ result were captured before reboot; this file records the durable summary. ## Production policy and routing audit -The production predicate is additive: property-enabled GPUs use display-aware -placement only when runtime geometry covers all aligned client FB, while GB206 -retains the tested partial-window exception. Other partial geometries are not -generalized. `make -C tests check` runs `tests/bar1_p2p_policy_test.c` to verify -the policy truth table and that the existing exception cannot become a -rejection. +At the commit validated by this record, the production predicate was additive: +property-enabled GPUs used display-aware placement when runtime geometry +covered all aligned client FB, while GB206 retained the tested partial-window +exception. `make -C tests check` ran `tests/bar1_p2p_policy_test.c` to verify +that policy truth table. + +The policy was subsequently generalized to use only the existing BAR1 P2P +device property and runtime geometry. Any property-enabled GPU with a non-empty +aligned client FB range and non-empty aligned static BAR1 window may use the +same partial-window behavior. This record remains hardware evidence for GB206; +it does not claim that other partial-window implementations were tested. + +### Post-generalization GB206 regression + +On 2026-08-05, the generalized runtime policy was built as all five kernel +modules, signed with the enrolled Secure Boot key, installed on kernel +`7.0.0-29-generic`, and exercised on the same two RTX 5060 Ti GPUs. The bounded +boundary run completed with 492 inside passes and no data errors. On the +partial-coverage GPU, the established 64 MiB spanning candidate and a prepared +4 MiB outside candidate were rejected by the retained static-aperture bounds +checks. Local access remained healthy and the immediate three-iteration inside +recovery passed. The other GPU reached ordinary allocation exhaustion without +an unsafe mapping or data error. + +`simpleP2P` passed before and after the boundary run at 13.05 and 13.03 GB/s. +`p2pBandwidthLatencyTest` measured 14.09 GB/s in each unidirectional direction +and 27.79 GB/s bidirectionally. The post-load kernel log contained the expected +fail-closed boundary diagnostics and no Xid, assertion, IOMMU/MMU fault, AER +error, oops, panic, or hung-task report. This regression confirms unchanged +GB206 behavior; other partial-window GPU implementations remain hardware +validation follow-ups. Generated HAL dispatch, the global `pcieP2PType` default, registry precedence, and the GH100 BAR1 routing source were unchanged. Their pre/post hashes matched. From 7e24c0bea27c108cbea006e58a222d9503943e76 Mon Sep 17 00:00:00 2001 From: "Duc P. Tran" <1055172+ducphuc@users.noreply.github.com> Date: Thu, 6 Aug 2026 23:33:37 -0700 Subject: [PATCH 16/21] Serialize FORCE_PCIE DMA-BUF physical-address operations Static physical addresses normally allow nv_dma_buf_get_phys_addresses() and nv_dma_buf_put_phys_addresses() to skip the API and GPU locks because the memory descriptor is immutable. FORCE_PCIE is different: it maps and unmaps the BAR1 aperture and updates per-GPU RUSD statistics. Take the lock for every non-default mapping type so concurrent registration and deregistration cannot race the RUSD seqlock. --- kernel-open/nvidia/nv-dmabuf.c | 25 ++++++++++++++++++++----- 1 file changed, 20 insertions(+), 5 deletions(-) diff --git a/kernel-open/nvidia/nv-dmabuf.c b/kernel-open/nvidia/nv-dmabuf.c index 21897f2c27..782207fb27 100644 --- a/kernel-open/nvidia/nv-dmabuf.c +++ b/kernel-open/nvidia/nv-dmabuf.c @@ -572,7 +572,15 @@ nv_dma_buf_put_phys_addresses ( return; } - if (!priv->static_phys_addrs) + // + // See the matching comment in nv_dma_buf_get_phys_addresses(): locking + // can be skipped only for MAPPING_TYPE_DEFAULT static phys addr configs. + // MAPPING_TYPE_FORCE_PCIE unmap still calls into kbusUnmapFbAperture_HAL(), + // which updates per-GPU RUSD statistics and is not safe to run + // concurrently without the GPU lock. + // + if (!priv->static_phys_addrs || + (priv->mapping_type != NV_DMABUF_EXPORT_MAPPING_TYPE_DEFAULT)) { status = rm_acquire_api_lock(sp); if (WARN_ON(status != NV_OK)) @@ -633,11 +641,18 @@ nv_dma_buf_get_phys_addresses ( } // - // Locking is not needed for static phys address configs because the memdesc - // is not expected to change in this case and we hold the refcount on the - // owner GPU and memory before referencing it. + // Locking can be skipped for static phys address configs because the + // memdesc is not expected to change in this case and we hold the + // refcount on the owner GPU and memory before referencing it. This only + // holds for MAPPING_TYPE_DEFAULT: RM's static_phys_addrs determination + // also covers MAPPING_TYPE_FORCE_PCIE (any GPU with static BAR1 + // enabled), but that mapping type does a real BAR1 aperture + // map/unmap and a per-GPU RUSD statistics update on every call + // (see kbusMapFbApertureSingle()/kbusUpdateRusdStatistics()), which + // are not safe to run concurrently without the GPU lock. // - if (!priv->static_phys_addrs) + if (!priv->static_phys_addrs || + (priv->mapping_type != NV_DMABUF_EXPORT_MAPPING_TYPE_DEFAULT)) { status = rm_acquire_api_lock(sp); if (status != NV_OK) From 7952e6c5f8c41f2bc14bf603c610baa0662d3d8a Mon Sep 17 00:00:00 2001 From: "Duc P. Tran" <1055172+ducphuc@users.noreply.github.com> Date: Thu, 6 Aug 2026 23:33:16 -0700 Subject: [PATCH 17/21] Add generic non-coherent DMA-BUF GDR policy Define a default-off policy for non-coherent FORCE_PCIE DMA-BUF exports without a chip, architecture, or device-ID allowlist. Evaluate every driver-supported GPU using runtime static-BAR1, BAR, MIG, mapping-type, and coherence predicates. Add standalone tests for admission, overflow-safe BAR1 range containment, importer DMA-mask coverage, identity-IOMMU requirements, and Linux P2PDMA distance. Preserve the existing coherent path. --- .../nvidia/dmabuf-gdr-topology-policy.h | 44 +++++++++++++++++ kernel-open/nvidia/nv-reg.h | 24 ++++++++++ .../nvalloc/unix/include/dmabuf_gdr_policy.h | 47 +++++++++++++++++++ src/nvidia/arch/nvalloc/unix/include/nv-reg.h | 24 ++++++++++ src/nvidia/interface/nvrm_registry.h | 4 ++ tests/.gitignore | 2 + tests/Makefile | 25 +++++++--- tests/README.md | 10 ++++ tests/dmabuf_gdr_policy_test.c | 39 +++++++++++++++ tests/dmabuf_gdr_topology_policy_test.c | 34 ++++++++++++++ 10 files changed, 246 insertions(+), 7 deletions(-) create mode 100644 kernel-open/nvidia/dmabuf-gdr-topology-policy.h create mode 100644 src/nvidia/arch/nvalloc/unix/include/dmabuf_gdr_policy.h create mode 100644 tests/dmabuf_gdr_policy_test.c create mode 100644 tests/dmabuf_gdr_topology_policy_test.c diff --git a/kernel-open/nvidia/dmabuf-gdr-topology-policy.h b/kernel-open/nvidia/dmabuf-gdr-topology-policy.h new file mode 100644 index 0000000000..95811977d6 --- /dev/null +++ b/kernel-open/nvidia/dmabuf-gdr-topology-policy.h @@ -0,0 +1,44 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 Duc P. Tran + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef DMABUF_GDR_TOPOLOGY_POLICY_H +#define DMABUF_GDR_TOPOLOGY_POLICY_H + +/* Overflow-safe check that the importer can address the complete BAR. */ +#define DMABUF_GDR_BAR_ADDRESSABLE(barStart, barSize, dmaMask) \ + (((barSize) != 0) && ((barStart) <= (dmaMask)) && \ + (((barSize) - 1) <= ((dmaMask) - (barStart)))) + +/* + * Keep the non-coherent topology exception deliberately narrow. Linux must + * approve the complete P2PDMA path and the importer must be in an identity + * IOMMU domain. The experimental path then uses dma_map_resource() instead of + * the stock FORCE_PCIE IOMMU bypass. + */ +#define DMABUF_GDR_TOPOLOGY_ALLOWED(enabled, identityIommu, p2pDistance, \ + barStart, barSize, dmaMask) \ + ((enabled) && (identityIommu) && \ + ((p2pDistance) >= 0) && \ + DMABUF_GDR_BAR_ADDRESSABLE((barStart), (barSize), (dmaMask))) + +#endif // DMABUF_GDR_TOPOLOGY_POLICY_H diff --git a/kernel-open/nvidia/nv-reg.h b/kernel-open/nvidia/nv-reg.h index decf092752..66f863dfab 100644 --- a/kernel-open/nvidia/nv-reg.h +++ b/kernel-open/nvidia/nv-reg.h @@ -1015,6 +1015,27 @@ #define NV_GPU_INIT_ON_PROBE NV_REG_STRING(__NV_GPU_INIT_ON_PROBE) #define NV_GPU_INIT_ON_PROBE_DEFAULT 0 +/* + * Option: NVreg_ExperimentalDmaBufP2P + * + * Description: + * + * Enables the capability-gated validation path for exporting framebuffer + * memory from non-coherent GPUs through DMA-BUF with FORCE_PCIE mappings. This + * option is experimental, defaults to disabled, and accepts importers only + * when Linux approves the complete P2PDMA path in an identity IOMMU domain and + * the importer's DMA mask covers BAR1. Coherent GPUs retain the stock path; + * other non-coherent topologies remain rejected. + * + * Possible values: + * 0 - Disable the experimental path (default) + * 1 - Enable the experimental non-coherent FORCE_PCIE path + */ +#define __NV_EXPERIMENTAL_DMABUF_P2P ExperimentalDmaBufP2P +#define NV_REG_EXPERIMENTAL_DMABUF_P2P \ + NV_REG_STRING(__NV_EXPERIMENTAL_DMABUF_P2P) +#define NV_REG_EXPERIMENTAL_DMABUF_P2P_DEFAULT 0 + #if defined(NV_DEFINE_REGISTRY_KEY_TABLE) /* @@ -1057,6 +1078,8 @@ NV_DEFINE_REG_ENTRY_GLOBAL(__NV_TEGRA_GPU_PG_MASK, 0); NV_DEFINE_REG_ENTRY_GLOBAL(__NV_ENABLE_NONBLOCKING_OPEN, 1); NV_DEFINE_REG_ENTRY_GLOBAL(__NV_EXCLUDE_ALL_GPUS, NV_EXCLUDE_ALL_GPUS_DEFAULT); NV_DEFINE_REG_ENTRY_GLOBAL(__NV_GPU_INIT_ON_PROBE, NV_GPU_INIT_ON_PROBE_DEFAULT); +NV_DEFINE_REG_ENTRY_GLOBAL(__NV_EXPERIMENTAL_DMABUF_P2P, + NV_REG_EXPERIMENTAL_DMABUF_P2P_DEFAULT); NV_DEFINE_REG_STRING_ENTRY(__NV_COHERENT_GPU_MEMORY_MODE, NULL); NV_DEFINE_REG_STRING_ENTRY(__NV_REGISTRY_DWORDS, NULL); @@ -1125,6 +1148,7 @@ nv_parm_t nv_parms[] = { NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_ENABLE_SYSTEM_MEMORY_POOLS), NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_OS_ENABLE_CXL_SUPPORT), NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_ENABLE_NON_PREEMTABLE_DEBUGGER_SESSION), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_EXPERIMENTAL_DMABUF_P2P), {NULL, NULL} }; diff --git a/src/nvidia/arch/nvalloc/unix/include/dmabuf_gdr_policy.h b/src/nvidia/arch/nvalloc/unix/include/dmabuf_gdr_policy.h new file mode 100644 index 0000000000..556bf9b5bd --- /dev/null +++ b/src/nvidia/arch/nvalloc/unix/include/dmabuf_gdr_policy.h @@ -0,0 +1,47 @@ +/* + * SPDX-FileCopyrightText: Copyright (c) 2026 Duc P. Tran + * SPDX-License-Identifier: MIT + * + * Permission is hereby granted, free of charge, to any person obtaining a + * copy of this software and associated documentation files (the "Software"), + * to deal in the Software without restriction, including without limitation + * the rights to use, copy, modify, merge, publish, distribute, sublicense, + * and/or sell copies of the Software, and to permit persons to whom the + * Software is furnished to do so, subject to the following conditions: + * + * The above copyright notice and this permission notice shall be included in + * all copies or substantial portions of the Software. + * + * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR + * IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, + * FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL + * THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER + * LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING + * FROM, OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER + * DEALINGS IN THE SOFTWARE. + */ + +#ifndef DMABUF_GDR_POLICY_H +#define DMABUF_GDR_POLICY_H + +/* + * The exception is deliberately narrower than the stock coherent path. It is + * available only for an explicitly enabled non-coherent FORCE_PCIE export + * backed by an active static BAR1 mapping, with BAR1 and MIG exclusions + * retained. Coherent GPUs continue to use the stock path. + */ +#define DMABUF_GDR_NONCOHERENT_ALLOWED(enabled, coherent, forcePcie, \ + staticBar1Enabled, bar1Disabled, migEnabled) \ + ((enabled) && !(coherent) && (forcePcie) && (staticBar1Enabled) && \ + !(bar1Disabled) && !(migEnabled)) + +/* + * Overflow-safe half-open range containment. Subtraction is performed only + * after ordering checks, and size is compared against the remaining window. + */ +#define DMABUF_GDR_RANGE_CONTAINED(start, size, windowStart, windowSize) \ + (((size) != 0) && ((windowSize) != 0) && ((start) >= (windowStart)) && \ + (((start) - (windowStart)) <= (windowSize)) && \ + ((size) <= ((windowSize) - ((start) - (windowStart))))) + +#endif // DMABUF_GDR_POLICY_H diff --git a/src/nvidia/arch/nvalloc/unix/include/nv-reg.h b/src/nvidia/arch/nvalloc/unix/include/nv-reg.h index decf092752..66f863dfab 100644 --- a/src/nvidia/arch/nvalloc/unix/include/nv-reg.h +++ b/src/nvidia/arch/nvalloc/unix/include/nv-reg.h @@ -1015,6 +1015,27 @@ #define NV_GPU_INIT_ON_PROBE NV_REG_STRING(__NV_GPU_INIT_ON_PROBE) #define NV_GPU_INIT_ON_PROBE_DEFAULT 0 +/* + * Option: NVreg_ExperimentalDmaBufP2P + * + * Description: + * + * Enables the capability-gated validation path for exporting framebuffer + * memory from non-coherent GPUs through DMA-BUF with FORCE_PCIE mappings. This + * option is experimental, defaults to disabled, and accepts importers only + * when Linux approves the complete P2PDMA path in an identity IOMMU domain and + * the importer's DMA mask covers BAR1. Coherent GPUs retain the stock path; + * other non-coherent topologies remain rejected. + * + * Possible values: + * 0 - Disable the experimental path (default) + * 1 - Enable the experimental non-coherent FORCE_PCIE path + */ +#define __NV_EXPERIMENTAL_DMABUF_P2P ExperimentalDmaBufP2P +#define NV_REG_EXPERIMENTAL_DMABUF_P2P \ + NV_REG_STRING(__NV_EXPERIMENTAL_DMABUF_P2P) +#define NV_REG_EXPERIMENTAL_DMABUF_P2P_DEFAULT 0 + #if defined(NV_DEFINE_REGISTRY_KEY_TABLE) /* @@ -1057,6 +1078,8 @@ NV_DEFINE_REG_ENTRY_GLOBAL(__NV_TEGRA_GPU_PG_MASK, 0); NV_DEFINE_REG_ENTRY_GLOBAL(__NV_ENABLE_NONBLOCKING_OPEN, 1); NV_DEFINE_REG_ENTRY_GLOBAL(__NV_EXCLUDE_ALL_GPUS, NV_EXCLUDE_ALL_GPUS_DEFAULT); NV_DEFINE_REG_ENTRY_GLOBAL(__NV_GPU_INIT_ON_PROBE, NV_GPU_INIT_ON_PROBE_DEFAULT); +NV_DEFINE_REG_ENTRY_GLOBAL(__NV_EXPERIMENTAL_DMABUF_P2P, + NV_REG_EXPERIMENTAL_DMABUF_P2P_DEFAULT); NV_DEFINE_REG_STRING_ENTRY(__NV_COHERENT_GPU_MEMORY_MODE, NULL); NV_DEFINE_REG_STRING_ENTRY(__NV_REGISTRY_DWORDS, NULL); @@ -1125,6 +1148,7 @@ nv_parm_t nv_parms[] = { NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_ENABLE_SYSTEM_MEMORY_POOLS), NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_OS_ENABLE_CXL_SUPPORT), NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_ENABLE_NON_PREEMTABLE_DEBUGGER_SESSION), + NV_DEFINE_PARAMS_TABLE_ENTRY(__NV_EXPERIMENTAL_DMABUF_P2P), {NULL, NULL} }; diff --git a/src/nvidia/interface/nvrm_registry.h b/src/nvidia/interface/nvrm_registry.h index 2e4a95a208..dc67d2577b 100644 --- a/src/nvidia/interface/nvrm_registry.h +++ b/src/nvidia/interface/nvrm_registry.h @@ -1130,6 +1130,10 @@ #define NV_REG_STR_RM_PCIEP2P_TYPE_AUTO (0x00000002) #define NV_REG_STR_RM_PCIEP2P_TYPE_DEFAULT NV_REG_STR_RM_PCIEP2P_TYPE_MAILBOX +// Experimental, default-off non-coherent DMA-BUF FORCE_PCIE validation path, +// including a Linux-approved importer path in an identity IOMMU domain. +#define NV_REG_STR_EXPERIMENTAL_DMABUF_P2P "ExperimentalDmaBufP2P" + // // Type: DWORD // Enables/Disables the WAR for bug 1630288 where we disable 3rd-party peer mappings diff --git a/tests/.gitignore b/tests/.gitignore index 95e69555a7..0a304ebb0b 100644 --- a/tests/.gitignore +++ b/tests/.gitignore @@ -1 +1,3 @@ /bar1_p2p_policy_test +/dmabuf_gdr_policy_test +/dmabuf_gdr_topology_policy_test diff --git a/tests/Makefile b/tests/Makefile index 1589c5525d..92782c7c7d 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -1,18 +1,29 @@ CC ?= cc -POLICY_TEST := bar1_p2p_policy_test -POLICY_HEADER := ../src/nvidia/src/kernel/gpu/bus/arch/turing/bar1_p2p_policy.h +POLICY_TESTS := bar1_p2p_policy_test dmabuf_gdr_policy_test \ + dmabuf_gdr_topology_policy_test +BAR1_POLICY_HEADER := ../src/nvidia/src/kernel/gpu/bus/arch/turing/bar1_p2p_policy.h +DMABUF_GDR_POLICY_HEADER := ../src/nvidia/arch/nvalloc/unix/include/dmabuf_gdr_policy.h +DMABUF_GDR_TOPOLOGY_POLICY_HEADER := ../kernel-open/nvidia/dmabuf-gdr-topology-policy.h POLICY_TEST_CFLAGS := -std=c11 -Wall -Wextra -Werror .PHONY: all check clean -all: $(POLICY_TEST) +all: $(POLICY_TESTS) -$(POLICY_TEST): $(POLICY_TEST).c $(POLICY_HEADER) +bar1_p2p_policy_test: bar1_p2p_policy_test.c $(BAR1_POLICY_HEADER) $(CC) $(CPPFLAGS) $(CFLAGS) $(POLICY_TEST_CFLAGS) -o $@ $< -check: $(POLICY_TEST) - ./$(POLICY_TEST) +dmabuf_gdr_policy_test: dmabuf_gdr_policy_test.c $(DMABUF_GDR_POLICY_HEADER) + $(CC) $(CPPFLAGS) $(CFLAGS) $(POLICY_TEST_CFLAGS) -o $@ $< + +dmabuf_gdr_topology_policy_test: dmabuf_gdr_topology_policy_test.c $(DMABUF_GDR_TOPOLOGY_POLICY_HEADER) + $(CC) $(CPPFLAGS) $(CFLAGS) $(POLICY_TEST_CFLAGS) -o $@ $< + +check: $(POLICY_TESTS) + ./bar1_p2p_policy_test + ./dmabuf_gdr_policy_test + ./dmabuf_gdr_topology_policy_test clean: - $(RM) $(POLICY_TEST) + $(RM) $(POLICY_TESTS) diff --git a/tests/README.md b/tests/README.md index d991be436d..df389fd00d 100644 --- a/tests/README.md +++ b/tests/README.md @@ -11,3 +11,13 @@ available only when BAR1 P2P is enabled by the existing device property and both the aligned client framebuffer and available static BAR1 window are non-empty. Partial, exact, and larger-than-client coverage are accepted without an implementation-specific exception. + +It also verifies that the experimental non-coherent DMA-BUF GDR exception +remains default-off, does not replace the stock coherent path, and requires +FORCE_PCIE, static BAR1, and the existing BAR1/MIG exclusions. Its range checks +cover inside, spanning, outside, empty, and overflowing layouts. + +The topology-policy test keeps the non-coherent importer exception limited to +the default-off experiment, a Linux-approved P2PDMA path, an identity IOMMU +domain, and an importer DMA mask that covers the complete BAR1 resource. Each +eligibility predicate has an explicit negative case. diff --git a/tests/dmabuf_gdr_policy_test.c b/tests/dmabuf_gdr_policy_test.c new file mode 100644 index 0000000000..9de9dd21b7 --- /dev/null +++ b/tests/dmabuf_gdr_policy_test.c @@ -0,0 +1,39 @@ +/* SPDX-License-Identifier: MIT */ + +#include +#include + +#include "../src/nvidia/arch/nvalloc/unix/include/dmabuf_gdr_policy.h" + +int main(void) +{ + uint64_t windowStart = UINT64_C(0x20000000); + uint64_t windowSize = UINT64_C(0x3dfe00000); + uint64_t windowEnd = windowStart + windowSize; + uint64_t emptySize = 0; + + assert(DMABUF_GDR_NONCOHERENT_ALLOWED(1, 0, 1, 1, 0, 0)); + assert(!DMABUF_GDR_NONCOHERENT_ALLOWED(0, 0, 1, 1, 0, 0)); + assert(!DMABUF_GDR_NONCOHERENT_ALLOWED(1, 1, 1, 1, 0, 0)); + assert(!DMABUF_GDR_NONCOHERENT_ALLOWED(1, 0, 0, 1, 0, 0)); + assert(!DMABUF_GDR_NONCOHERENT_ALLOWED(1, 0, 1, 0, 0, 0)); + assert(!DMABUF_GDR_NONCOHERENT_ALLOWED(1, 0, 1, 1, 1, 0)); + assert(!DMABUF_GDR_NONCOHERENT_ALLOWED(1, 0, 1, 1, 0, 1)); + + assert(DMABUF_GDR_RANGE_CONTAINED(windowStart, UINT64_C(0x1000), + windowStart, windowSize)); + assert(DMABUF_GDR_RANGE_CONTAINED(windowEnd - UINT64_C(0x1000), + UINT64_C(0x1000), windowStart, windowSize)); + assert(!DMABUF_GDR_RANGE_CONTAINED(windowEnd - UINT64_C(0x1000), + UINT64_C(0x2000), windowStart, windowSize)); + assert(!DMABUF_GDR_RANGE_CONTAINED(windowEnd, UINT64_C(0x1000), + windowStart, windowSize)); + assert(!DMABUF_GDR_RANGE_CONTAINED(windowStart - UINT64_C(0x1000), + UINT64_C(0x1000), windowStart, windowSize)); + assert(!DMABUF_GDR_RANGE_CONTAINED(windowStart, emptySize, + windowStart, windowSize)); + assert(!DMABUF_GDR_RANGE_CONTAINED(UINT64_MAX - UINT64_C(0x1000), + UINT64_C(0x2000), windowStart, windowSize)); + + return 0; +} diff --git a/tests/dmabuf_gdr_topology_policy_test.c b/tests/dmabuf_gdr_topology_policy_test.c new file mode 100644 index 0000000000..6ffa497fe8 --- /dev/null +++ b/tests/dmabuf_gdr_topology_policy_test.c @@ -0,0 +1,34 @@ +/* SPDX-License-Identifier: MIT */ + +#include +#include + +#include "../kernel-open/nvidia/dmabuf-gdr-topology-policy.h" + +int main(void) +{ + uint64_t barStart = UINT64_C(0x26000000000); + uint64_t barSize = UINT64_C(0x400000000); + uint64_t exactMask = barStart + barSize - 1; + uint64_t wideMask = UINT64_C(0xffffffffffffffff); + + assert(DMABUF_GDR_BAR_ADDRESSABLE(barStart, barSize, exactMask)); + assert(DMABUF_GDR_BAR_ADDRESSABLE(barStart, barSize, wideMask)); + assert(!DMABUF_GDR_BAR_ADDRESSABLE(barStart, barSize, exactMask - 1)); + assert(!DMABUF_GDR_BAR_ADDRESSABLE(barStart, UINT64_C(0), wideMask)); + assert(!DMABUF_GDR_BAR_ADDRESSABLE(UINT64_MAX - UINT64_C(0x1000), + UINT64_C(0x2000), UINT64_MAX)); + + assert(DMABUF_GDR_TOPOLOGY_ALLOWED(1, 1, 4, + barStart, barSize, wideMask)); + assert(!DMABUF_GDR_TOPOLOGY_ALLOWED(0, 1, 4, + barStart, barSize, wideMask)); + assert(!DMABUF_GDR_TOPOLOGY_ALLOWED(1, 0, 4, + barStart, barSize, wideMask)); + assert(!DMABUF_GDR_TOPOLOGY_ALLOWED(1, 1, -1, + barStart, barSize, wideMask)); + assert(!DMABUF_GDR_TOPOLOGY_ALLOWED(1, 1, 4, + barStart, barSize, exactMask - 1)); + + return 0; +} From fd9cf24eb8de92bcfe4b602112730c0a85229653 Mon Sep 17 00:00:00 2001 From: "Duc P. Tran" <1055172+ducphuc@users.noreply.github.com> Date: Thu, 6 Aug 2026 23:33:29 -0700 Subject: [PATCH 18/21] Allow safe non-coherent FORCE_PCIE DMA-BUF exports Allow the explicitly enabled non-coherent path to report and admit FORCE_PCIE exports only when static BAR1 is active, BAR1 is usable, and MIG is disabled. Require every exported range to be nonzero and wholly contained in static BAR1 using overflow-safe checks. Validate address translation before modifying ranges and unwind aperture mappings on every failure. Preserve the stock coherent export path and all existing ownership and memory-type validation. Keep RmGpuDirectRdmaForceSPA restricted to coherent GPUs when relaxing the non-coherent FORCE_PCIE rejection: it is a coherent-platform hypervisor workaround, not a DMA-BUF GDR capability requirement, and must stay unreachable for non-coherent GPUs. Express both admission decisions as policy macros (DMABUF_GDR_NONCOHERENT_ALLOWED, DMABUF_GDR_USE_GRDMA_SPA) with regression coverage for all four FORCE_PCIE/coherent/forceSPA combinations. --- .../nvalloc/unix/include/dmabuf_gdr_policy.h | 9 ++ src/nvidia/arch/nvalloc/unix/src/osapi.c | 110 +++++++++++++++++- src/nvidia/src/kernel/gpu/bus/kern_bus.c | 27 ++++- .../gpu/subdevice/subdevice_ctrl_gpu_kernel.c | 36 +++++- tests/dmabuf_gdr_policy_test.c | 12 ++ 5 files changed, 185 insertions(+), 9 deletions(-) diff --git a/src/nvidia/arch/nvalloc/unix/include/dmabuf_gdr_policy.h b/src/nvidia/arch/nvalloc/unix/include/dmabuf_gdr_policy.h index 556bf9b5bd..5d5d00e498 100644 --- a/src/nvidia/arch/nvalloc/unix/include/dmabuf_gdr_policy.h +++ b/src/nvidia/arch/nvalloc/unix/include/dmabuf_gdr_policy.h @@ -35,6 +35,15 @@ ((enabled) && !(coherent) && (forcePcie) && (staticBar1Enabled) && \ !(bar1Disabled) && !(migEnabled)) +/* + * The RmGpuDirectRdmaForceSPA hypervisor workaround is a coherent-platform + * address-translation override, not a DMA-BUF GDR capability requirement. + * It must stay unreachable for non-coherent GPUs regardless of whether the + * experimental non-coherent FORCE_PCIE path is permitted. + */ +#define DMABUF_GDR_USE_GRDMA_SPA(forcePcie, coherent, forceSpa) \ + ((forcePcie) && (coherent) && (forceSpa)) + /* * Overflow-safe half-open range containment. Subtraction is performed only * after ordering checks, and size is compared against the remaining window. diff --git a/src/nvidia/arch/nvalloc/unix/src/osapi.c b/src/nvidia/arch/nvalloc/unix/src/osapi.c index 894cd66387..adbfa1d797 100644 --- a/src/nvidia/arch/nvalloc/unix/src/osapi.c +++ b/src/nvidia/arch/nvalloc/unix/src/osapi.c @@ -61,6 +61,7 @@ #include #include #include "nv-reg.h" +#include "dmabuf_gdr_policy.h" #include "nv-firmware-registry.h" #include "core/hal_mgr.h" #include "gpu/device/device.h" @@ -1372,6 +1373,17 @@ static NvU32 RmDmabufMmapGetCpuCacheType( #endif } +static NvBool +_isExperimentalDmaBufP2PEnabled(OBJGPU *pGpu) +{ + NvU32 data = 0; + + return (osReadRegistryDword(pGpu, + NV_REG_STR_EXPERIMENTAL_DMABUF_P2P, + &data) == NV_OK) && + (data != 0); +} + static NV_STATUS RmDmabufVerifyMemHandle( OBJGPU *pGpu, @@ -1481,13 +1493,25 @@ RmDmabufGetClientAndDevice( if (mappingType == NV_DMABUF_EXPORT_MAPPING_TYPE_FORCE_PCIE) { KernelBus *pKernelBus = GPU_GET_KERNEL_BUS(pGpu); - - if (!pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING) || + NvBool bCoherent = pGpu->getProperty( + pGpu, + PDB_PROP_GPU_COHERENT_CPU_MAPPING); + NvBool bExperimentalAllowed = + DMABUF_GDR_NONCOHERENT_ALLOWED( + _isExperimentalDmaBufP2PEnabled(pGpu), + bCoherent, + NV_TRUE, + kbusIsStaticBar1Enabled(pGpu, pKernelBus), + pKernelBus->bBar1Disabled, + IS_MIG_ENABLED(pGpu)); + + if ((!bCoherent && !bExperimentalAllowed) || pKernelBus->bBar1Disabled || IS_MIG_ENABLED(pGpu)) { return NV_ERR_NOT_SUPPORTED; } + } if (IS_MIG_ENABLED(pGpu)) @@ -5795,10 +5819,17 @@ NV_STATUS NV_API_CALL rm_dma_buf_map_mem_handle( RsClient *pClient; NvU64 idx; NvU64 barOffset; + NvU64 translatedStart; KernelBus *pKernelBus; + NvBool bCoherent; NvBool bForcePcie; + NvBool bApertureMapped = NV_FALSE; + NvBool bExperimentalAllowed; pKernelBus = GPU_GET_KERNEL_BUS(pGpu); + bCoherent = pGpu->getProperty( + pGpu, + PDB_PROP_GPU_COHERENT_CPU_MAPPING); if (!bStaticPhysAddrs) { @@ -5829,18 +5860,91 @@ NV_STATUS NV_API_CALL rm_dma_buf_map_mem_handle( (BUS_MAP_FB_FLAGS_MAP_UNICAST | BUS_MAP_FB_FLAGS_ALLOW_DISCONTIG), pDevice), Done); + bApertureMapped = NV_TRUE; bForcePcie = (mappingType == NV_DMABUF_EXPORT_MAPPING_TYPE_FORCE_PCIE); + bExperimentalAllowed = + DMABUF_GDR_NONCOHERENT_ALLOWED( + _isExperimentalDmaBufP2PEnabled(pGpu), + bCoherent, + bForcePcie, + kbusIsStaticBar1Enabled(pGpu, pKernelBus), + pKernelBus->bBar1Disabled, + IS_MIG_ENABLED(pGpu)); + + if (bExperimentalAllowed) + { + NvU64 staticStart = + pKernelBus->bar1[GPU_GFID_PF].staticBar1.startOffset; + NvU64 staticSize = + pKernelBus->bar1[GPU_GFID_PF].staticBar1.size; + + for (idx = 0; idx < pMemArea->numRanges; idx++) + { + MemoryRange range = pMemArea->pRanges[idx]; + + if (!DMABUF_GDR_RANGE_CONTAINED(range.start, range.size, + staticStart, staticSize)) + { + NV_PRINTF( + LEVEL_ERROR, + "GDR-DMABUF stage=map gpu=%u range=%llu " + "start=0x%llx size=0x%llx staticStart=0x%llx " + "staticSize=0x%llx status=outside_static_bar1\n", + gpuGetInstance(pGpu), + idx, + range.start, + range.size, + staticStart, + staticSize); + rmStatus = NV_ERR_NOT_SUPPORTED; + goto UnmapFbAperture; + } + } + } NV_ASSERT_OK_OR_GOTO(rmStatus, kbusGetGpuFbPhysAddressForRdma(pGpu, pKernelBus, bForcePcie, &barOffset), - Done); + UnmapFbAperture); + + // Validate all additions before changing any range in-place. + for (idx = 0; idx < pMemArea->numRanges; idx++) + { + if (!portSafeAddU64(pMemArea->pRanges[idx].start, + barOffset, &translatedStart)) + { + rmStatus = NV_ERR_INVALID_ADDRESS; + goto UnmapFbAperture; + } + } for (idx = 0; idx < pMemArea->numRanges; idx++) { pMemArea->pRanges[idx].start += barOffset; } + + goto Done; + +UnmapFbAperture: + if (bApertureMapped) + { + NV_STATUS unmapStatus = kbusUnmapFbAperture_HAL( + pGpu, pKernelBus, pMemDesc, *pMemArea, + BUS_MAP_FB_FLAGS_MAP_UNICAST); + + if (unmapStatus != NV_OK) + { + NV_PRINTF( + LEVEL_ERROR, + "GDR-DMABUF stage=map_cleanup gpu=%u " + "mapStatus=0x%x unmapStatus=0x%x\n", + gpuGetInstance(pGpu), rmStatus, unmapStatus); + } + + pMemArea->pRanges = NULL; + pMemArea->numRanges = 0; + } } Done: diff --git a/src/nvidia/src/kernel/gpu/bus/kern_bus.c b/src/nvidia/src/kernel/gpu/bus/kern_bus.c index d3d45ec586..9d66d3e293 100644 --- a/src/nvidia/src/kernel/gpu/bus/kern_bus.c +++ b/src/nvidia/src/kernel/gpu/bus/kern_bus.c @@ -37,6 +37,7 @@ #include "nvdevid.h" #include "containers/eheap_old.h" #include "gpu/bus/p2p_api.h" +#include "dmabuf_gdr_policy.h" #include "gpu/gsp/gsp_static_config.h" #include "vgpu/rpc.h" @@ -1312,8 +1313,24 @@ kbusGetGpuFbPhysAddressForRdma_IMPL NvU64 *pPhysAddr ) { - if((bForcePcie) && - (!pGpu->getProperty(pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING))) + NvBool bCoherent = pGpu->getProperty( + pGpu, PDB_PROP_GPU_COHERENT_CPU_MAPPING); + NvU32 experimental = 0; + NvBool bExperimentalEnabled = + (osReadRegistryDword( + pGpu, NV_REG_STR_EXPERIMENTAL_DMABUF_P2P, + &experimental) == NV_OK) && + (experimental != 0); + NvBool bExperimentalAllowed = + DMABUF_GDR_NONCOHERENT_ALLOWED( + bExperimentalEnabled, + bCoherent, + bForcePcie, + kbusIsStaticBar1Enabled(pGpu, pKernelBus), + pKernelBus->bBar1Disabled, + IS_MIG_ENABLED(pGpu)); + + if (bForcePcie && !bCoherent && !bExperimentalAllowed) { return NV_ERR_NOT_SUPPORTED; } @@ -1323,7 +1340,11 @@ kbusGetGpuFbPhysAddressForRdma_IMPL // if the RmGpuDirectRdmaForceSPA regkey is set. // This is a stop-gap measure until hypervisor ensures GPA==SPA. // - if (bForcePcie && pKernelBus->bGrdmaForceSpa) + // bCoherent is required here: this is a coherent-platform hypervisor + // workaround, and must stay unreachable for non-coherent GPUs even when + // the experimental non-coherent DMA-BUF GDR path allows FORCE_PCIE. + // + if (DMABUF_GDR_USE_GRDMA_SPA(bForcePcie, bCoherent, pKernelBus->bGrdmaForceSpa)) { *pPhysAddr = pKernelBus->grdmaBar1Spa; } diff --git a/src/nvidia/src/kernel/gpu/subdevice/subdevice_ctrl_gpu_kernel.c b/src/nvidia/src/kernel/gpu/subdevice/subdevice_ctrl_gpu_kernel.c index 64e95216de..86a00af491 100644 --- a/src/nvidia/src/kernel/gpu/subdevice/subdevice_ctrl_gpu_kernel.c +++ b/src/nvidia/src/kernel/gpu/subdevice/subdevice_ctrl_gpu_kernel.c @@ -56,6 +56,7 @@ #include "gpu/mem_mgr/mem_mgr.h" #include "virtualization/hypervisor/hypervisor.h" #include "gpu/mem_sys/kern_mem_sys.h" +#include "dmabuf_gdr_policy.h" #include "gpu/nvenc/nvencsession.h" #include "kernel/gpu/fifo/kernel_fifo.h" #include "gpu/ce/kernel_ce.h" @@ -86,6 +87,17 @@ #define INDEX_FORWARD_TO_PHYSICAL 0x80000000 ct_assert(INDEX_FORWARD_TO_PHYSICAL == DRF_NUM(2080, _CTRL_GPU_INFO_INDEX, _RESERVED, 1)); +static NvBool +_isExperimentalDmaBufP2PEnabled(OBJGPU *pGpu) +{ + NvU32 data = 0; + + return (osReadRegistryDword(pGpu, + NV_REG_STR_EXPERIMENTAL_DMABUF_P2P, + &data) == NV_OK) && + (data != 0); +} + static NV_STATUS getGpuInfos(Subdevice *pSubdevice, NV2080_CTRL_GPU_GET_INFO_V2_PARAMS *pParams, NvBool bCanAccessHw) @@ -526,14 +538,32 @@ getGpuInfos(Subdevice *pSubdevice, NV2080_CTRL_GPU_GET_INFO_V2_PARAMS *pParams, } case NV2080_CTRL_GPU_INFO_INDEX_DMABUF_CAPABILITY: { + KernelBus *pKernelBus = GPU_GET_KERNEL_BUS(pGpu); + NvBool bOsDmabuf = osDmabufIsSupported(); + NvBool bApm = gpuIsApmFeatureEnabled(pGpu); + NvBool bCoherent = pGpu->getProperty( + pGpu, + PDB_PROP_GPU_COHERENT_CPU_MAPPING); + NvBool bExperimental = + _isExperimentalDmaBufP2PEnabled(pGpu); + NvBool bExperimentalAllowed = + DMABUF_GDR_NONCOHERENT_ALLOWED( + bExperimental, + bCoherent, + NV_TRUE, + kbusIsStaticBar1Enabled(pGpu, pKernelBus), + pKernelBus->bBar1Disabled, + IS_MIG_ENABLED(pGpu)); + data = NV2080_CTRL_GPU_INFO_INDEX_DMABUF_CAPABILITY_NO; - if (osDmabufIsSupported() && - (!gpuIsApmFeatureEnabled(pGpu)) && - (!NVCPU_IS_PPC64LE)) + if (bOsDmabuf && + !NVCPU_IS_PPC64LE && + (!bApm || bExperimentalAllowed)) { data = NV2080_CTRL_GPU_INFO_INDEX_DMABUF_CAPABILITY_YES; } + break; } case NV2080_CTRL_GPU_INFO_INDEX_IS_RESETLESS_MIG_SUPPORTED: diff --git a/tests/dmabuf_gdr_policy_test.c b/tests/dmabuf_gdr_policy_test.c index 9de9dd21b7..ef91556e97 100644 --- a/tests/dmabuf_gdr_policy_test.c +++ b/tests/dmabuf_gdr_policy_test.c @@ -20,6 +20,18 @@ int main(void) assert(!DMABUF_GDR_NONCOHERENT_ALLOWED(1, 0, 1, 1, 1, 0)); assert(!DMABUF_GDR_NONCOHERENT_ALLOWED(1, 0, 1, 1, 0, 1)); + /* forcePcie=1, coherent=1, forceSpa=1 -> SPA */ + assert(DMABUF_GDR_USE_GRDMA_SPA(1, 1, 1)); + /* forcePcie=1, coherent=1, forceSpa=0 -> normal FB base */ + assert(!DMABUF_GDR_USE_GRDMA_SPA(1, 1, 0)); + /* forcePcie=1, coherent=0, forceSpa=1 -> normal FB base (regression case) */ + assert(!DMABUF_GDR_USE_GRDMA_SPA(1, 0, 1)); + /* forcePcie=1, coherent=0, forceSpa=0 -> normal FB base */ + assert(!DMABUF_GDR_USE_GRDMA_SPA(1, 0, 0)); + /* forcePcie=0 never selects SPA regardless of coherence/forceSpa */ + assert(!DMABUF_GDR_USE_GRDMA_SPA(0, 1, 1)); + assert(!DMABUF_GDR_USE_GRDMA_SPA(0, 0, 1)); + assert(DMABUF_GDR_RANGE_CONTAINED(windowStart, UINT64_C(0x1000), windowStart, windowSize)); assert(DMABUF_GDR_RANGE_CONTAINED(windowEnd - UINT64_C(0x1000), From 989a5866ff3e0b286f850421f4658826345c4b53 Mon Sep 17 00:00:00 2001 From: "Duc P. Tran" <1055172+ducphuc@users.noreply.github.com> Date: Thu, 6 Aug 2026 23:33:29 -0700 Subject: [PATCH 19/21] Validate non-coherent DMA-BUF GDR importers Admit the experimental non-coherent path only when the importer uses an identity IOMMU domain, its DMA mask covers BAR1, and Linux pci_p2pdma_distance() accepts the complete path, including valid NODE topologies. Map BAR1 through the importer DMA API with skipIommu disabled. Keep the existing coherent topology decisions and IOMMU-bypass behavior unchanged. --- kernel-open/common/inc/nv-pci.h | 2 +- kernel-open/common/inc/nv.h | 3 + kernel-open/nvidia/nv-dmabuf.c | 13 +++- kernel-open/nvidia/nv-pci.c | 89 +++++++++++++++++++++-- src/nvidia/arch/nvalloc/unix/include/nv.h | 3 + src/nvidia/arch/nvalloc/unix/src/osinit.c | 11 +++ 6 files changed, 108 insertions(+), 13 deletions(-) diff --git a/kernel-open/common/inc/nv-pci.h b/kernel-open/common/inc/nv-pci.h index f2882dde73..a3587679e0 100644 --- a/kernel-open/common/inc/nv-pci.h +++ b/kernel-open/common/inc/nv-pci.h @@ -36,7 +36,7 @@ int nv_pci_count_devices(void); NvU8 nv_find_pci_capability(struct pci_dev *, NvU8); int nvidia_dev_get_pci_info(const NvU8 *, struct pci_dev **, NvU64 *, NvU64 *); nv_linux_state_t * find_pci(NvU32, NvU8, NvU8, NvU8); -NvBool nv_pci_is_valid_topology_for_direct_pci(nv_state_t *, struct pci_dev *); +NvBool nv_pci_is_valid_topology_for_direct_pci(nv_state_t *, struct pci_dev *, NvBool *); NvBool nv_pci_has_common_pci_switch(nv_state_t *nv, struct pci_dev *); void nv_pci_tegra_boost_clocks(struct device *dev); diff --git a/kernel-open/common/inc/nv.h b/kernel-open/common/inc/nv.h index f2256a5e1a..9f19cb31b6 100644 --- a/kernel-open/common/inc/nv.h +++ b/kernel-open/common/inc/nv.h @@ -584,6 +584,9 @@ typedef struct nv_state_t /* Bool to check if dma-buf is supported */ NvBool dma_buf_supported; + /* Default-off non-coherent DMA-BUF GDR validation state from RM */ + NvBool experimental_dmabuf_p2p_enabled; + /* Bool to check if the device received a shutdown notification */ NvBool is_shutdown; diff --git a/kernel-open/nvidia/nv-dmabuf.c b/kernel-open/nvidia/nv-dmabuf.c index 782207fb27..cea5ac6431 100644 --- a/kernel-open/nvidia/nv-dmabuf.c +++ b/kernel-open/nvidia/nv-dmabuf.c @@ -1044,8 +1044,11 @@ nv_dma_buf_attach( if (priv->mapping_type == NV_DMABUF_EXPORT_MAPPING_TYPE_FORCE_PCIE) { + NvBool skip_iommu; + if(!nv_pci_is_valid_topology_for_direct_pci(priv->nv, - to_pci_dev(attachment->dev))) + to_pci_dev(attachment->dev), + &skip_iommu)) { nv_printf(NV_DBG_ERRORS, "NVRM: dma-buf attach failed: " @@ -1054,7 +1057,7 @@ nv_dma_buf_attach( goto unlock_priv; } - priv->skip_iommu = NV_TRUE; + priv->skip_iommu = skip_iommu; } else { @@ -1123,8 +1126,10 @@ nv_dma_buf_map( } // - // For MAPPING_TYPE_FORCE_PCIE on coherent platforms, - // get the BAR1 PFN scatterlist instead of C2C pages. + // For MAPPING_TYPE_FORCE_PCIE, get the BAR1 PFN scatterlist instead of + // C2C pages. Stock coherent platforms bypass IOMMU mapping after their + // existing topology check. The experimental non-coherent path instead + // maps BAR1 through the importer's DMA API. // // If nv->coherent is true, that could mean two things: // 1. GPU memory has struct page from memory onlining(NUMA) diff --git a/kernel-open/nvidia/nv-pci.c b/kernel-open/nvidia/nv-pci.c index 82338b8e22..88b53af758 100644 --- a/kernel-open/nvidia/nv-pci.c +++ b/kernel-open/nvidia/nv-pci.c @@ -27,11 +27,15 @@ #include "nv-msi.h" #include "nv-hypervisor.h" #include "nv-reg.h" +#include "dmabuf-gdr-topology-policy.h" #if defined(NV_VGPU_KVM_BUILD) #include "nv-vgpu-vfio-interface.h" #endif #include +#if defined(CONFIG_PCI_P2PDMA) +#include +#endif #include #include @@ -2890,24 +2894,93 @@ nv_pci_count_devices(void) */ NvBool nv_pci_is_valid_topology_for_direct_pci( nv_state_t *nv, - struct pci_dev *peer + struct pci_dev *peer, + NvBool *skip_iommu ) { struct pci_dev *pdev0 = to_pci_dev(nv->dma_dev->dev); struct pci_dev *pdev1 = peer; + NvBool result = NV_FALSE; + NvBool identity_iommu = NV_FALSE; + NvBool bar_addressable = NV_FALSE; + NvBool experimental = nv->experimental_dmabuf_p2p_enabled; + NvS32 p2p_distance = -1; + NvU64 dma_mask = dma_get_mask(&pdev1->dev); + + *skip_iommu = NV_TRUE; if (!nv->coherent) { - return NV_FALSE; - } +#if defined(CONFIG_PCI_P2PDMA) && defined(NV_IOMMU_IS_DMA_DOMAIN_PRESENT) + struct iommu_domain *domain; + domain = iommu_get_domain_for_dev(&pdev1->dev); + if (domain != NULL) + { + identity_iommu = (domain->type == IOMMU_DOMAIN_IDENTITY); + } - if (pdev0->dev.iommu_group == pdev1->dev.iommu_group) - return NV_TRUE; + bar_addressable = DMABUF_GDR_BAR_ADDRESSABLE( + nv->bars[NV_GPU_BAR_INDEX_FB].cpu_address, + nv->bars[NV_GPU_BAR_INDEX_FB].size, + dma_mask); - if (pdev1->dev.iommu_group == NULL) - return nv_pci_has_common_pci_switch(nv, peer); + if (experimental && identity_iommu && bar_addressable) + { + p2p_distance = pci_p2pdma_distance(pdev0, &pdev1->dev, NV_TRUE); + } - return NV_FALSE; + result = DMABUF_GDR_TOPOLOGY_ALLOWED( + experimental, + identity_iommu, + p2p_distance, + nv->bars[NV_GPU_BAR_INDEX_FB].cpu_address, + nv->bars[NV_GPU_BAR_INDEX_FB].size, + dma_mask); + + if (result) + { + // Map BAR1 through the importer's DMA API; do not bypass its IOMMU. + *skip_iommu = NV_FALSE; + } +#endif + } + else if (pdev0->dev.iommu_group == pdev1->dev.iommu_group) + { + result = NV_TRUE; + } + else if (pdev1->dev.iommu_group == NULL) + { + result = nv_pci_has_common_pci_switch(nv, peer); + } + else + { + result = NV_FALSE; + } + + if (experimental && !nv->coherent) + { + nv_printf( + NV_DBG_INFO, + "NVRM: DMA-BUF GDR topology: " + "gpu=%04x:%02x:%02x.%u importer=%04x:%02x:%02x.%u " + "identityIommu=%u p2pDistance=%d barAddressable=%u " + "skipIommu=%u result=%u\n", + nv->pci_info.domain, + nv->pci_info.bus, + nv->pci_info.slot, + nv->pci_info.function, + pci_domain_nr(pdev1->bus), + pdev1->bus->number, + PCI_SLOT(pdev1->devfn), + PCI_FUNC(pdev1->devfn), + identity_iommu, + p2p_distance, + bar_addressable, + *skip_iommu, + result); + } + + return result; } NvBool nv_pci_has_common_pci_switch( diff --git a/src/nvidia/arch/nvalloc/unix/include/nv.h b/src/nvidia/arch/nvalloc/unix/include/nv.h index 3db4fb3f53..c666bf1802 100644 --- a/src/nvidia/arch/nvalloc/unix/include/nv.h +++ b/src/nvidia/arch/nvalloc/unix/include/nv.h @@ -584,6 +584,9 @@ typedef struct nv_state_t /* Bool to check if dma-buf is supported */ NvBool dma_buf_supported; + /* Default-off non-coherent DMA-BUF GDR validation state from RM */ + NvBool experimental_dmabuf_p2p_enabled; + /* Bool to check if the device received a shutdown notification */ NvBool is_shutdown; diff --git a/src/nvidia/arch/nvalloc/unix/src/osinit.c b/src/nvidia/arch/nvalloc/unix/src/osinit.c index b3fd343726..b438aa3977 100644 --- a/src/nvidia/arch/nvalloc/unix/src/osinit.c +++ b/src/nvidia/arch/nvalloc/unix/src/osinit.c @@ -665,12 +665,23 @@ RmInitGpuInfoWithRmApi if (status == NV_OK) { + NvU32 experimental = 0; + nvp->b_mobile_config_enabled = (pGpuInfoParams->gpuInfoList[0].data == NV2080_CTRL_GPU_INFO_INDEX_MOBILE_CONFIG_ENABLED_YES); nv->dma_buf_supported = (pGpuInfoParams->gpuInfoList[1].data == NV2080_CTRL_GPU_INFO_INDEX_DMABUF_CAPABILITY_YES); + nv->experimental_dmabuf_p2p_enabled = NV_FALSE; + + (void)osReadRegistryDword( + pGpu, + NV_REG_STR_EXPERIMENTAL_DMABUF_P2P, + &experimental); + + nv->experimental_dmabuf_p2p_enabled = (experimental != 0); + } nv->coherent = From 6ad4e52bdaf488c43e255d6e4d1831b18f00465b Mon Sep 17 00:00:00 2001 From: "Duc P. Tran" <1055172+ducphuc@users.noreply.github.com> Date: Thu, 6 Aug 2026 23:34:16 -0700 Subject: [PATCH 20/21] Record same-host DMA-BUF GDR transport validation Document successful ib_write_bw and NCCL NET/IB/GDRDMA operation on two RTX 5060 Ti GPUs with a ConnectX-6 Lx in NODE topology. Record the required private CUDA userspace capability patch, exact NCCL environment, module and library identities, MAC-counter evidence, default-off registry option, and unvalidated hardware and topology scope. --- ...nccl-same-host-gdr-transport-2026-08-06.md | 245 ++++++++++++++++++ 1 file changed, 245 insertions(+) create mode 100644 validation/nccl-same-host-gdr-transport-2026-08-06.md diff --git a/validation/nccl-same-host-gdr-transport-2026-08-06.md b/validation/nccl-same-host-gdr-transport-2026-08-06.md new file mode 100644 index 0000000000..fac8d94286 --- /dev/null +++ b/validation/nccl-same-host-gdr-transport-2026-08-06.md @@ -0,0 +1,245 @@ +# Same-host GPUDirect RDMA transport validation — 2026-08-06 + +## Revisions + +- Branch: `feature/consumer-geforce-dmabuf-gdr` +- Validated module tree: hardware-tested provenance commit `12f9bb93` +- Driver/kernel: `610.43.03` / `7.0.0-29-generic` +- Secure Boot signer: local Secure Boot Module Signature key (enrolled MOK) +- CUDA userspace: private one-byte-patched `libcuda.so.610.43.03` + +## System identity + +| Component | Detail | +|---|---| +| GPU 0 | NVIDIA GeForce RTX 5060 Ti, PCI `0000:41:00.0` | +| GPU 1 | NVIDIA GeForce RTX 5060 Ti, PCI `0000:42:00.0` | +| NIC | NVIDIA ConnectX-6 Lx dual-port 25 GbE RoCE (MT2894), PCI `0000:62:00.0`/`.1`, firmware `26.49.1014` | +| RDMA devices | `mlx5_0` (port state Active), `mlx5_1` (port state Active) | +| `nvidia.ko` | `85879824254ecad9febdd0b6cc7944c9b21fe4ca310d3d91eb3ae9dbff147154`, srcversion `97587514A0900FB6CC5FF86` | +| Stock `libcuda` | SHA-256 `ba35b4baccf427f74f1b7c600297ae0cd4ff860f381aba65c3d0b88b8c5e95bc` | +| Patched `libcuda` | SHA-256 `f013ffac50fd6d9bf4d82142164889a81cc0bf4cde0741e0167826a1f3232c7a` | + +The validated module tree contained the FORCE_PCIE physical-address locking +change represented by this PR series (the "Serialize FORCE_PCIE DMA-BUF +physical-address operations" commit). The subsequent coherent-only ForceSPA +guard does not affect the tested configuration, since +`RmGpuDirectRdmaForceSPA` was not enabled during this test. +`NVreg_ExperimentalDmaBufP2P=1` is set in +`/etc/modprobe.d/nvidia-graphics-drivers.conf` and was active for this test +as confirmed through `/proc/driver/nvidia/params`. + +## Required CUDA userspace prerequisite + +The kernel branch is necessary on this GB206 system, but it is not sufficient +by itself. CUDA 13.3 in the stock 610.43.03 `libcuda` does not advertise the +DMA-BUF capability for these consumer GPUs. All successful `ib_write_bw` and +NCCL runs in this record used a private patched copy through: + +```sh +LD_LIBRARY_PATH=$HOME/libcuda-patched +``` + +The validation used a version-specific capability-gate modification applied +to a private copy of `libcuda.so.610.43.03`. The system CUDA library was not +modified. The userspace modification is outside the scope of this repository +and is not distributed here. The stock and modified SHA-256 values above, +together with the reproduction plan's provenance checks, identify the exact +libraries used. Without the userspace change, applications stop at CUDA's +consumer-device capability gate before reaching the kernel export path tested +here. This record therefore validates the combination of the experimental +kernel path and modified CUDA userspace, not the kernel branch in isolation +and not stock CUDA support. + +## Objective + +Validate that NCCL uses the GPUDirect RDMA (NET/IB) transport for same-host +communication between the two RTX 5060 Ti GPUs, and determine whether +collective traffic traverses the external Ethernet links or is completed +entirely within the ConnectX-6 Lx adapter. + +## Test configuration + +- Single host, two RTX 5060 Ti GPUs (non-coherent, static-BAR1-eligible). +- ConnectX-6 Lx dual-port 25 GbE RoCE adapter. +- GPU-to-NIC topology: `NODE` for both GPUs; the GPUs and NIC occupy separate + IOMMU groups and the importer uses an identity IOMMU domain. +- NCCL 2.30.7 with CUDA 13.3; nccl-tests 2.19.6 built with MPI + (`all_reduce_perf_mpi`). +- Two MPI ranks, one rank per GPU. +- `NCCL_P2P_DISABLE=1` to force the network transport instead of CUDA P2P. +- `NCCL_SHM_DISABLE=1`, `NCCL_DMABUF_ENABLE=1`, and + `NCCL_NET_GDR_LEVEL=SYS` to make the intended transport choice explicit. + +The recorded NCCL run used: + +```sh +export LD_LIBRARY_PATH=$HOME/libcuda-patched${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH} +export NCCL_DEBUG=INFO +export NCCL_DEBUG_SUBSYS=INIT,NET,GRAPH,P2P,SHM +export NCCL_P2P_DISABLE=1 +export NCCL_SHM_DISABLE=1 +export NCCL_IB_DISABLE=0 +export NCCL_DMABUF_ENABLE=1 +export NCCL_NET_GDR_LEVEL=SYS +export NCCL_SOCKET_IFNAME='=enp98s0f0np0' +export NCCL_IB_HCA='=mlx5_0:1,mlx5_1:1' +export CUDA_VISIBLE_DEVICES=0,1 + +mpirun -np 2 --bind-to none \ + -x CUDA_VISIBLE_DEVICES -x LD_LIBRARY_PATH -x NCCL_DEBUG -x NCCL_DEBUG_SUBSYS \ + -x NCCL_P2P_DISABLE -x NCCL_SHM_DISABLE -x NCCL_IB_DISABLE \ + -x NCCL_DMABUF_ENABLE -x NCCL_NET_GDR_LEVEL -x NCCL_SOCKET_IFNAME \ + -x NCCL_IB_HCA \ + ./build/all_reduce_perf_mpi -b 64M -e 1G -f 2 -g 1 +``` + +Both ranks saw both GPUs. The MPI-enabled nccl-tests binary assigned rank 0 to +GPU 0 and rank 1 to GPU 1; rank-specific `CUDA_VISIBLE_DEVICES` masking had +failed its local-GPU-count validation and was not used for the recorded run. + +## Transport selection + +NCCL selected the InfiniBand/RoCE transport for both ranks: + +``` +Using network IB +NET/IB/.../GDRDMA +GPU Direct RDMA Enabled +``` + +The communicator initialized using GPUDirect RDMA rather than CUDA IPC, +shared memory, or CUDA P2P. + +## GPUDirect RDMA validation + +Independent verification using `ib_write_bw --use_cuda_dmabuf` on both GPU +indices confirmed that the patched-userspace/experimental-kernel combination: + +- exported CUDA device memory through DMA-BUF, +- registered the exported memory with the mlx5 RDMA driver, and +- completed RDMA write bandwidth testing. + +This confirms GPUDirect RDMA is functional end-to-end on this platform through +the non-coherent FORCE_PCIE DMA-BUF export path added on this branch. The tests +used the same private `libcuda` directory described above. + +The generic `--use_cuda_dmabuf` path exercises the default DMA-BUF mapping. +The explicit `--use_cuda_pcie_mapping` option is required to request the BAR1 +`FORCE_PCIE` translation that is governed by the kernel's non-coherent gate. +On this host, reloading the NVIDIA modules with +`NVreg_ExperimentalDmaBufP2P=0` caused the explicit PCIe mapping run to fail +at `cuMemGetHandleForAddressRange` with CUDA error `801`, which is the expected +rejection for the non-coherent `FORCE_PCIE` path. + +The two same-host endpoints were started from the perftest build directory as: + +```sh +# Server, terminal 1 +LD_LIBRARY_PATH=$HOME/libcuda-patched \ + ./ib_write_bw -d mlx5_0 -i 1 -F --report_gbits \ + --use_cuda=0 --use_cuda_dmabuf + +# Client, terminal 2 +LD_LIBRARY_PATH=$HOME/libcuda-patched \ + ./ib_write_bw -d mlx5_1 -i 1 -F --report_gbits \ + --use_cuda=0 --use_cuda_dmabuf 10.200.0.1 +``` + +The pair was repeated with `--use_cuda=1` to exercise the other GPU. These are +same-host tests over the two active RoCE ports, not a multi-host result. + +## Internal forwarding validation + +To determine whether NCCL traffic traversed the physical Ethernet links, the +ConnectX-6 Lx IEEE 802.3 MAC counters (PPCNT, group 0) were sampled before and +after the NCCL workload. + +Observed counter deltas: + +``` +mlx5_0 + TX frames +4 + RX frames +4 + TX bytes +576 + RX bytes +576 +mlx5_1 + TX frames +4 + RX frames +4 + TX bytes +576 + RX bytes +576 +``` + +These counters measure frames that traverse the Ethernet MAC. The deltas are +consistent with control-plane traffic only (four small frames per port) and +are strong evidence that the NCCL collective payload did not traverse the +external 25 GbE interfaces. They do not, by themselves, constitute a formal +proof of every internal adapter datapath stage. + +## Conclusion + +- NCCL correctly selects the GPUDirect RDMA (NET/IB/GDRDMA) transport on this + platform. +- GPUDirect RDMA memory registration and RDMA communication operate correctly + through the non-coherent FORCE_PCIE DMA-BUF export path when combined with + the private CUDA userspace capability patch. +- Workload-scale RDMA activity combined with negligible Ethernet MAC activity + is consistent with, and provides strong evidence for, ConnectX-6 Lx + adapter-internal forwarding. Physical-wire counter calibration remains + required before making a categorical forwarding claim. + +Inferred data path: + +``` +GPU0 -> GPUDirect RDMA -> ConnectX-6 Lx internal forwarding -> GPUDirect RDMA -> GPU1 +``` + +rather than: + +``` +GPU0 -> 25 GbE Port 0 -> external Ethernet fabric -> 25 GbE Port 1 -> GPU1 +``` + +The measurements indicate that NCCL can exercise the GPUDirect RDMA transport +without workload-scale traffic appearing at the external Ethernet MACs. The +specific internal adapter stages, and VF-to-VF RoCE behavior in virtualized +deployments, remain follow-up validation rather than results established by +this record. + +## Health and teardown observations + +The validated runs completed successfully and both GPUs remained available to +`nvidia-smi` afterward. The post-run kernel-log inspection found no new NVIDIA +Xid, RM assertion, IOMMU fault, AER error, BAR1 failure, mlx5 reset, kernel +warning, or DMA-BUF cleanup warning attributable to the workload. The record +does not claim concurrent registration stress, suspend/resume, driver reload, +or reboot coverage; those remain part of the requested follow-up matrix. + +## Validation limits and requested follow-ups + +This is one implementation and one host configuration. It does not establish +hardware validation for non-GB206 GPUs, other NIC families, PCIe-switch +(`PIX`/`PXB`), same-host `PHB`, cross-socket `SYS`, translated-IOMMU, or +multi-host topologies. It also does not validate stock `libcuda`, GDS/cuFile, +or VF-to-VF operation. Those configurations should repeat direct DMA-BUF RDMA +registration, bidirectional data validation, NCCL transport inspection, +concurrent register/deregister stress, cleanup, and kernel-fault checks. + +The driver intentionally leaves `NVreg_ExperimentalDmaBufP2P` disabled by +default. A negative run with the option disabled should continue to reject the +non-coherent path, and a translated-IOMMU configuration should remain rejected +by the identity-domain gate. + +## Feature-gate configuration + +The experimental option may be supplied directly or through the driver's +aggregate registry string: + +- Module parameter: `NVreg_ExperimentalDmaBufP2P=1` +- Equivalent via the aggregate string: `NVreg_RegistryDwords="ExperimentalDmaBufP2P=1"` + +`/etc/modprobe.d/nvidia-graphics-drivers.conf` sets +`options nvidia NVreg_ExperimentalDmaBufP2P=1` using the correct name, and +that is what was in effect for both the `ib_write_bw` and NCCL runs recorded +here — `DMABUF_GDR_NONCOHERENT_ALLOWED()` requires the `enabled` term +unconditionally, so none of this testing would have succeeded otherwise. From 85e9d1d525ded51b10ef0b47b8362bb5e28a0339 Mon Sep 17 00:00:00 2001 From: "Duc P. Tran" <1055172+ducphuc@users.noreply.github.com> Date: Thu, 6 Aug 2026 23:53:13 -0700 Subject: [PATCH 21/21] Add same-host DMA-BUF GDR reproduction plan Provide a reproducible positive, negative, stress, teardown, and physical-wire calibration matrix for the default-off non-coherent DMA-BUF GDR path. Require module and patched-libcuda provenance, runtime topology gates, both GPUs, actual NCCL GDRDMA connectors, labeled counter evidence, and post-test kernel and GPU health. --- .../nccl-same-host-gdr-reproduction-plan.md | 1612 +++++++++++++++++ 1 file changed, 1612 insertions(+) create mode 100644 validation/nccl-same-host-gdr-reproduction-plan.md diff --git a/validation/nccl-same-host-gdr-reproduction-plan.md b/validation/nccl-same-host-gdr-reproduction-plan.md new file mode 100644 index 0000000000..11aa839854 --- /dev/null +++ b/validation/nccl-same-host-gdr-reproduction-plan.md @@ -0,0 +1,1612 @@ +# Same-Host DMA-BUF GPUDirect RDMA Reproduction Plan + +## Purpose + +Reproduce and independently validate the experimental same-host DMA-BUF GPUDirect RDMA path introduced by the driver change. + +The validation must establish five distinct properties: + +1. The intended experimental kernel modules are installed and loaded. +2. The experimental non-coherent DMA-BUF P2P path is enabled only when explicitly requested. +3. CUDA device memory can be exported through DMA-BUF and registered with mlx5 for RDMA without falling back to legacy pointer-based registration. +4. NCCL can establish and execute a two-rank `NET/IB/.../GDRDMA` collective between two GPUs on the same host. +5. The resulting same-host RDMA workload shows strong evidence of ConnectX internal forwarding rather than external Ethernet MAC/PHY traversal. + +The plan must preserve raw evidence sufficient for independent review and must include positive, negative, stress, teardown, and physical-wire calibration tests. + +--- + +## 1. Test Variables and Repository Locations + +Use absolute paths so that perftest, NCCL, driver, and evidence capture do not depend on the invoking shell's working directory. + +Run the command blocks in Bash. Enable pipeline failure propagation before +capturing any workload through `tee`, so a failed workload cannot be reported +as successful merely because `tee` exited successfully: + +```bash +set -o pipefail +``` + +Define: + +```bash +export DRIVER_SRC="$HOME/src/open-gpu-kernel-modules" +export PERFTEST_DIR="$HOME/src/perftest" +export NCCL_TESTS_DIR="$HOME/src/nccl-tests" +export PATCHED_LIBCUDA_DIR="$HOME/libcuda-patched" + +export OUT="$HOME/gdr-validation/$(date +%Y%m%d-%H%M%S)" +mkdir -p "$OUT" +``` + +Define expected binaries: + +```bash +export IB_WRITE_BW="$PERFTEST_DIR/ib_write_bw" +export NCCL_ALLREDUCE="$NCCL_TESTS_DIR/build/all_reduce_perf_mpi" +``` + +Do not proceed if either binary is missing: + +```bash +test -x "$IB_WRITE_BW" +test -x "$NCCL_ALLREDUCE" +``` + +Record these variables: + +```bash +env | grep -E \ +'^(DRIVER_SRC|PERFTEST_DIR|NCCL_TESTS_DIR|PATCHED_LIBCUDA_DIR|OUT|IB_WRITE_BW|NCCL_ALLREDUCE)=' \ +> "$OUT/test-paths.txt" +``` + +--- + +## 2. Preserve Repository and Build Provenance + +Record the exact source revisions used for every relevant repository. + +```bash +for repo in \ + "$DRIVER_SRC" \ + "$PERFTEST_DIR" \ + "$NCCL_TESTS_DIR" +do + { + echo "=== $repo ===" + git -C "$repo" status --short + git -C "$repo" rev-parse HEAD + git -C "$repo" branch --show-current + git -C "$repo" log -1 --oneline + } >> "$OUT/repository-state.txt" +done +``` + +For the driver tree, additionally preserve: + +```bash +git -C "$DRIVER_SRC" diff \ + > "$OUT/driver-working-tree.diff" + +git -C "$DRIVER_SRC" diff --cached \ + > "$OUT/driver-index.diff" +``` + +If validation is intended for a particular PR commit, record the expected commit hash separately and compare it against the checked-out tree. + +Do not claim reproduction of a specific PR state if the source revision is ambiguous or contains unrecorded modifications. + +--- + +## 3. Capture Baseline Platform State + +Before loading, reloading, or exercising the experimental driver path, capture: + +```bash +uname -a > "$OUT/uname.txt" + +cat /proc/cmdline > "$OUT/kernel-cmdline.txt" + +lspci -nnk > "$OUT/lspci-nnk.txt" +lspci -tv > "$OUT/lspci-tree.txt" + +nvidia-smi -q > "$OUT/nvidia-smi-q-before.txt" +nvidia-smi topo -m > "$OUT/nvidia-smi-topo.txt" + +nvidia-smi \ + --query-gpu=index,name,pci.bus_id,memory.total,memory.used \ + --format=csv,noheader \ + > "$OUT/gpu-state-before.txt" + +ibv_devices > "$OUT/ibv-devices.txt" +ibv_devinfo > "$OUT/ibv-devinfo.txt" +ibdev2netdev > "$OUT/ibdev2netdev.txt" + +ip -br addr > "$OUT/ip-addresses.txt" +ip -br link > "$OUT/ip-links.txt" + +sudo mst status -v > "$OUT/mst-status.txt" + +sudo devlink port show \ + > "$OUT/devlink-ports.txt" + +sudo devlink dev show \ + > "$OUT/devlink-devices.txt" + +sudo dmesg -T > "$OUT/dmesg-before.txt" +``` + +Record software versions: + +```bash +nvidia-smi > "$OUT/nvidia-smi-version.txt" + +nvcc --version \ + > "$OUT/nvcc-version.txt" 2>&1 || true + +git -C "$NCCL_TESTS_DIR" describe --tags --always --dirty \ + > "$OUT/nccl-tests-version.txt" + +mpirun --version \ + > "$OUT/mpi-version.txt" 2>&1 || true + +"$IB_WRITE_BW" --version \ + > "$OUT/perftest-version.txt" 2>&1 || true + +mst version \ + > "$OUT/mft-version.txt" 2>&1 || true +``` + +--- + +## 4. Discover Devices Instead of Hard-Coding Them + +The procedure must discover and then explicitly record: + +- GPU 0 PCI BDF +- GPU 1 PCI BDF +- RDMA device A +- RDMA device B +- corresponding Ethernet interfaces +- corresponding MST devices +- NIC PCI functions +- RoCE IP addresses + +Reference-system values may be recorded in examples, but the executable procedure must not depend on them. + +Example discovery: + +```bash +nvidia-smi \ + --query-gpu=index,pci.bus_id,name \ + --format=csv,noheader \ + | tee "$OUT/gpu-pci-map.txt" + +ibdev2netdev \ + | tee "$OUT/rdma-netdev-map.txt" + +sudo mst status -v \ + | tee "$OUT/mst-map.txt" +``` + +After discovery, define explicit environment variables: + +```bash +export GPU0=0 +export GPU1=1 + +export RDMA0="mlx5_0" +export RDMA1="mlx5_1" + +export NETDEV0="enp98s0f0np0" +export NETDEV1="enp98s0f1np1" + +export MST0="/dev/mst/mt4127_pciconf0" +export MST1="/dev/mst/mt4127_pciconf0.1" + +export NIC_BDF0="0000:62:00.0" +export NIC_BDF1="0000:62:00.1" + +export ROCE_IP0="10.200.0.1" +export ROCE_IP1="10.200.0.2" +``` + +Those values are examples from the reference host. A reproducing agent must derive and set the local equivalents. + +Record the resolved values: + +```bash +env | grep -E \ +'^(GPU[01]|RDMA[01]|NETDEV[01]|MST[01]|NIC_BDF[01]|ROCE_IP[01])=' \ +> "$OUT/resolved-devices.txt" +``` + +--- + +## 5. Verify the Experimental Kernel Module Provenance + +This is a driver-validation prerequisite, not optional metadata. + +Record loaded NVIDIA module information: + +```bash +for mod in nvidia nvidia_modeset nvidia_drm nvidia_uvm nvidia_peermem; do + { + echo "=== $mod ===" + modinfo "$mod" 2>&1 || true + if [ -r "/sys/module/$mod/srcversion" ]; then + echo -n "loaded srcversion: " + cat "/sys/module/$mod/srcversion" + fi + } >> "$OUT/module-provenance.txt" +done +``` + +Record hashes of installed module files: + +```bash +for mod in nvidia nvidia_modeset nvidia_drm nvidia_uvm nvidia_peermem; do + path=$(modinfo -n "$mod" 2>/dev/null || true) + if [ -n "$path" ] && [ -f "$path" ]; then + sha256sum "$path" + fi +done > "$OUT/module-sha256.txt" +``` + +Record Secure Boot signer metadata: + +```bash +for mod in nvidia nvidia_modeset nvidia_drm nvidia_uvm nvidia_peermem; do + { + echo "=== $mod ===" + modinfo -F signer "$mod" 2>/dev/null || true + modinfo -F sig_key "$mod" 2>/dev/null || true + modinfo -F sig_hashalgo "$mod" 2>/dev/null || true + } +done > "$OUT/module-signatures.txt" +``` + +The reproducing agent must verify that the loaded module `srcversion` and installed hashes correspond to the intended experimental build. + +Do not proceed with the positive validation if the module identity is ambiguous. + +--- + +## 6. Verify the Feature Gate + +Require: + +```bash +grep '^ExperimentalDmaBufP2P:' /proc/driver/nvidia/params \ + | tee "$OUT/experimental-dmabuf-p2p.txt" +``` + +Positive testing requires: + +```text +ExperimentalDmaBufP2P: 1 +``` + +If it is `0`, positive-path testing must not proceed. + +Record the entire NVIDIA parameter set: + +```bash +cat /proc/driver/nvidia/params \ + > "$OUT/nvidia-params.txt" +``` + +The explicit feature gate is part of the validation contract. A successful transfer obtained through another configuration does not validate the experimental path. + +--- + +## 7. Verify BAR1 and Topology Preconditions + +Record BAR1 state for every GPU: + +```bash +nvidia-smi -q -d MEMORY \ + > "$OUT/nvidia-memory-details.txt" +``` + +Capture any driver-specific BAR1 geometry/topology diagnostics added by the PR. + +The validation must establish: + +- static BAR1 space required by the experimental path is available; +- the selected GPU and NIC satisfy the driver's topology acceptance criteria; +- the selected GPU and NIC are not in an unsupported IOMMU relationship; +- the driver accepts the relevant identity-domain or topology condition. + +Record IOMMU groups: + +```bash +for bdf in \ + "$(nvidia-smi --query-gpu=pci.bus_id --format=csv,noheader | sed -n '1p')" \ + "$(nvidia-smi --query-gpu=pci.bus_id --format=csv,noheader | sed -n '2p')" \ + "$NIC_BDF0" \ + "$NIC_BDF1" +do + bdf=${bdf#00000000:} + bdf=${bdf#0000:} + + dev="/sys/bus/pci/devices/0000:$bdf" + + echo "=== 0000:$bdf ===" + readlink -f "$dev/iommu_group" 2>/dev/null || true +done > "$OUT/iommu-groups.txt" +``` + +Capture IOMMU kernel messages: + +```bash +sudo dmesg -T | grep -Ei \ +'iommu|amd-vi|dmar' \ +> "$OUT/iommu-kernel-log.txt" +``` + +IOMMU-group membership alone does not establish the active domain type. During +the first successful DMA-BUF attachment, require and preserve the driver's +runtime topology decision: + +```bash +sudo dmesg -T | grep 'DMA-BUF GDR topology:' \ + > "$OUT/dmabuf-gdr-topology.txt" +``` + +For every importer used by the positive test, require a corresponding accepted +decision containing values equivalent to: + +```text +identityIommu=1 +p2pDistance= +barAddressable=1 +skipIommu=0 +result=1 +``` + +Capture the diagnostic after the positive attachment if it is not present yet +at this precondition stage. A missing accepted runtime decision makes the +topology result incomplete. + +A reproduction cannot be considered equivalent if it bypasses topology checks that are part of the feature's safety boundary. + +--- + +## 8. Verify No Unexpected `nvidia-peermem` Fallback + +Record whether `nvidia_peermem` is loaded: + +```bash +lsmod | grep -E '^nvidia_peermem\b' \ + > "$OUT/nvidia-peermem-loaded.txt" || true +``` + +Record module parameters if present: + +```bash +find /sys/module/nvidia_peermem/parameters \ + -maxdepth 1 -type f -print -exec cat {} \; \ + > "$OUT/nvidia-peermem-params.txt" 2>&1 || true +``` + +Positive DMA-BUF validation must establish that the successful registration used DMA-BUF, not the legacy `nvidia-peermem` pointer-registration path. + +If necessary, run a controlled validation with `nvidia_peermem` unloaded when system configuration permits it, provided doing so does not destabilize display or other required services. + +The test must not infer DMA-BUF usage merely from transfer success. + +--- + +## 9. Validate Patched `libcuda` Provenance + +The private CUDA userspace library is part of the experiment and must be independently identified. + +Record all files in the private library directory: + +```bash +find "$PATCHED_LIBCUDA_DIR" \ + -maxdepth 1 \( -type f -o -type l \) \ + -ls \ + > "$OUT/patched-libcuda-files.txt" +``` + +Hash relevant library files: + +```bash +find "$PATCHED_LIBCUDA_DIR" \ + -maxdepth 1 -type f \ + -name 'libcuda.so*' \ + -exec sha256sum {} \; \ + > "$OUT/patched-libcuda-sha256.txt" +``` + +Record stock CUDA library resolution and hashes: + +```bash +ldconfig -p | grep 'libcuda.so' \ + > "$OUT/system-libcuda-resolution.txt" + +while read -r path; do + [ -f "$path" ] && sha256sum "$path" +done < <( + ldconfig -p | + awk '/libcuda\.so/{print $NF}' | + sort -u +) > "$OUT/system-libcuda-sha256.txt" +``` + +Preserve the userspace patch manifest or diff: + +```bash +find "$PATCHED_LIBCUDA_DIR" -maxdepth 1 -type f \ + \( -name '*patch*.json' -o -name 'PATCH*' \) \ + -exec cp -- {} "$OUT/" \; +``` + +Record the expected source driver version. For the reference validation, the patched library must correspond to the reviewed 610.43.03 userspace implementation. + +Verify that the system library itself has not been replaced by the experiment. + +--- + +## 10. Verify Runtime Loading of the Private `libcuda` + +Setting `LD_LIBRARY_PATH` is insufficient evidence. + +Use: + +```bash +export LD_LIBRARY_PATH="$PATCHED_LIBCUDA_DIR${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" +``` + +For at least one paired perftest run, capture dynamic-loader resolution on the +server. This command waits for a client; run the matching Section 12 client in +a second terminal rather than invoking it alone: + +```bash +LD_DEBUG=libs \ +"$IB_WRITE_BW" \ + -d "$RDMA0" \ + -i 1 \ + -F \ + --report_gbits \ + --use_cuda="$GPU0" \ + --use_cuda_dmabuf \ + > "$OUT/perftest-loader-run.txt" \ + 2> "$OUT/perftest-loader-debug.txt" + +loader_status=$? +printf '%s\n' "$loader_status" > "$OUT/perftest-loader-status.txt" +test "$loader_status" -eq 0 +``` + +If practical, also inspect the live process: + +```bash +grep -E 'libcuda\.so' /proc//maps +``` + +and preserve that output. + +The acceptance criterion is that the perftest and NCCL processes map the private patched `libcuda.so`, not the stock system library. + +--- + +## 11. Verify Both GPUs and RoCE Interfaces + +Require at least two visible GPUs: + +```bash +nvidia-smi -L \ + | tee "$OUT/gpu-list.txt" +``` + +Require both RoCE ports to be active: + +```bash +for dev in "$NETDEV0" "$NETDEV1"; do + ethtool "$dev" +done > "$OUT/ethernet-link-state.txt" +``` + +For the reference hardware, expected link state is: + +```text +Speed: 25000Mb/s +Link detected: yes +``` + +Record RDMA state: + +```bash +ibv_devinfo -d "$RDMA0" \ + > "$OUT/${RDMA0}-devinfo.txt" + +ibv_devinfo -d "$RDMA1" \ + > "$OUT/${RDMA1}-devinfo.txt" +``` + +--- + +## 12. Validate DMA-BUF GPUDirect RDMA with `perftest` + +This is the low-level positive-path validation. + +### Server + +Run: + +```bash +set -o pipefail +cd "$PERFTEST_DIR" + +LD_LIBRARY_PATH="$PATCHED_LIBCUDA_DIR${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" \ +"$IB_WRITE_BW" \ + -d "$RDMA0" \ + -i 1 \ + -F \ + --report_gbits \ + --use_cuda="$GPU0" \ + --use_cuda_dmabuf \ + 2>&1 | tee "$OUT/ib-write-server.txt" + +server_status=${PIPESTATUS[0]} +printf '%s\n' "$server_status" > "$OUT/ib-write-server.status" +test "$server_status" -eq 0 +``` + +### Client + +In a second terminal: + +```bash +set -o pipefail +cd "$PERFTEST_DIR" + +LD_LIBRARY_PATH="$PATCHED_LIBCUDA_DIR${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" \ +"$IB_WRITE_BW" \ + -d "$RDMA1" \ + -i 1 \ + -F \ + --report_gbits \ + --use_cuda="$GPU0" \ + --use_cuda_dmabuf \ + "$ROCE_IP0" \ + 2>&1 | tee "$OUT/ib-write-client.txt" + +client_status=${PIPESTATUS[0]} +printf '%s\n' "$client_status" > "$OUT/ib-write-client.status" +test "$client_status" -eq 0 +``` + +The exact GPU assignment may be varied later for topology coverage. + +### Acceptance criteria + +Require all of the following: + +1. CUDA device-memory allocation succeeds. +2. CUDA DMA-BUF FD export succeeds. +3. A DMA-BUF registration API succeeds: + - `ibv_reg_dmabuf_mr`, or + - a DMA-BUF-aware `ibv_reg_mr_ex` path. +4. Registration does not fall back to ordinary pointer-based `ibv_reg_mr`. +5. RDMA transfer completes successfully. +6. The test produces plausible bandwidth. + +Do not require a specific verbs function name if perftest implementation differences permit equivalent DMA-BUF registration paths. + +Preserve the complete server and client logs. + +Repeat the server/client pair with `--use_cuda="$GPU1"` on both endpoints and +write to distinct `ib-write-gpu1-*.txt` logs. Positive validation requires +successful DMA-BUF registration and transfer from both GPUs; a single-GPU pass +is incomplete for the two-GPU NCCL configuration. + +--- + +## 13. Verify the MPI-Enabled NCCL Binary + +Use only: + +```bash +"$NCCL_ALLREDUCE" +``` + +Verify linkage: + +```bash +ldd "$NCCL_ALLREDUCE" \ + | tee "$OUT/nccl-allreduce-ldd.txt" + +ldd "$NCCL_ALLREDUCE" \ + | grep -i libmpi \ + > "$OUT/nccl-mpi-linkage.txt" +``` + +Verify MPI: + +```bash +set -o pipefail +mpirun -np 2 \ + bash -c ' + echo "rank=$OMPI_COMM_WORLD_RANK size=$OMPI_COMM_WORLD_SIZE pid=$$" + ' \ + | tee "$OUT/mpi-rank-check.txt" + +mpi_status=${PIPESTATUS[0]} +printf '%s\n' "$mpi_status" > "$OUT/mpi-rank-check.status" +test "$mpi_status" -eq 0 +``` + +Require: + +```text +rank=0 size=2 +rank=1 size=2 +``` + +--- + +## 14. GPU Visibility for `nccl-tests` + +Expose both GPUs to both MPI processes: + +```bash +export CUDA_VISIBLE_DEVICES=0,1 +``` + +Do **not** use: + +```bash +export CUDA_VISIBLE_DEVICES=$OMPI_COMM_WORLD_LOCAL_RANK +``` + +for this MPI-enabled `nccl-tests` configuration. + +That rank-specific masking was experimentally shown to fail because each MPI process sees only one GPU while `nccl-tests` validates the total local GPU requirement for the two local MPI ranks: + +```text +Invalid number of GPUs: 2 requested but only 1 were found. +``` + +The successful configuration exposes both GPUs and allows `all_reduce_perf_mpi` to assign: + +```text +Rank 0 → GPU 0 +Rank 1 → GPU 1 +``` + +internally. + +This assignment must be confirmed from the NCCL log rather than assumed. + +--- + +## 15. Configure NCCL to Exercise the Network Path + +Set: + +```bash +export LD_LIBRARY_PATH="$PATCHED_LIBCUDA_DIR${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" + +export NCCL_DEBUG=INFO +export NCCL_DEBUG_SUBSYS=INIT,NET,GRAPH,P2P,SHM + +export NCCL_P2P_DISABLE=1 +export NCCL_SHM_DISABLE=1 + +export NCCL_IB_DISABLE=0 +export NCCL_DMABUF_ENABLE=1 +export NCCL_NET_GDR_LEVEL=SYS + +export NCCL_IB_HCA="=${RDMA0}:1,${RDMA1}:1" +export NCCL_SOCKET_IFNAME="=$NETDEV0" +``` + +The intention is to prevent ordinary same-host CUDA P2P and SHM transport from satisfying the collective. + +Record all relevant environment variables: + +```bash +env | grep -E \ +'^(CUDA_VISIBLE_DEVICES|LD_LIBRARY_PATH|NCCL_)' \ +> "$OUT/nccl-environment.txt" +``` + +--- + +## 16. Capture RDMA Counters Before NCCL + +Use labeled output. + +```bash +capture_rdma_counters() { + for dev in "$RDMA0" "$RDMA1"; do + for counter in \ + port_xmit_data \ + port_rcv_data \ + port_xmit_packets \ + port_rcv_packets + do + path="/sys/class/infiniband/$dev/ports/1/counters/$counter" + + if [ -r "$path" ]; then + printf '%s %-24s %s\n' \ + "$dev" \ + "$counter" \ + "$(cat "$path")" + fi + done + done +} + +capture_rdma_counters \ + > "$OUT/rdma-counters-before.txt" +``` + +Also capture available RoCE hardware counters: + +```bash +capture_roce_hw_counters() { + for dev in "$RDMA0" "$RDMA1"; do + for counter in \ + rx_write_requests \ + rx_read_requests \ + rx_atomic_requests \ + req_transport_retries_exceeded \ + req_rnr_retries_exceeded \ + local_ack_timeout_err \ + packet_seq_err \ + out_of_sequence \ + roce_adp_retrans \ + roce_adp_retrans_to + do + path="/sys/class/infiniband/$dev/ports/1/hw_counters/$counter" + + if [ -r "$path" ]; then + printf '%s %-36s %s\n' \ + "$dev" \ + "$counter" \ + "$(cat "$path")" + fi + done + done +} + +capture_roce_hw_counters \ + > "$OUT/rdma-hw-counters-before.txt" +``` + +--- + +## 17. Capture Firmware IEEE 802.3 MAC Counters Before NCCL + +Use the ConnectX `PPCNT` register through MFT. + +First verify the register layout: + +```bash +sudo mlxreg \ + -d "$MST0" \ + --show_reg PPCNT \ + > "$OUT/ppcnt-definition.txt" +``` + +Require that `grp=0` exposes IEEE 802.3 counters including: + +```text +a_frames_transmitted_ok +a_frames_received_ok +a_octets_transmitted_ok +a_octets_received_ok +``` + +Capture both functions: + +```bash +sudo mlxreg \ + -d "$MST0" \ + --reg_name PPCNT \ + --get \ + --indexes "local_port=1,grp=0" \ + > "$OUT/ppcnt-port0-before.txt" + +sudo mlxreg \ + -d "$MST1" \ + --reg_name PPCNT \ + --get \ + --indexes "local_port=1,grp=0" \ + > "$OUT/ppcnt-port1-before.txt" +``` + +Relevant 64-bit values are: + +```text +a_frames_transmitted_ok = (high << 32) | low +a_frames_received_ok = (high << 32) | low + +a_octets_transmitted_ok = (high << 32) | low +a_octets_received_ok = (high << 32) | low +``` + +These counters form the principal MAC-level evidence for determining whether workload-scale traffic reaches the external Ethernet MAC. + +--- + +## 18. Capture Kernel and GPU Health Baseline Immediately Before Workload + +Record: + +```bash +sudo dmesg -T \ + > "$OUT/dmesg-pre-nccl.txt" + +nvidia-smi \ + --query-gpu=index,memory.used,utilization.gpu,temperature.gpu \ + --format=csv,noheader \ + > "$OUT/gpu-health-pre-nccl.txt" +``` + +Capture any PR-specific diagnostics relevant to: + +- BAR1 mapping state +- DMA-BUF registration state +- active mappings +- topology classification +- FORCE_PCIE or equivalent internal path state + +--- + +## 19. Run the Two-Rank NCCL Collective + +Run: + +```bash +set -o pipefail +cd "$NCCL_TESTS_DIR" + +mpirun -np 2 \ + --bind-to none \ + -x CUDA_VISIBLE_DEVICES \ + -x LD_LIBRARY_PATH \ + -x NCCL_DEBUG \ + -x NCCL_DEBUG_SUBSYS \ + -x NCCL_P2P_DISABLE \ + -x NCCL_SHM_DISABLE \ + -x NCCL_IB_DISABLE \ + -x NCCL_DMABUF_ENABLE \ + -x NCCL_NET_GDR_LEVEL \ + -x NCCL_IB_HCA \ + -x NCCL_SOCKET_IFNAME \ + "$NCCL_ALLREDUCE" \ + -b 64M \ + -e 1G \ + -f 2 \ + -g 1 \ + 2>&1 | tee "$OUT/nccl-mpi.log" + +nccl_status=${PIPESTATUS[0]} +printf '%s\n' "$nccl_status" > "$OUT/nccl-mpi.status" +test "$nccl_status" -eq 0 +``` + +Require successful completion. + +--- + +## 20. Verify the NCCL Communicator and GPU Assignment + +Extract: + +```bash +grep -E \ +'Rank|nranks|nRanks|nNodes|localRanks|Channel .*0.*1|Using network IB|GPU Direct RDMA Enabled|GDRDMA' \ +"$OUT/nccl-mpi.log" \ +> "$OUT/nccl-validation-lines.txt" +``` + +Require: + +```text +Rank 0 ... device 0 +Rank 1 ... device 1 +``` + +and: + +```text +rank 0 nranks 2 +rank 1 nranks 2 +``` + +and channel topology containing both ranks: + +```text +Channel ... : 0 1 +``` + +Reject: + +- two independent rank-0 communicators; +- both ranks mapped to the same GPU; +- single-rank operation. + +--- + +## 21. Verify the Actual NCCL Transport + +Backend initialization alone is insufficient. + +Require established connector lines equivalent to: + +```text +0[0] -> 1[1] ... via NET/IB/.../GDRDMA +1[1] -> 0[0] ... via NET/IB/.../GDRDMA +``` + +Also preserve: + +```text +Using network IB +GPU Direct RDMA Enabled +``` + +Record any aggregated network device selected by NCCL, such as: + +```text +mlx5_0+mlx5_1 +``` + +Acceptance requires actual `GDRDMA` channel connectors, not merely discovery of a GDR-capable HCA. + +--- + +## 22. Capture RDMA and MAC Counters After NCCL + +Immediately repeat the RDMA captures: + +```bash +capture_rdma_counters \ + > "$OUT/rdma-counters-after.txt" + +capture_roce_hw_counters \ + > "$OUT/rdma-hw-counters-after.txt" +``` + +Capture `PPCNT grp=0`: + +```bash +sudo mlxreg \ + -d "$MST0" \ + --reg_name PPCNT \ + --get \ + --indexes "local_port=1,grp=0" \ + > "$OUT/ppcnt-port0-after.txt" + +sudo mlxreg \ + -d "$MST1" \ + --reg_name PPCNT \ + --get \ + --indexes "local_port=1,grp=0" \ + > "$OUT/ppcnt-port1-after.txt" +``` + +Calculate and preserve numeric deltas. + +The reference run observed: + +```text +mlx5_0: + a_frames_transmitted_ok +4 + a_frames_received_ok +4 + a_octets_transmitted_ok +576 + a_octets_received_ok +576 + +mlx5_1: + a_frames_transmitted_ok +4 + a_frames_received_ok +4 + a_octets_transmitted_ok +576 + a_octets_received_ok +576 +``` + +Exact values are not an acceptance requirement. The criterion is that MAC traffic remains negligible relative to the NCCL workload while RDMA activity increases materially. + +--- + +## 23. Required Physical-Wire Counter Calibration + +This control is required if the final report makes a statement about internal forwarding. + +Use a documented physical path: either two hosts connected through the tested +ports or a verified external loop/switch path between the two ports. Do not use +this control when the route could be satisfied internally without traversing +the MAC/PHY. + +On the single-host two-port reference configuration, capture `PPCNT grp=0`, run +a bounded port-to-port transfer, and capture the counters again. Use a control +port distinct from other perftest processes: + +```bash +export WIRE_CONTROL_PORT=18525 + +sudo mlxreg -d "$MST0" --reg_name PPCNT --get \ + --indexes "local_port=1,grp=0" \ + > "$OUT/wire-ppcnt-port0-before.txt" +sudo mlxreg -d "$MST1" --reg_name PPCNT --get \ + --indexes "local_port=1,grp=0" \ + > "$OUT/wire-ppcnt-port1-before.txt" + +LD_LIBRARY_PATH="$PATCHED_LIBCUDA_DIR${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" \ +timeout 45s "$IB_WRITE_BW" -d "$RDMA0" -i 1 -F --report_gbits -D 10 \ + -p "$WIRE_CONTROL_PORT" --use_cuda="$GPU0" --use_cuda_dmabuf \ + > "$OUT/wire-server.txt" 2>&1 & +wire_server_pid=$! + +sleep 1 + +LD_LIBRARY_PATH="$PATCHED_LIBCUDA_DIR${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" \ +timeout 45s "$IB_WRITE_BW" -d "$RDMA1" -i 1 -F --report_gbits -D 10 \ + -p "$WIRE_CONTROL_PORT" --use_cuda="$GPU0" --use_cuda_dmabuf \ + "$ROCE_IP0" > "$OUT/wire-client.txt" 2>&1 +wire_client_status=$? + +wait "$wire_server_pid" +wire_server_status=$? + +printf 'server=%s\nclient=%s\n' \ + "$wire_server_status" "$wire_client_status" \ + > "$OUT/wire-control.status" +test "$wire_server_status" -eq 0 +test "$wire_client_status" -eq 0 + +sudo mlxreg -d "$MST0" --reg_name PPCNT --get \ + --indexes "local_port=1,grp=0" \ + > "$OUT/wire-ppcnt-port0-after.txt" +sudo mlxreg -d "$MST1" --reg_name PPCNT --get \ + --indexes "local_port=1,grp=0" \ + > "$OUT/wire-ppcnt-port1-after.txt" +``` + +Decode the four 64-bit counters using the same high/low-word calculation as +Section 17. Preserve the numeric before, after, and delta values in +`$OUT/wire-ppcnt-deltas.txt`. + +Expected physical-wire behavior: + +```text +MST0 TX octets ≈ MST1 RX octets +MST1 TX octets ≈ MST0 RX octets +``` + +with increases commensurate with the transfer volume. + +This establishes experimentally that: + +1. the selected `PPCNT` counters do respond to actual external Ethernet traffic; and +2. the near-zero MAC deltas observed during same-host NCCL are not caused by a dead or irrelevant counter source. + +The evidentiary comparison is: + +```text +Physical-wire control: + large MAC counter deltas + +Same-host NCCL GDR: + large RDMA activity + negligible MAC counter deltas +``` + +This differential result provides strong evidence for adapter-internal forwarding. + +--- + +## 24. Feature-Gate Negative Control + +This is the primary negative control for the driver change. + +Disable: + +```text +NVreg_ExperimentalDmaBufP2P=0 +``` + +using the supported module reload or reboot procedure for the test system. + +After reload/reboot, require: + +```bash +grep '^ExperimentalDmaBufP2P:' /proc/driver/nvidia/params +``` + +to report: + +```text +ExperimentalDmaBufP2P: 0 +``` + +Repeat the low-level DMA-BUF registration test. + +The expected result is that the experimental non-coherent path is unavailable or rejected. + +Acceptance criteria: + +- the new experimental path does not become usable when the feature gate is disabled; +- failure occurs before successful RDMA use of the prohibited mapping; +- no crash, Xid, IOMMU fault, BAR1 corruption, leaked registration, or other unsafe behavior occurs. + +Do not require a specific errno unless the driver ABI explicitly guarantees one. + +Restore: + +```text +NVreg_ExperimentalDmaBufP2P=1 +``` + +before subsequent positive or stress tests. + +--- + +## 25. NCCL Transport Negative Controls + +These are secondary controls. Run each in an isolated subshell restored from +the known-positive environment, and write each result to a distinct log. Do +not carry a variable changed by one control into the next. + +Define a reusable bounded workload: + +```bash +run_nccl_control() { + local label=$1 + set -o pipefail + + mpirun -np 2 --bind-to none \ + -x CUDA_VISIBLE_DEVICES -x LD_LIBRARY_PATH \ + -x NCCL_DEBUG -x NCCL_DEBUG_SUBSYS \ + -x NCCL_P2P_DISABLE -x NCCL_SHM_DISABLE \ + -x NCCL_IB_DISABLE -x NCCL_DMABUF_ENABLE \ + -x NCCL_NET_GDR_LEVEL -x NCCL_IB_HCA \ + -x NCCL_SOCKET_IFNAME \ + "$NCCL_ALLREDUCE" -b 64M -e 256M -f 2 -g 1 \ + 2>&1 | tee "$OUT/nccl-${label}.log" + + local status=${PIPESTATUS[0]} + printf '%s\n' "$status" > "$OUT/nccl-${label}.status" + return "$status" +} + +set_positive_nccl_environment() { + export CUDA_VISIBLE_DEVICES=0,1 + export NCCL_P2P_DISABLE=1 + export NCCL_SHM_DISABLE=1 + export NCCL_IB_DISABLE=0 + export NCCL_DMABUF_ENABLE=1 + export NCCL_NET_GDR_LEVEL=SYS + export NCCL_IB_HCA="=${RDMA0}:1,${RDMA1}:1" + export NCCL_SOCKET_IFNAME="=$NETDEV0" +} +``` + +### P2P allowed + +Run with only CUDA P2P restored: + +```bash +( + set_positive_nccl_environment + export NCCL_P2P_DISABLE=0 + run_nccl_control p2p-allowed +) +``` + +Expected outcome: + +- NCCL may select CUDA P2P for the same-host topology; +- `NET/IB/.../GDRDMA` is no longer required. + +### IB disabled + +Run with only the IB backend disabled: + +```bash +( + set_positive_nccl_environment + export NCCL_IB_DISABLE=1 + run_nccl_control ib-disabled +) +``` + +Expected outcome: + +- no `NET/IB/.../GDRDMA` connectors. + +### DMA-BUF disabled + +Run with only DMA-BUF disabled: + +```bash +( + set_positive_nccl_environment + export NCCL_DMABUF_ENABLE=0 + run_nccl_control dmabuf-disabled +) +``` + +Interpret the result carefully because NCCL may have another supported registration path. + +Do not describe a resulting successful collective as DMA-BUF GDR unless the registration mechanism is independently established. + +--- + +## 26. Registration/Deregistration Stress + +Because the driver change modifies DMA/BAR1 mapping behavior and the relevant locking path, execute repeated registration and teardown under concurrency. + +At minimum: + +- multiple processes; +- repeated CUDA allocation; +- DMA-BUF export; +- RDMA registration; +- transfer; +- deregistration; +- FD close; +- CUDA free. + +Run enough iterations to exercise overlapping registration and deregistration. + +Example conceptual stress matrix: + +```text +1 process × 1 GPU × repeated registration +2 processes × same GPU +2 processes × different GPUs +concurrent register/deregister +rapid teardown/recreate +``` + +The following bounded reference loop exercises overlapping registrations on +both GPUs. Adjust the duration or round count upward only after this baseline +passes. Each pair uses a distinct TCP control port and preserves separate +endpoint logs: + +```bash +export STRESS_ROUNDS=20 +: > "$OUT/stress-failures.txt" + +run_stress_pair() { + local gpu=$1 + local port=$2 + local label=$3 + + LD_LIBRARY_PATH="$PATCHED_LIBCUDA_DIR${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" \ + timeout 45s "$IB_WRITE_BW" -d "$RDMA0" -i 1 -F --report_gbits \ + -D 5 -p "$port" --use_cuda="$gpu" --use_cuda_dmabuf \ + > "$OUT/stress-${label}-server.log" 2>&1 & + local server_pid=$! + + sleep 1 + + LD_LIBRARY_PATH="$PATCHED_LIBCUDA_DIR${LD_LIBRARY_PATH:+:$LD_LIBRARY_PATH}" \ + timeout 45s "$IB_WRITE_BW" -d "$RDMA1" -i 1 -F --report_gbits \ + -D 5 -p "$port" --use_cuda="$gpu" --use_cuda_dmabuf \ + "$ROCE_IP0" > "$OUT/stress-${label}-client.log" 2>&1 + local client_status=$? + + wait "$server_pid" + local server_status=$? + + printf 'server=%s\nclient=%s\n' \ + "$server_status" "$client_status" \ + > "$OUT/stress-${label}.status" + + if [ "$server_status" -ne 0 ] || [ "$client_status" -ne 0 ]; then + printf '%s server=%s client=%s\n' \ + "$label" "$server_status" "$client_status" \ + >> "$OUT/stress-failures.txt" + return 1 + fi +} + +for round in $(seq 1 "$STRESS_ROUNDS"); do + port0=$((19000 + round * 2)) + port1=$((port0 + 1)) + + run_stress_pair "$GPU0" "$port0" "r${round}-gpu0" & + stress_pid0=$! + run_stress_pair "$GPU1" "$port1" "r${round}-gpu1" & + stress_pid1=$! + + wait "$stress_pid0" || true + wait "$stress_pid1" || true +done + +test ! -s "$OUT/stress-failures.txt" +``` + +This provides a minimum of 40 overlapping registration/transfer/teardown +cycles on the two-GPU reference host. Record any host-specific reduction in +parallelism or iteration count rather than silently changing the matrix. + +Where practical, include both NIC functions. + +Preserve: + +- process exit status; +- iteration count; +- kernel log; +- GPU memory before/after; +- BAR1 diagnostics before/after. + +Any deadlock, refcount leak, stale mapping, BAR1 exhaustion, use-after-free warning, GPU reset, mlx5 reset, or IOMMU fault is a failure. + +--- + +## 27. Post-Test Teardown and Resource Validation + +After all workloads complete, verify that no registrations remain unexpectedly retained. + +Capture: + +```bash +nvidia-smi \ + --query-gpu=index,memory.used,utilization.gpu,temperature.gpu \ + --format=csv,noheader \ + > "$OUT/gpu-health-after.txt" + +nvidia-smi -q \ + > "$OUT/nvidia-smi-q-after.txt" + +sudo dmesg -T \ + > "$OUT/dmesg-after.txt" +``` + +Compare GPU memory usage against the baseline. + +Small runtime allocator differences may be acceptable, but persistent workload-sized memory retention requires investigation. + +Capture any driver diagnostics exposing: + +- outstanding DMA-BUF mappings; +- BAR1 mappings; +- registration counts; +- peer mappings; +- cleanup/refcount state. + +--- + +## 28. Kernel Health Audit + +Generate a focused before/after kernel-log report. + +Search for: + +```bash +grep -Ei \ +'Xid|NVRM|assert|BAR1|dma.?buf|IOMMU|AMD-Vi|DMAR|AER|PCIe.*error|mlx5.*(error|reset|fatal)|page fault|use-after-free|refcount|WARN|BUG|Call Trace' \ +"$OUT/dmesg-after.txt" \ +> "$OUT/kernel-health-findings.txt" +``` + +Compare against the pre-test log so pre-existing messages are not misclassified as test regressions. + +A successful validation requires no new unexpected: + +- NVIDIA Xids; +- RM assertions; +- BAR1 failures; +- IOMMU faults; +- AER faults; +- mlx5 resets; +- DMA-BUF lifecycle warnings; +- kernel WARN/BUG reports; +- memory-corruption indicators. + +--- + +## 29. Post-Test CUDA and GPU P2P Health Check + +After the stress and negative-control phases, verify that the GPUs still function normally. + +Run an ordinary CUDA smoke test or known-good CUDA sample. + +Also verify ordinary CUDA P2P capability/state where expected. + +At minimum: + +```bash +nvidia-smi +``` + +must succeed and both GPUs must remain operational. + +Prefer an actual CUDA allocation/kernel/copy test rather than relying solely on `nvidia-smi`. + +The validation is incomplete if the experimental workload succeeds but leaves the GPU or driver in a degraded state. + +--- + +## 30. Result Classification + +### DMA-BUF GPUDirect RDMA: PASS + +Require: + +- intended experimental kernel module loaded; +- `ExperimentalDmaBufP2P: 1`; +- topology prerequisites accepted; +- patched `libcuda` verified at runtime; +- CUDA DMA-BUF FD export succeeds; +- DMA-BUF verbs registration succeeds; +- no pointer-registration fallback; +- RDMA transfer completes. + +### NCCL GPUDirect RDMA: PASS + +Additionally require: + +- one two-rank MPI communicator; +- rank 0 and rank 1 use separate GPUs; +- `Using network IB`; +- actual connectors report `NET/IB/.../GDRDMA`; +- workload-scale RDMA activity is observed. + +### Internal ConnectX forwarding: STRONGLY SUPPORTED + +Require: + +- successful NCCL GDRDMA result; +- substantial RDMA activity; +- negligible IEEE 802.3 MAC deltas during NCCL; +- required physical-wire control shows that the same MAC counters increase substantially when traffic actually traverses the external ports. + +Describe this as **strong evidence for adapter-internal forwarding**, not formal proof of every internal ASIC stage. + +### Feature-gate safety: PASS + +Require: + +- `NVreg_ExperimentalDmaBufP2P=0` prevents use of the experimental path; +- rejection occurs safely; +- no crash or resource leak occurs; +- re-enabling the feature restores the positive path. + +### Stability: PASS + +Require: + +- no new Xid; +- no kernel assertion; +- no IOMMU fault; +- no AER fault; +- no mlx5 reset; +- no BAR1 failure; +- no DMA-BUF cleanup warning; +- no persistent registration or workload-sized GPU-memory leak; +- post-test CUDA operation succeeds. + +--- + +## 31. Required Evidence Artifacts + +The commands above write a flat evidence directory. Preserve at minimum: + +```text +repository-state.txt +driver-working-tree.diff +driver-index.diff +module-provenance.txt +module-sha256.txt +module-signatures.txt +patched-libcuda-files.txt +patched-libcuda-sha256.txt +system-libcuda-resolution.txt +system-libcuda-sha256.txt +perftest-loader-debug.txt +perftest-loader-run.txt +perftest-loader-status.txt +uname.txt +kernel-cmdline.txt +lspci-tree.txt +nvidia-smi-topo.txt +gpu-pci-map.txt +iommu-groups.txt +ibdev2netdev.txt +mst-status.txt +devlink-ports.txt +nvidia-params.txt +experimental-dmabuf-p2p.txt +dmabuf-gdr-topology.txt +ib-write-server.txt +ib-write-server.status +ib-write-client.txt +ib-write-client.status +ib-write-gpu1-*.txt +nccl-environment.txt +nccl-mpi.log +nccl-mpi.status +nccl-validation-lines.txt +nccl-p2p-allowed.{log,status} +nccl-ib-disabled.{log,status} +nccl-dmabuf-disabled.{log,status} +rdma-counters-{before,after}.txt +rdma-hw-counters-{before,after}.txt +ppcnt-port{0,1}-{before,after}.txt +wire-ppcnt-port{0,1}-{before,after}.txt +wire-ppcnt-deltas.txt +wire-control.status +wire-{server,client}.txt +gpu-state-before.txt +gpu-health-pre-nccl.txt +gpu-health-after.txt +dmesg-before.txt +dmesg-pre-nccl.txt +dmesg-after.txt +kernel-health-findings.txt +stress-*.{log,status} +stress-failures.txt +``` + +Raw evidence must be retained even when a summarized report is generated. + +--- + +## 32. Final Agent Report + +Produce a result matrix: + +| Validation | Evidence | Result | +|---|---|---| +| Correct experimental module loaded | hashes/srcversion/signature | PASS/FAIL | +| Experimental feature enabled | `/proc/driver/nvidia/params` | PASS/FAIL | +| BAR1/topology prerequisites | driver + PCI/IOMMU diagnostics | PASS/FAIL | +| Patched `libcuda` provenance | hashes + runtime mapping | PASS/FAIL | +| CUDA DMA-BUF export | perftest log | PASS/FAIL | +| DMA-BUF verbs registration | perftest log | PASS/FAIL | +| No legacy MR fallback | registration-path evidence | PASS/FAIL | +| RDMA transfer | perftest result | PASS/FAIL | +| MPI two-rank communicator | NCCL log | PASS/FAIL | +| Separate GPU assignment | NCCL rank/device log | PASS/FAIL | +| NCCL IB transport | `Using network IB` | PASS/FAIL | +| NCCL GDRDMA connector | `NET/IB/.../GDRDMA` | PASS/FAIL | +| Workload-scale RDMA activity | RDMA counter deltas | PASS/FAIL | +| Negligible external MAC traffic | `PPCNT grp=0` deltas | PASS/FAIL | +| Physical MAC-counter calibration | forced-wire control | PASS/FAIL | +| Internal forwarding inference | differential evidence | STRONGLY SUPPORTED / NOT SUPPORTED | +| Feature disabled safely | `NVreg_ExperimentalDmaBufP2P=0` control | PASS/FAIL | +| Concurrent registration stress | stress logs | PASS/FAIL | +| Clean teardown | mapping/memory diagnostics | PASS/FAIL | +| Kernel health | Xid/IOMMU/AER/mlx5/BAR1 audit | PASS/FAIL | +| Post-test CUDA health | CUDA smoke test | PASS/FAIL | + +The report must distinguish **observed facts** from **inferred architecture**. + +A successful final result should be phrased approximately as: + +```text +Experimental DMA-BUF GPUDirect RDMA path: PASS +NCCL NET/IB/GDRDMA communication: PASS +Feature-gate default-off behavior: PASS +Concurrent registration/deregistration stability: PASS +Post-test driver/GPU health: PASS + +Same-host ConnectX internal forwarding: +STRONGLY SUPPORTED by workload-scale RDMA activity combined with +negligible IEEE 802.3 MAC activity, calibrated against a physical-wire +control that produces workload-scale MAC counter increments. +``` + +The reproduction plan should be committed separately from the executed validation record as: + +```text +validation/nccl-same-host-gdr-reproduction-plan.md +``` + +with a documentation-only commit such as: + +```text +Add same-host DMA-BUF GDR reproduction plan +```