diff --git a/README.md b/README.md index 2999d9671c..30cd34449b 100644 --- a/README.md +++ b/README.md @@ -1,7 +1,7 @@ # NVIDIA driver 610.43.03 with P2P for RTX 3090, RTX 4090, and RTX 5090 -This enables P2P on consumer GPUs with the 610.43.03 driver version. No kernel parameters -are needed for the default behavior, just build, install, and go. +This enables P2P on consumer GPUs with the 610.43.03 driver version. The current branch +requires the IOMMU passthrough configuration described below. See the [tinygrad 550.54.15-p2p README](https://github.com/tinygrad/open-gpu-kernel-modules/blob/550.54.15-p2p/README.md) for the original description of the approach. @@ -24,9 +24,10 @@ NVLink where it is. For PCIe pairs, transfers write directly to the other GPU's address over DMA. > [!WARNING] -> IOMMU must be in passthrough mode (`iommu=pt`), not translating, or DMA will go through -> IOMMU page tables and transfers will fail. This is very dangerous if you run untrusted -> software or devices. +> IOMMU must currently be in passthrough mode (`iommu=pt`), not translating. In particular, +> the experimental hugetlb registration path does not yet handle scatterlist entries merged +> by a translated IOMMU. Do not use translated mode until that path is fixed and validated. +> Passthrough mode weakens DMA isolation and is unsafe with untrusted software or devices. ## How to use @@ -52,17 +53,19 @@ options nvidia NVreg_RegistryDwords="RMForceP2PType=1" This branch also includes an experimental path that accelerates `cudaHostRegister` by several orders of magnitude when the registered buffer is backed by 1G hugepages, and -shrinks the device page tables used for such mappings. It is enabled automatically. This -path skips some of the per-4K-page bookkeeping the stock driver performs, so it may -misbehave in edge cases the stock driver handles correctly. +shrinks the device page tables used for such mappings. It is enabled automatically for a +non-empty registration that is hugepage-aligned, is an exact multiple of the hugepage +size, and stays within one hugetlb VMA. Other layouts use the normal per-page array path. +The fast path still skips some base-page bookkeeping and remains experimental. ## Potential issues If P2P transfers are slow, make sure your IOMMU is in passthrough (`pt`) mode and that ACS -is disabled. ACS on root ports forces all GPU-to-GPU traffic through the CPU root complex, -killing P2P bandwidth. ACS can be disabled in BIOS, with the -`pcie_acs_override=downstream,multifunction` kernel parameter (if your kernel supports it), -or with an ACS override patch applied to the kernel. +redirect is not forcing GPU-to-GPU traffic through the root complex. Prefer a firmware ACS +control. If the kernel supports the upstream per-device option, use a narrowly scoped +`pci=disable_acs_redir=[;...]` setting and verify the resulting IOMMU groups. +Disabling ACS redirect weakens device isolation; do not use the broad `pcie_acs_override` +patch or kernel parameter. ## Sample `p2pBandwidthLatencyTest` output diff --git a/kernel-open/nvidia/os-mlock.c b/kernel-open/nvidia/os-mlock.c index 994c326328..e038aa736f 100644 --- a/kernel-open/nvidia/os-mlock.c +++ b/kernel-open/nvidia/os-mlock.c @@ -264,6 +264,54 @@ static void nv_free_page_array(struct page **pages) os_free_mem((NvU8 *)pages - NV_PAGE_ARRAY_HEADER_SIZE); } +static NvBool nv_hugetlb_fast_path_eligible( + struct vm_area_struct *vma, + unsigned long start, + NvU64 page_count, + unsigned long *hpage_size_out, + unsigned int *compound_order_out +) +{ + NvU64 range_size; + NvU64 pages_per_hugepage; + unsigned long end; + unsigned long hpage_size; + + if (!vma || !is_vm_hugetlb_page(vma) || page_count == 0) + return NV_FALSE; + + if (page_count > ((NvU64)ULONG_MAX / PAGE_SIZE)) + return NV_FALSE; + + range_size = page_count * PAGE_SIZE; + if (range_size > ULONG_MAX || start > (ULONG_MAX - (unsigned long)range_size)) + return NV_FALSE; + + end = start + (unsigned long)range_size; + hpage_size = vma_kernel_pagesize(vma); + + if (hpage_size < PAGE_SIZE || hpage_size > NV_U32_MAX || + (hpage_size & (hpage_size - 1)) != 0) + { + return NV_FALSE; + } + + if ((start & (hpage_size - 1)) != 0 || + (range_size & (hpage_size - 1)) != 0 || + end > vma->vm_end) + { + return NV_FALSE; + } + + pages_per_hugepage = hpage_size / PAGE_SIZE; + if ((pages_per_hugepage & (pages_per_hugepage - 1)) != 0) + return NV_FALSE; + + *hpage_size_out = hpage_size; + *compound_order_out = ilog2(pages_per_hugepage); + return NV_TRUE; +} + NV_STATUS NV_API_CALL os_lock_user_pages( void *address, NvU64 page_count, @@ -303,47 +351,48 @@ NV_STATUS NV_API_CALL os_lock_user_pages( * unfaulted 1GB hugepages (~37ms per fault). */ { - struct vm_area_struct *vma = vma_lookup(mm, (unsigned long)address); - - if (vma && is_vm_hugetlb_page(vma)) + unsigned long start = (unsigned long)address; + struct vm_area_struct *vma = vma_lookup(mm, start); + unsigned long hpage_size; + unsigned int order; + + if (nv_hugetlb_fast_path_eligible(vma, + start, + page_count, + &hpage_size, + &order)) { - unsigned long hpage_size = vma_kernel_pagesize(vma); - unsigned int order = ilog2(hpage_size / PAGE_SIZE); - NvU64 pages_per_hp = 1ULL << order; + NvU64 num_hugepages = page_count >> order; - if ((page_count & (pages_per_hp - 1)) == 0) + rmStatus = nv_alloc_page_array(num_hugepages, order, &user_pages); + if (rmStatus != NV_OK) { - NvU64 num_hugepages = page_count >> order; + nv_mmap_read_unlock(mm); + nv_printf(NV_DBG_ERRORS, + "NVRM: failed to allocate hugepage table!\n"); + return rmStatus; + } - rmStatus = nv_alloc_page_array(num_hugepages, order, &user_pages); - if (rmStatus != NV_OK) + for (i = 0; i < num_hugepages; i++) + { + ret = NV_PIN_USER_PAGES(start + i * hpage_size, + 1, + gup_flags, + &user_pages[i]); + if (ret != 1) { + for (j = 0; j < i; j++) + NV_UNPIN_USER_PAGE(user_pages[j]); + nv_free_page_array(user_pages); nv_mmap_read_unlock(mm); - nv_printf(NV_DBG_ERRORS, - "NVRM: failed to allocate hugepage table!\n"); - return rmStatus; - } - - for (i = 0; i < num_hugepages; i++) - { - ret = NV_PIN_USER_PAGES( - (unsigned long)address + i * hpage_size, - 1, gup_flags, &user_pages[i]); - if (ret != 1) - { - for (j = 0; j < i; j++) - NV_UNPIN_USER_PAGE(user_pages[j]); - nv_free_page_array(user_pages); - nv_mmap_read_unlock(mm); - return NV_ERR_INVALID_ADDRESS; - } + return NV_ERR_INVALID_ADDRESS; } + } - nv_mmap_read_unlock(mm); + nv_mmap_read_unlock(mm); - *page_array = user_pages; - return NV_OK; - } + *page_array = user_pages; + return NV_OK; } } diff --git a/src/nvidia/arch/nvalloc/unix/include/nv-reg.h b/src/nvidia/arch/nvalloc/unix/include/nv-reg.h index 93999d4f3c..decf092752 100644 --- a/src/nvidia/arch/nvalloc/unix/include/nv-reg.h +++ b/src/nvidia/arch/nvalloc/unix/include/nv-reg.h @@ -1051,7 +1051,7 @@ NV_DEFINE_REG_ENTRY_GLOBAL(__NV_NVLINK_DISABLE, 0); NV_DEFINE_REG_ENTRY_GLOBAL(__NV_ENABLE_PCIE_RELAXED_ORDERING_MODE, 0); NV_DEFINE_REG_ENTRY_GLOBAL(__NV_REGISTER_PCI_DRIVER, 1); NV_DEFINE_REG_ENTRY_GLOBAL(__NV_REGISTER_PLATFORM_DEVICE_DRIVER, 1); -NV_DEFINE_REG_ENTRY_GLOBAL(__NV_ENABLE_RESIZABLE_BAR, 0); +NV_DEFINE_REG_ENTRY_GLOBAL(__NV_ENABLE_RESIZABLE_BAR, 1); NV_DEFINE_REG_ENTRY_GLOBAL(__NV_ENABLE_DBG_BREAKPOINT, 0); NV_DEFINE_REG_ENTRY_GLOBAL(__NV_TEGRA_GPU_PG_MASK, 0); NV_DEFINE_REG_ENTRY_GLOBAL(__NV_ENABLE_NONBLOCKING_OPEN, 1); diff --git a/src/nvidia/arch/nvalloc/unix/src/osmemdesc.c b/src/nvidia/arch/nvalloc/unix/src/osmemdesc.c index 89ee179c4d..6b7cb20f84 100644 --- a/src/nvidia/arch/nvalloc/unix/src/osmemdesc.c +++ b/src/nvidia/arch/nvalloc/unix/src/osmemdesc.c @@ -1243,7 +1243,7 @@ osDestroyOsDescriptorPageArray // Read before nv_unregister_user_pages frees the nv_alloc_t. compoundOrder = nv_get_compound_order(pPrivate); - if (compoundOrder > 0) + if (compoundOrder > 0 || IS_DISCONTIG_AND_DYNGRAN_ENABLED(pMemDesc)) osPageCount = pMemDesc->PageCount; else osPageCount = NV_RM_PAGES_TO_OS_PAGES(pMemDesc->PageCount); diff --git a/src/nvidia/src/kernel/gpu/bif/kernel_bif.c b/src/nvidia/src/kernel/gpu/bif/kernel_bif.c index 322bbc9730..269b8fea70 100644 --- a/src/nvidia/src/kernel/gpu/bif/kernel_bif.c +++ b/src/nvidia/src/kernel/gpu/bif/kernel_bif.c @@ -1127,8 +1127,11 @@ _kbifInitRegistryOverrides { NvU32 data32; - // P2P Override: default to both reads+writes enabled so BAR1 P2P works out of the box - pKernelBif->p2pOverride = 0x11; + // Enable BAR1 P2P reads and writes without overriding platform atomic capabilities. + pKernelBif->p2pOverride = + DRF_DEF(_REG_STR, _CL_FORCE_P2P, _READ, _ENABLE) | + DRF_DEF(_REG_STR, _CL_FORCE_P2P, _WRITE, _ENABLE) | + DRF_DEF(_REG_STR, _CL_FORCE_P2P, _ATOMICS, _DEFAULT); if (osReadRegistryDword(pGpu, NV_REG_STR_CL_FORCE_P2P, &data32) == NV_OK) { pKernelBif->p2pOverride = data32; @@ -2069,4 +2072,3 @@ kbifWaitForConfigAccessAfterReset_IMPL return NV_ERR_GENERIC; } - diff --git a/src/nvidia/src/kernel/mem_mgr/io_vaspace.c b/src/nvidia/src/kernel/mem_mgr/io_vaspace.c index eaa8cecab1..d3256379c8 100644 --- a/src/nvidia/src/kernel/mem_mgr/io_vaspace.c +++ b/src/nvidia/src/kernel/mem_mgr/io_vaspace.c @@ -59,15 +59,15 @@ iovaspaceConstruct__IMPL void iovaspaceDestruct_IMPL(OBJIOVASPACE *pIOVAS) { - // OBJVASPACE *pVAS = staticCast(pIOVAS, OBJVASPACE); - - // TODO: might keep p2p mappings... - // if (pIOVAS->mappingCount != 0) - // { - // NV_PRINTF(LEVEL_ERROR, "%lld left-over mappings in IOVAS 0x%x\n", - // pIOVAS->mappingCount, pVAS->vaspaceId); - // DBG_BREAKPOINT(); - // } + OBJVASPACE *pVAS = staticCast(pIOVAS, OBJVASPACE); + + if (pIOVAS->mappingCount != 0) + { + NV_PRINTF(LEVEL_WARNING, + "%" NvU64_fmtu " left-over mappings in IOVAS 0x%x\n", + pIOVAS->mappingCount, + pVAS->vaspaceId); + } } NV_STATUS @@ -600,12 +600,9 @@ OBJIOVASPACE *iovaspaceFromMapping(PIOVAMAPPING pIovaMapping) OBJIOVASPACE *pIOVAS = iovaspaceFromId(pIovaMapping->iovaspaceId); // - // The IOVASPACE has to be there as the mapping is referencing it. If it's - // not, the mapping has been left dangling outlasting the IOVAS it was - // under. + // A missing IOVAS means the mapping was left dangling and outlived the + // address space it belonged to. The destroy path reports that condition. // - // NV_ASSERT(pIOVAS != NULL); - return pIOVAS; } @@ -613,7 +610,14 @@ void iovaMappingDestroy(PIOVAMAPPING pIovaMapping) { OBJIOVASPACE *pIOVAS = iovaspaceFromMapping(pIovaMapping); - if (pIOVAS == NULL) return; + if (pIOVAS == NULL) + { + NV_PRINTF(LEVEL_WARNING, + "IOVA mapping outlived IOVAS 0x%x\n", + pIovaMapping->iovaspaceId); + return; + } + iovaspaceDestroyMapping(pIOVAS, pIovaMapping); } diff --git a/src/nvidia/src/kernel/rmapi/nv_gpu_ops.c b/src/nvidia/src/kernel/rmapi/nv_gpu_ops.c index 3de77d045a..9a005fd148 100644 --- a/src/nvidia/src/kernel/rmapi/nv_gpu_ops.c +++ b/src/nvidia/src/kernel/rmapi/nv_gpu_ops.c @@ -3967,8 +3967,7 @@ nvGpuOpsBuildExternalAllocPtes NvBool isPeerSupported, NvBool isBar1P2PSupported, NvU32 peerId, - gpuExternalMappingInfo *pGpuExternalMappingInfo, - RmPhysAddr bar1BusAddr + gpuExternalMappingInfo *pGpuExternalMappingInfo ) { NV_STATUS status = NV_OK; @@ -4125,14 +4124,7 @@ nvGpuOpsBuildExternalAllocPtes NvU32 ptePcfHw = 0; nvFieldSetBool(&pPteFmt->fldValid, NV_TRUE, pte.v8); - if ((aperture == GMMU_APERTURE_PEER) && isBar1P2PSupported) - { - gmmuFieldSetAperture(&pPteFmt->fldAperture, GMMU_APERTURE_SYS_COH, pte.v8); - } - else - { - gmmuFieldSetAperture(&pPteFmt->fldAperture, aperture, pte.v8); - } + gmmuFieldSetAperture(&pPteFmt->fldAperture, aperture, pte.v8); nvFieldSet32(&pPteFmt->fldKind, kind, pte.v8); ptePcfSw |= vol ? (1 << SW_MMU_PCF_UNCACHED_IDX) : 0; @@ -4176,14 +4168,7 @@ nvGpuOpsBuildExternalAllocPtes if (nvFieldIsValid32(&pPteFmt->fldAtomicDisable.desc)) nvFieldSetBool(&pPteFmt->fldAtomicDisable, !atomic, pte.v8); - if ((aperture == GMMU_APERTURE_PEER) && isBar1P2PSupported) - { - gmmuFieldSetAperture(&pPteFmt->fldAperture, GMMU_APERTURE_SYS_NONCOH, pte.v8); - } - else - { - gmmuFieldSetAperture(&pPteFmt->fldAperture, aperture, pte.v8); - } + gmmuFieldSetAperture(&pPteFmt->fldAperture, aperture, pte.v8); if (!isCompressedKind) { @@ -4194,11 +4179,6 @@ nvGpuOpsBuildExternalAllocPtes } } - if ((aperture == GMMU_APERTURE_PEER) && isBar1P2PSupported) - { - fabricBaseAddress = bar1BusAddr; - } - if ((aperture == GMMU_APERTURE_PEER) && !isBar1P2PSupported) { nvFieldSet32(&pPteFmt->fldPeerIndex, peerId, pte.v8); @@ -4458,8 +4438,7 @@ nvGpuOpsBuildExternalAllocPhysAddrs NvBool isPeerSupported, NvBool isBar1P2PSupported, NvU32 peerId, - UvmGpuExternalPhysAddrInfo *pGpuExternalPhysAddrInfo, - RmPhysAddr bar1BusAddr + UvmGpuExternalPhysAddrInfo *pGpuExternalPhysAddrInfo ) { NV_STATUS status = NV_OK; @@ -4537,11 +4516,6 @@ nvGpuOpsBuildExternalAllocPhysAddrs return NV_ERR_BUFFER_TOO_SMALL; - if ((aperture == GMMU_APERTURE_PEER) && isBar1P2PSupported) - { - fabricBaseAddress = bar1BusAddr; - } - if ((aperture == GMMU_APERTURE_PEER) && !isBar1P2PSupported) { // @@ -4684,7 +4658,6 @@ NV_STATUS nvGpuOpsGetExternalAllocPtesOrPhysAddrs(struct gpuAddressSpace *vaSpac Memory *pMemory = NULL; PMEMORY_DESCRIPTOR pMemDesc = NULL; OBJGPU *pMappingGpu = NULL; - RmPhysAddr bar1BusAddr = 0; NvU32 peerId = 0; NvBool isSliSupported = NV_FALSE; NvBool isPeerSupported = NV_FALSE; @@ -4824,8 +4797,6 @@ NV_STATUS nvGpuOpsGetExternalAllocPtesOrPhysAddrs(struct gpuAddressSpace *vaSpac &peerId); if (status != NV_OK) goto freeGpaMemdesc; - - bar1BusAddr = gpumgrGetGpuPhysFbAddr(pAdjustedMemDesc->pGpu); } // @@ -4914,15 +4885,14 @@ NV_STATUS nvGpuOpsGetExternalAllocPtesOrPhysAddrs(struct gpuAddressSpace *vaSpac isPeerSupported, isBar1P2PSupported, peerId, - pGpuExternalMappingInfo, - bar1BusAddr); + pGpuExternalMappingInfo); } if (pGpuExternalPhysAddrInfo != NULL) { status = nvGpuOpsBuildExternalAllocPhysAddrs(pVAS, vaSpace->device->session, pMappingGpu, pAdjustedMemDesc, pMemory, offset, size, isIndirectPeerSupported, isPeerSupported, - isBar1P2PSupported, peerId, pGpuExternalPhysAddrInfo, bar1BusAddr); + isBar1P2PSupported, peerId, pGpuExternalPhysAddrInfo); } freeGpaMemdesc: @@ -11076,7 +11046,7 @@ NV_STATUS nvGpuOpsGetChannelResourcePtes(struct gpuAddressSpace *vaSpace, status = nvGpuOpsBuildExternalAllocPtes(pVAS, vaSpace->device->session, pMappingGpu, pMemDesc, NULL, offset, size, NV_FALSE, NV_FALSE, - NV_FALSE, 0, pGpuExternalMappingInfo, 0); + NV_FALSE, 0, pGpuExternalMappingInfo); _nvGpuOpsLocksRelease(&acquiredLocks); threadStateFree(&threadState, THREAD_STATE_FLAGS_NONE);