diff --git a/go.mod b/go.mod index 052545ff8..10927cdca 100644 --- a/go.mod +++ b/go.mod @@ -4,7 +4,7 @@ go 1.25.0 require ( github.com/Masterminds/semver/v3 v3.5.0 - github.com/NVIDIA/go-nvlib v0.11.0 + github.com/NVIDIA/go-nvlib v0.12.0 github.com/NVIDIA/go-nvml v0.13.3-1 github.com/containerd/nri v0.12.0 github.com/cyphar/filepath-securejoin v0.7.0 diff --git a/go.sum b/go.sum index a67efdd9d..1598df512 100644 --- a/go.sum +++ b/go.sum @@ -2,8 +2,8 @@ cyphar.com/go-pathrs v0.2.5 h1:SnX9FBvnoyn3lUs1dkMgZ52bAETpirNu3FTRh5HlRik= cyphar.com/go-pathrs v0.2.5/go.mod h1:y8f1EMG7r+hCuFf/rXsKqMJrJAUoADZGNh5/vZPKcGc= github.com/Masterminds/semver/v3 v3.5.0 h1:kQceYJfbupGfZOKZQg0kou0DgAKhzDg2NZPAwZ/2OOE= github.com/Masterminds/semver/v3 v3.5.0/go.mod h1:4V+yj/TJE1HU9XfppCwVMZq3I84lprf4nC11bSS5beM= -github.com/NVIDIA/go-nvlib v0.11.0 h1:J6c9deWGJ1x4yY7fKg+aOdm2v5+WmCIeCLsuaO3tRtA= -github.com/NVIDIA/go-nvlib v0.11.0/go.mod h1:uQNH63NoDuSfn/1lixD1D1Hvhko/xdnBHmc4H1mFUlY= +github.com/NVIDIA/go-nvlib v0.12.0 h1:LICVlUGlDnpbwQv64rOZQn11xQae1J+c+dvH9CCi7jc= +github.com/NVIDIA/go-nvlib v0.12.0/go.mod h1:J5M/QPIJJtaipjdONqevSnfgBlkW49uVWX5cFOoQpoA= github.com/NVIDIA/go-nvml v0.13.3-1 h1:P76U2h88OZSiMtdhRsJjSF5DXyXUqHIXKeDicVAaae0= github.com/NVIDIA/go-nvml v0.13.3-1/go.mod h1:ahi2psRYoa+wYUBIrZPRO+wJs9lcvMhxSSkjjvsJJNQ= github.com/blang/semver/v4 v4.0.0 h1:1PFHFE6yCCTv8C1TeyNNarDzntLi7wMI5i/pzqYIsAM= diff --git a/vendor/github.com/NVIDIA/go-nvlib/pkg/nvlib/device/device.go b/vendor/github.com/NVIDIA/go-nvlib/pkg/nvlib/device/device.go index df0085b8c..254617f0e 100644 --- a/vendor/github.com/NVIDIA/go-nvlib/pkg/nvlib/device/device.go +++ b/vendor/github.com/NVIDIA/go-nvlib/pkg/nvlib/device/device.go @@ -95,6 +95,8 @@ func (d *device) GetArchitectureAsString() (string, error) { return "Hopper", nil case nvml.DEVICE_ARCH_BLACKWELL: return "Blackwell", nil + case nvml.DEVICE_ARCH_RUBIN: + return "Rubin", nil case nvml.DEVICE_ARCH_UNKNOWN: return "Unknown", nil } @@ -379,8 +381,8 @@ func (d *device) VisitMigProfiles(visit func(MigProfile) error) error { return fmt.Errorf("error getting GPU Instance profile info: %v", ret) } - for j := nvml.COMPUTE_INSTANCE_PROFILE_COUNT - 1; j >= 0; j-- { - for k := nvml.COMPUTE_INSTANCE_ENGINE_PROFILE_COUNT - 1; k >= 0; k-- { + for j := 0; j < nvml.COMPUTE_INSTANCE_PROFILE_COUNT; j++ { + for k := 0; k < nvml.COMPUTE_INSTANCE_ENGINE_PROFILE_COUNT; k++ { p, err := d.lib.NewMigProfile(i, j, k, giProfileInfo.MemorySizeMB, memory.Total) if err != nil { return fmt.Errorf("error creating MIG profile: %v", err) @@ -402,25 +404,6 @@ func (d *device) VisitMigProfiles(visit func(MigProfile) error) error { if (pi.C < pi.G) && ((pi.C * 2) > (pi.G + 1)) { continue } - // NOTE: As we iterate through the profiles by GPU instance count and Compute instance count, we will find revisions - // of the COMPUTE_INSTANCE_PROFILE that have the same slice count. In these cases, we need to ensure that we pick the - // COMPUTE_INSTANCE_PROFILE with the maximum multiprocessor (SM) count, that is still valid for the GPU instance profile. - // - // For example: On systems like the H100, the MIG profiles 1g.12gb and 1g.24gb are both supported and will have one cislice. - // Given the higher memory capacity, the 1g.24gb profile would be compatible with both the COMPUTE_INSTANCE_PROFILE_1_SLICE - // and the COMPUTE_INSTANCE_PROFILE_1_SLICE_REV1. However, the 1g.12gb profile would only be compatible with the - // COMPUTE_INSTANCE_PROFILE_1_SLICE. To ensure that we have the CI profile with the maximum compatible SM count, the 1g.12gb - // should use the COMPUTE_INSTANCE_PROFILE_1_SLICE and 1g.24gb should use the COMPUTE_INSTANCE_PROFILE_1_SLICE_REV1. - // - // While iterating through the GPU instance profiles (d.GetGpuInstanceProfileInfo(i)), we need to ensure that we pick the - // correct CI profile/revision. For the above example, we will find that GPU_INSTANCE_PROFILE_1_SLICE is only compatible with - // COMPUTE_INSTANCE_PROFILE_1_SLICE. The GPU_INSTANCE_PROFILE_1_SLICE_REV2 (when available for systems like the H100) can be - // deployed with both COMPUTE_INSTANCE_PROFILE_1_SLICE and COMPUTE_INSTANCE_PROFILE_1_SLICE_REV1, but to maximize performance, - // we should pick the COMPUTE_INSTANCE_PROFILE_1_SLICE_REV1 profile. The following check is a temporary workaround to ensure - // that we pick the correct COMPUTE_INSTANCE_PROFILE revision. - if pi.CIProfileID == nvml.COMPUTE_INSTANCE_PROFILE_1_SLICE_REV1 && pi.GIProfileID != nvml.GPU_INSTANCE_PROFILE_1_SLICE_REV2 { - continue - } err = visit(p) if err != nil { diff --git a/vendor/github.com/NVIDIA/go-nvlib/pkg/nvlib/device/mig_profile.go b/vendor/github.com/NVIDIA/go-nvlib/pkg/nvlib/device/mig_profile.go index 6ab94ad2e..af79006f0 100644 --- a/vendor/github.com/NVIDIA/go-nvlib/pkg/nvlib/device/mig_profile.go +++ b/vendor/github.com/NVIDIA/go-nvlib/pkg/nvlib/device/mig_profile.go @@ -101,7 +101,8 @@ func (d *devicelib) NewMigProfile(giProfileID, ciProfileID, ciEngProfileID int, ciSlices = 4 case nvml.COMPUTE_INSTANCE_PROFILE_6_SLICE: ciSlices = 6 - case nvml.COMPUTE_INSTANCE_PROFILE_7_SLICE: + case nvml.COMPUTE_INSTANCE_PROFILE_7_SLICE, + nvml.COMPUTE_INSTANCE_PROFILE_7_SLICE_NVL: ciSlices = 7 case nvml.COMPUTE_INSTANCE_PROFILE_8_SLICE: ciSlices = 8 diff --git a/vendor/github.com/NVIDIA/go-nvlib/pkg/nvpci/mock.go b/vendor/github.com/NVIDIA/go-nvlib/pkg/nvpci/mock.go index 375cfb584..ff0fbbbd7 100644 --- a/vendor/github.com/NVIDIA/go-nvlib/pkg/nvpci/mock.go +++ b/vendor/github.com/NVIDIA/go-nvlib/pkg/nvpci/mock.go @@ -79,7 +79,7 @@ func (m *MockNvpci) AddMockA100(address string, numaNode int, sriov *SriovInfo) return err } - err = createNVIDIAgpuFiles(deviceDir) + err = CreateMockA100SysfsFiles(deviceDir) if err != nil { return err } @@ -139,7 +139,11 @@ func (m *MockNvpci) AddMockA100(address string, numaNode int, sriov *SriovInfo) return nil } -func createNVIDIAgpuFiles(deviceDir string) error { +// CreateMockA100SysfsFiles populates deviceDir with the sysfs attribute files +// of an A100-like GPU (vendor, class, device, subsystem ids, driver symlink, +// config space, and resources). It is shared by mock packages that need an +// NVIDIA PCI device fixture, such as nvmdev. +func CreateMockA100SysfsFiles(deviceDir string) error { vendor, err := os.Create(filepath.Join(deviceDir, "vendor")) if err != nil { return err @@ -167,6 +171,24 @@ func createNVIDIAgpuFiles(deviceDir string) error { return err } + subsystemVendor, err := os.Create(filepath.Join(deviceDir, "subsystem_vendor")) + if err != nil { + return err + } + _, err = fmt.Fprintf(subsystemVendor, "0x%x", PCINvidiaVendorID) + if err != nil { + return err + } + + subsystemDevice, err := os.Create(filepath.Join(deviceDir, "subsystem_device")) + if err != nil { + return err + } + _, err = subsystemDevice.WriteString("0x16c0") + if err != nil { + return err + } + _, err = os.Create(filepath.Join(deviceDir, "nvidia")) if err != nil { return err @@ -233,7 +255,7 @@ func (m *MockNvpci) createVf(pfAddress string, id, iommu_group, numaNode int) er return err } - err = createNVIDIAgpuFiles(deviceDir) + err = CreateMockA100SysfsFiles(deviceDir) if err != nil { return err } diff --git a/vendor/github.com/NVIDIA/go-nvlib/pkg/nvpci/nvpci.go b/vendor/github.com/NVIDIA/go-nvlib/pkg/nvpci/nvpci.go index a322ea26d..f1fd841b3 100644 --- a/vendor/github.com/NVIDIA/go-nvlib/pkg/nvpci/nvpci.go +++ b/vendor/github.com/NVIDIA/go-nvlib/pkg/nvpci/nvpci.go @@ -17,6 +17,7 @@ package nvpci import ( + "errors" "fmt" "os" "path" @@ -107,20 +108,22 @@ func (s *SriovInfo) IsVF() bool { // NvidiaPCIDevice represents a PCI device for an NVIDIA product. type NvidiaPCIDevice struct { - Path string - Address string - Vendor uint16 - Class uint32 - ClassName string - Device uint16 - DeviceName string - Driver string - IommuGroup int - IommuFD string - NumaNode int - Config *ConfigSpace - Resources MemoryResources - SriovInfo SriovInfo + Path string + Address string + Vendor uint16 + Class uint32 + ClassName string + Device uint16 + SubsystemVendor uint16 + SubsystemDevice uint16 + DeviceName string + Driver string + IommuGroup int + IommuFD string + NumaNode int + Config *ConfigSpace + Resources MemoryResources + SriovInfo SriovInfo } // IsVGAController if class == 0x300. @@ -256,6 +259,28 @@ func (p *nvpci) GetNvidiaDeviceByPciBusID(address string) (*NvidiaPCIDevice, err return p.getNvidiaDeviceByPciBusID(address, nil) } +// readPCIFieldString reads a sysfs PCI attribute file and returns its contents with any surrounding whitespaces trimmed. +func readPCIFieldString(devicePath, field string) (string, error) { + raw, err := os.ReadFile(path.Join(devicePath, field)) + if err != nil { + return "", fmt.Errorf("unable to read PCI %s for %s: %w", field, devicePath, err) + } + return strings.TrimSpace(string(raw)), nil +} + +// readPCIField reads a sysfs PCI attribute file and parses it as an unsigned integer. +func readPCIField(devicePath, field string, bitSize int) (uint64, error) { + str, err := readPCIFieldString(devicePath, field) + if err != nil { + return 0, err + } + val, err := strconv.ParseUint(str, 0, bitSize) + if err != nil { + return 0, fmt.Errorf("unable to parse PCI %s for %s: %w", field, devicePath, err) + } + return val, nil +} + func (p *nvpci) getNvidiaDeviceByPciBusID(address string, cache map[string]*NvidiaPCIDevice) (*NvidiaPCIDevice, error) { if cache != nil { if pciDevice, exists := cache[address]; exists { @@ -264,38 +289,51 @@ func (p *nvpci) getNvidiaDeviceByPciBusID(address string, cache map[string]*Nvid } devicePath := filepath.Join(p.pciDevicesRoot, address) - vendor, err := os.ReadFile(path.Join(devicePath, "vendor")) + vendorID, err := readPCIField(devicePath, "vendor", 16) if err != nil { - return nil, fmt.Errorf("unable to read PCI device vendor id for %s: %v", address, err) - } - vendorStr := strings.TrimSpace(string(vendor)) - vendorID, err := strconv.ParseUint(vendorStr, 0, 16) - if err != nil { - return nil, fmt.Errorf("unable to convert vendor string to uint16: %v", vendorStr) + return nil, err } if uint16(vendorID) != PCINvidiaVendorID && uint16(vendorID) != PCIMellanoxVendorID { return nil, nil } - class, err := os.ReadFile(path.Join(devicePath, "class")) + classID, err := readPCIField(devicePath, "class", 32) + if err != nil { + return nil, err + } + + deviceID, err := readPCIField(devicePath, "device", 16) + if err != nil { + return nil, err + } + + numaStr, err := readPCIFieldString(devicePath, "numa_node") if err != nil { - return nil, fmt.Errorf("unable to read PCI device class for %s: %v", address, err) + return nil, err } - classStr := strings.TrimSpace(string(class)) - classID, err := strconv.ParseUint(classStr, 0, 32) + // numa_node is parsed as a signed integer since "-1" is a valid value meaning "no NUMA affinity". + numaNode, err := strconv.ParseInt(numaStr, 0, 64) if err != nil { - return nil, fmt.Errorf("unable to convert class string to uint32: %v", classStr) + return nil, fmt.Errorf("unable to parse PCI numa_node for %s: %w", devicePath, err) } - device, err := os.ReadFile(path.Join(devicePath, "device")) + // Tolerate missing subsystem files: some environments (e.g. certain virtualised or passthrough PCI topologies) + // do not expose them, so the IDs will default to 0. + subsystemVendorID, err := readPCIField(devicePath, "subsystem_vendor", 16) if err != nil { - return nil, fmt.Errorf("unable to read PCI device id for %s: %v", address, err) + if !errors.Is(err, os.ErrNotExist) { + return nil, err + } + p.logger.Warningf("subsystem_vendor file not found for %s", address) } - deviceStr := strings.TrimSpace(string(device)) - deviceID, err := strconv.ParseUint(deviceStr, 0, 16) + + subsystemDeviceID, err := readPCIField(devicePath, "subsystem_device", 16) if err != nil { - return nil, fmt.Errorf("unable to convert device string to uint16: %v", deviceStr) + if !errors.Is(err, os.ErrNotExist) { + return nil, err + } + p.logger.Warningf("subsystem_device file not found for %s", address) } driver, err := getDriver(devicePath) @@ -314,16 +352,6 @@ func (p *nvpci) getNvidiaDeviceByPciBusID(address string, cache map[string]*Nvid p.logger.Warningf("unable to detect IOMMU FD for %s: %v", address, err) } - numa, err := os.ReadFile(path.Join(devicePath, "numa_node")) - if err != nil { - return nil, fmt.Errorf("unable to read PCI NUMA node for %s: %v", address, err) - } - numaStr := strings.TrimSpace(string(numa)) - numaNode, err := strconv.ParseInt(numaStr, 0, 64) - if err != nil { - return nil, fmt.Errorf("unable to convert NUMA node string to int64: %v", numaNode) - } - config := &ConfigSpace{ Path: path.Join(devicePath, "config"), } @@ -391,20 +419,22 @@ func (p *nvpci) getNvidiaDeviceByPciBusID(address string, cache map[string]*Nvid } nvdevice := &NvidiaPCIDevice{ - Path: devicePath, - Address: address, - Vendor: uint16(vendorID), - Class: uint32(classID), - Device: uint16(deviceID), - Driver: driver, - IommuGroup: int(iommuGroup), - IommuFD: iommuFD, - NumaNode: int(numaNode), - Config: config, - Resources: resources, - DeviceName: deviceName, - ClassName: className, - SriovInfo: sriovInfo, + Path: devicePath, + Address: address, + Vendor: uint16(vendorID), + Class: uint32(classID), + Device: uint16(deviceID), + SubsystemVendor: uint16(subsystemVendorID), + SubsystemDevice: uint16(subsystemDeviceID), + Driver: driver, + IommuGroup: int(iommuGroup), + IommuFD: iommuFD, + NumaNode: int(numaNode), + Config: config, + Resources: resources, + DeviceName: deviceName, + ClassName: className, + SriovInfo: sriovInfo, } // Cache physical functions only as VF can't be a root device. @@ -498,32 +528,20 @@ func (p *nvpci) GetGPUByIndex(i int) (*NvidiaPCIDevice, error) { } func (p *nvpci) getSriovInfoForPhysicalFunction(devicePath string) (sriovInfo SriovInfo, err error) { - totalVfsPath := filepath.Join(devicePath, "sriov_totalvfs") - numVfsPath := filepath.Join(devicePath, "sriov_numvfs") - // No file for sriov_totalvfs exists? Not an SRIOV device, return nil - _, err = os.Stat(totalVfsPath) + _, err = os.Stat(filepath.Join(devicePath, "sriov_totalvfs")) if err != nil && os.IsNotExist(err) { return sriovInfo, nil } - sriovTotalVfs, err := os.ReadFile(totalVfsPath) - if err != nil { - return sriovInfo, fmt.Errorf("unable to read sriov_totalvfs: %v", err) - } - totalVfsStr := strings.TrimSpace(string(sriovTotalVfs)) - totalVfsInt, err := strconv.ParseUint(totalVfsStr, 10, 16) - if err != nil { - return sriovInfo, fmt.Errorf("unable to convert sriov_totalvfs to uint64: %v", err) - } - sriovNumVfs, err := os.ReadFile(numVfsPath) + totalVfsInt, err := readPCIField(devicePath, "sriov_totalvfs", 16) if err != nil { - return sriovInfo, fmt.Errorf("unable to read sriov_numvfs for: %v", err) + return sriovInfo, err } - numVfsStr := strings.TrimSpace(string(sriovNumVfs)) - numVfsInt, err := strconv.ParseUint(numVfsStr, 10, 16) + + numVfsInt, err := readPCIField(devicePath, "sriov_numvfs", 16) if err != nil { - return sriovInfo, fmt.Errorf("unable to convert sriov_numvfs to uint64: %v", err) + return sriovInfo, err } sriovInfo = SriovInfo{ diff --git a/vendor/modules.txt b/vendor/modules.txt index 53ec0614d..2a7d7adf4 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -7,7 +7,7 @@ cyphar.com/go-pathrs/procfs # github.com/Masterminds/semver/v3 v3.5.0 ## explicit; go 1.21 github.com/Masterminds/semver/v3 -# github.com/NVIDIA/go-nvlib v0.11.0 +# github.com/NVIDIA/go-nvlib v0.12.0 ## explicit; go 1.25.0 github.com/NVIDIA/go-nvlib/pkg/nvlib/device github.com/NVIDIA/go-nvlib/pkg/nvlib/info