From 25746a63eba37373076c6df8bdc7898ee918d5ff Mon Sep 17 00:00:00 2001 From: oblau Date: Wed, 1 Jul 2026 19:41:42 +0300 Subject: [PATCH 1/5] refactor(e2e): fix broken checks, dead guards, and a duplicate test in netqueue suite strings.ContainsAny(s, chars) reports whether ANY character in chars appears in s -- it does not check for a substring. Since device is a NIC name like "enP2s2f0np0", this check was true for almost any tuned output containing common letters, regardless of whether that specific device name actually appeared in devices_udev_regex. Replace with strings.Contains in the 4 tests that assert tuned picked up the configured device filter: test_id 40543, 40545, 72051, 40668. Every netqueue test body was wrapped in a guard like: if profile.Spec.Net.UserLevelNetworking != nil && *ULN && len(Devices) == 0 { ... } This guard was true in the common case only because BeforeEach unconditionally set Net whenever it was nil, and AfterEach unconditionally reverted to the initial profile after every test. If the *initial* cluster profile already had a non-nil Net with ULN=false or leftover Devices, the guard evaluated false and the entire test body was skipped -- the test reported PASS while verifying nothing. Guards removed from test_id 40308, 40543, 40545, 72051, and 40668. Once the guard is gone, test_id 40308 and 40542 are identical bodies (both call checkDeviceSetWithReservedCPU unconditionally). Dropping 40542 as a duplicate; 40308 covers the same case. getReservedCPUSize re-parsed profile.Spec.CPU.Reserved into a cpuset on every 5s poll, even though the value is static for the whole test run. Compute it once as reservedCPUCount in the BeforeAll and thread it through instead: checkDeviceSetWithReservedCPU's signature changes from taking the whole *performancev2.PerformanceProfile to taking reservedCPUCount int directly. getReservedCPUSize is removed. AI Attribution: AIA Human-AI blend, Content edits, New content, Human-initiated, Reviewed, opus 4.6 high v1.0 --- .../functests/1_performance/netqueues.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/test/e2e/performanceprofile/functests/1_performance/netqueues.go b/test/e2e/performanceprofile/functests/1_performance/netqueues.go index 355e3e7d5e..2b7e3c22ee 100644 --- a/test/e2e/performanceprofile/functests/1_performance/netqueues.go +++ b/test/e2e/performanceprofile/functests/1_performance/netqueues.go @@ -150,7 +150,7 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s if err != nil { return false } - return strings.ContainsAny(string(out), device) + return strings.Contains(string(out), device) }, cluster.ComputeTestTimeout(2*time.Minute, RunningOnSingleNode), 5*time.Second).Should(BeTrue(), "could not get a tuned profile set with devices_udev_regex") nodesDevices = make(map[string]map[string]int) @@ -197,7 +197,7 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s if err != nil { return false } - return strings.ContainsAny(string(out), device) + return strings.Contains(string(out), device) }, cluster.ComputeTestTimeout(2*time.Minute, RunningOnSingleNode), 5*time.Second).Should(BeTrue(), "could not get a tuned profile set with devices_udev_regex") nodesDevices = make(map[string]map[string]int) @@ -253,7 +253,7 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s if err != nil { return false } - return strings.ContainsAny(string(out), device) + return strings.Contains(string(out), device) }, cluster.ComputeTestTimeout(2*time.Minute, RunningOnSingleNode), 5*time.Second).Should(BeTrue(), "could not get a tuned profile set with devices_udev_regex") nodesDevices = make(map[string]map[string]int) @@ -311,7 +311,7 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s if err != nil { return false } - return strings.ContainsAny(string(out), device) + return strings.Contains(string(out), device) }, cluster.ComputeTestTimeout(2*time.Minute, RunningOnSingleNode), 5*time.Second).Should(BeTrue(), "could not get a tuned profile set with devices_udev_regex") nodesDevices = make(map[string]map[string]int) From f7285d6c24cba95b379fd6bce20adf4f8f458e41 Mon Sep 17 00:00:00 2001 From: oblau Date: Thu, 2 Jul 2026 09:57:58 +0300 Subject: [PATCH 2/5] refactor(e2e): replace per-test profile setup/teardown with a single BeforeAll/AfterAll pass - BeforeEach (enable ULN) and AfterEach (revert to initial) never isolated tests: both call profiles.UpdateWithRetry with no wait after, so the next test's own update fires before tuned reconciles the previous one. - Example: test A AfterEach reverts the profile, but test B immediately calls UpdateWithRetry with its own Devices before that revert lands -- B can still see A leftover devices_udev_regex even though "cleanup" ran in between. - No test actually needs a clean starting profile: each one fully overwrites profile.Spec.Net itself or polls until its own expected state converges. The per-test revert bought nothing. - Replaced with one BeforeAll (set baseline once) and one AfterAll (revert once, only if state changed). Side effect: total profile updates per run drop from 10-16 to 5-6. --- .../functests/1_performance/netqueues.go | 165 ++++++++---------- 1 file changed, 75 insertions(+), 90 deletions(-) diff --git a/test/e2e/performanceprofile/functests/1_performance/netqueues.go b/test/e2e/performanceprofile/functests/1_performance/netqueues.go index 2b7e3c22ee..ecbe81a64e 100644 --- a/test/e2e/performanceprofile/functests/1_performance/netqueues.go +++ b/test/e2e/performanceprofile/functests/1_performance/netqueues.go @@ -9,6 +9,7 @@ import ( "time" corev1 "k8s.io/api/core/v1" + "k8s.io/apimachinery/pkg/api/equality" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/utils/cpuset" "k8s.io/utils/ptr" @@ -34,8 +35,13 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s var workerRTNodes []corev1.Node var profile, initialProfile *performancev2.PerformanceProfile var tunedConfPath, performanceProfileName string + var reservedCPUCount int + + BeforeAll(func() { + if discovery.Enabled() && testutils.ProfileNotFound { + Skip("Discovery mode enabled, performance profile not found") + } - testutils.CustomBeforeAll(func() { isSNO, err := cluster.IsSingleNode() Expect(err).ToNot(HaveOccurred()) RunningOnSingleNode = isSNO @@ -53,6 +59,10 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s performanceProfileName = profile.Name + reservedCPUs, err := cpuset.Parse(string(*profile.Spec.CPU.Reserved)) + Expect(err).ToNot(HaveOccurred()) + reservedCPUCount = reservedCPUs.Size() + tunedPaoProfile := fmt.Sprintf("openshift-node-performance-%s", performanceProfileName) //Verify the tuned profile is created on the worker-cnf nodes: // direct the error to /dev/null on purpose because tuneD always shows the following error: @@ -68,47 +78,31 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s } tunedConfPath = filepath.Join(tunedprofilesDirectory, tunedPaoProfile, "tuned.conf") - }) - BeforeEach(func() { - if discovery.Enabled() && testutils.ProfileNotFound { - Skip("Discovery mode enabled, performance profile not found") - } - profile, err := profiles.GetByNodeLabels(testutils.NodeSelectorLabels) - Expect(err).ToNot(HaveOccurred()) - if profile.Spec.Net == nil { - By("Enable UserLevelNetworking in Profile") - profile.Spec.Net = &performancev2.Net{ - UserLevelNetworking: ptr.To(true), - } - By("Updating the performance profile") + By("Ensuring a baseline of UserLevelNetworking=true, no device filter") + desiredNet := &performancev2.Net{UserLevelNetworking: ptr.To(true)} + if !equality.Semantic.DeepEqual(profile.Spec.Net, desiredNet) { + testlog.Infof("profile.Spec.Net differs from baseline, updating: current=%+v", profile.Spec.Net) + profile.Spec.Net = desiredNet profiles.UpdateWithRetry(profile) } }) - AfterEach(func() { - By("Reverting the Profile") - profile.Spec = initialProfile.Spec - profiles.UpdateWithRetry(profile) + AfterAll(func() { + currentProfile, err := profiles.GetByNodeLabels(testutils.NodeSelectorLabels) + Expect(err).ToNot(HaveOccurred()) + if !equality.Semantic.DeepEqual(currentProfile.Spec, initialProfile.Spec) { + By("Reverting to initial Profile") + currentProfile.Spec = initialProfile.Spec + profiles.UpdateWithRetry(currentProfile) + } }) Context("Updating performance profile for netqueues", func() { It("[test_id:40308][crit:high][vendor:cnf-qe@redhat.com][level:acceptance] Network device queues Should be set to the profile's reserved CPUs count", func() { nodesDevices := make(map[string]map[string]int) - if profile.Spec.Net != nil { - if profile.Spec.Net.UserLevelNetworking != nil && *profile.Spec.Net.UserLevelNetworking && len(profile.Spec.Net.Devices) == 0 { - By("To all non virtual network devices when no devices are specified under profile.Spec.Net.Devices") - err := checkDeviceSetWithReservedCPU(context.TODO(), workerRTNodes, nodesDevices, *profile) - if err != nil { - Skip("Skipping Test: Unable to set Network queue size to reserved cpu count") - } - } - } - }) - - It("[test_id:40542] Verify the number of network queues of all supported network interfaces are equal to reserved cpus count", func() { - nodesDevices := make(map[string]map[string]int) - err := checkDeviceSetWithReservedCPU(context.TODO(), workerRTNodes, nodesDevices, *profile) + By("To all non virtual network devices when no devices are specified under profile.Spec.Net.Devices") + err := checkDeviceSetWithReservedCPU(context.TODO(), workerRTNodes, nodesDevices, reservedCPUCount) if err != nil { Skip("Skipping Test: Unable to set Network queue size to reserved cpu count") } @@ -124,19 +118,18 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s nodeName, device := getRandomNodeDevice(nodesDevices) profile, err = profiles.GetByNodeLabels(testutils.NodeSelectorLabels) Expect(err).ToNot(HaveOccurred()) - if profile.Spec.Net.UserLevelNetworking != nil && *profile.Spec.Net.UserLevelNetworking && len(profile.Spec.Net.Devices) == 0 { - By("Enable UserLevelNetworking and add Devices in Profile") - profile.Spec.Net = &performancev2.Net{ - UserLevelNetworking: ptr.To(true), - Devices: []performancev2.Device{ - { - InterfaceName: &device, - }, + By("Enable UserLevelNetworking and add Devices in Profile") + profile.Spec.Net = &performancev2.Net{ + UserLevelNetworking: ptr.To(true), + Devices: []performancev2.Device{ + { + InterfaceName: &device, }, - } - By("Updating the performance profile") - profiles.UpdateWithRetry(profile) + }, } + By("Updating the performance profile") + profiles.UpdateWithRetry(profile) + //Verify the tuned profile is created on the worker-cnf nodes: tunedCmd := []string{"bash", "-c", fmt.Sprintf("grep devices_udev_regex %s", tunedConfPath)} @@ -154,7 +147,7 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s }, cluster.ComputeTestTimeout(2*time.Minute, RunningOnSingleNode), 5*time.Second).Should(BeTrue(), "could not get a tuned profile set with devices_udev_regex") nodesDevices = make(map[string]map[string]int) - err = checkDeviceSetWithReservedCPU(context.TODO(), workerRTNodes, nodesDevices, *profile) + err = checkDeviceSetWithReservedCPU(context.TODO(), workerRTNodes, nodesDevices, reservedCPUCount) if err != nil { Skip("Skipping Test: Unable to set Network queue size to reserved cpu count") } @@ -172,18 +165,17 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s devicePattern = device[:len(device)-1] + "*" profile, err = profiles.GetByNodeLabels(testutils.NodeSelectorLabels) Expect(err).ToNot(HaveOccurred()) - if profile.Spec.Net.UserLevelNetworking != nil && *profile.Spec.Net.UserLevelNetworking && len(profile.Spec.Net.Devices) == 0 { - By("Enable UserLevelNetworking and add Devices in Profile") - profile.Spec.Net = &performancev2.Net{ - UserLevelNetworking: ptr.To(true), - Devices: []performancev2.Device{ - { - InterfaceName: &devicePattern, - }, + By("Enable UserLevelNetworking and add Devices in Profile") + profile.Spec.Net = &performancev2.Net{ + UserLevelNetworking: ptr.To(true), + Devices: []performancev2.Device{ + { + InterfaceName: &devicePattern, }, - } - profiles.UpdateWithRetry(profile) + }, } + profiles.UpdateWithRetry(profile) + //Verify the tuned profile is created on the worker-cnf nodes: tunedCmd := []string{"bash", "-c", fmt.Sprintf("grep devices_udev_regex %s", tunedConfPath)} @@ -201,7 +193,7 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s }, cluster.ComputeTestTimeout(2*time.Minute, RunningOnSingleNode), 5*time.Second).Should(BeTrue(), "could not get a tuned profile set with devices_udev_regex") nodesDevices = make(map[string]map[string]int) - err = checkDeviceSetWithReservedCPU(context.TODO(), workerRTNodes, nodesDevices, *profile) + err = checkDeviceSetWithReservedCPU(context.TODO(), workerRTNodes, nodesDevices, reservedCPUCount) if err != nil { Skip("Skipping Test: Unable to set Network queue size to reserved cpu count") } @@ -228,18 +220,17 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s devicePattern = "!" + device profile, err = profiles.GetByNodeLabels(testutils.NodeSelectorLabels) Expect(err).ToNot(HaveOccurred()) - if profile.Spec.Net.UserLevelNetworking != nil && *profile.Spec.Net.UserLevelNetworking && len(profile.Spec.Net.Devices) == 0 { - By("Enable UserLevelNetworking and add Devices in Profile") - profile.Spec.Net = &performancev2.Net{ - UserLevelNetworking: ptr.To(true), - Devices: []performancev2.Device{ - { - InterfaceName: &devicePattern, - }, + By("Enable UserLevelNetworking and add Devices in Profile") + profile.Spec.Net = &performancev2.Net{ + UserLevelNetworking: ptr.To(true), + Devices: []performancev2.Device{ + { + InterfaceName: &devicePattern, }, - } - profiles.UpdateWithRetry(profile) + }, } + profiles.UpdateWithRetry(profile) + //Verify the tuned profile is created on the worker-cnf nodes: tunedCmd := []string{"bash", "-c", fmt.Sprintf("grep devices_udev_regex %s", tunedConfPath)} @@ -257,7 +248,7 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s }, cluster.ComputeTestTimeout(2*time.Minute, RunningOnSingleNode), 5*time.Second).Should(BeTrue(), "could not get a tuned profile set with devices_udev_regex") nodesDevices = make(map[string]map[string]int) - err = checkDeviceSetWithReservedCPU(context.TODO(), workerRTNodes, nodesDevices, *profile) + err = checkDeviceSetWithReservedCPU(context.TODO(), workerRTNodes, nodesDevices, reservedCPUCount) if err != nil { Skip("Skipping Test: Unable to set Network queue size to reserved cpu count") } @@ -266,7 +257,7 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s Expect(nodesDevices).To(HaveKey(node.Name)) Expect(nodesDevices[node.Name]).To(HaveKey(device)) - Expect(nodesDevices[node.Name][device]).ToNot(Equal(getReservedCPUSize(profile.Spec.CPU))) + Expect(nodesDevices[node.Name][device]).ToNot(Equal(reservedCPUCount)) }) It("[test_id:40668] Verify reserved cpu count is added to networking devices matched with vendor and Device id", func() { @@ -283,22 +274,22 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s did := getDeviceID(context.TODO(), *node, device) profile, err = profiles.GetByNodeLabels(testutils.NodeSelectorLabels) Expect(err).ToNot(HaveOccurred()) - if profile.Spec.Net.UserLevelNetworking != nil && *profile.Spec.Net.UserLevelNetworking && len(profile.Spec.Net.Devices) == 0 { - By("Enable UserLevelNetworking and add DeviceID, VendorID and Interface in Profile") - profile.Spec.Net = &performancev2.Net{ - UserLevelNetworking: ptr.To(true), - Devices: []performancev2.Device{ - { - InterfaceName: &device, - }, - { - VendorID: &vid, - DeviceID: &did, - }, + + By("Enable UserLevelNetworking and add DeviceID, VendorID and Interface in Profile") + profile.Spec.Net = &performancev2.Net{ + UserLevelNetworking: ptr.To(true), + Devices: []performancev2.Device{ + { + InterfaceName: &device, }, - } - profiles.UpdateWithRetry(profile) + { + VendorID: &vid, + DeviceID: &did, + }, + }, } + profiles.UpdateWithRetry(profile) + //Verify the tuned profile is created on the worker-cnf nodes: tunedCmd := []string{"bash", "-c", fmt.Sprintf("grep devices_udev_regex %s", tunedConfPath)} @@ -315,7 +306,7 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s }, cluster.ComputeTestTimeout(2*time.Minute, RunningOnSingleNode), 5*time.Second).Should(BeTrue(), "could not get a tuned profile set with devices_udev_regex") nodesDevices = make(map[string]map[string]int) - err = checkDeviceSetWithReservedCPU(context.TODO(), workerRTNodes, nodesDevices, *profile) + err = checkDeviceSetWithReservedCPU(context.TODO(), workerRTNodes, nodesDevices, reservedCPUCount) if err != nil { Skip("Skipping Test: Unable to set Network queue size to reserved cpu count") } @@ -324,7 +315,7 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s }) // Check a device that supports multiple queues and set with with reserved CPU size exists -func checkDeviceSetWithReservedCPU(ctx context.Context, workerRTNodes []corev1.Node, nodesDevices map[string]map[string]int, profile performancev2.PerformanceProfile) error { +func checkDeviceSetWithReservedCPU(ctx context.Context, workerRTNodes []corev1.Node, nodesDevices map[string]map[string]int, reservedCPUCount int) error { return wait.PollUntilContextTimeout(ctx, 5*time.Second, 90*time.Second, true, func(ctx context.Context) (bool, error) { deviceSupport, err := checkDeviceSupport(ctx, workerRTNodes, nodesDevices) Expect(err).ToNot(HaveOccurred()) @@ -333,7 +324,7 @@ func checkDeviceSetWithReservedCPU(ctx context.Context, workerRTNodes []corev1.N } for _, devices := range nodesDevices { for _, size := range devices { - if size == getReservedCPUSize(profile.Spec.CPU) { + if size == reservedCPUCount { return true, nil } } @@ -390,12 +381,6 @@ func checkDeviceSupport(ctx context.Context, workernodes []corev1.Node, nodesDev return true, err } -func getReservedCPUSize(CPU *performancev2.CPU) int { - reservedCPUs, err := cpuset.Parse(string(*CPU.Reserved)) - Expect(err).ToNot(HaveOccurred()) - return reservedCPUs.Size() -} - func getVendorID(ctx context.Context, node corev1.Node, device string) string { cmd := []string{"bash", "-c", fmt.Sprintf("cat /sys/class/net/%s/device/vendor", device)} From 98010d803db06c6c39fc21a21977bc238a0684f6 Mon Sep 17 00:00:00 2001 From: oblau Date: Thu, 2 Jul 2026 10:53:30 +0300 Subject: [PATCH 3/5] refactor(e2e): discover multi-queue NICs once into a baseline map instead of per-test rediscovery - checkDeviceSupport was called independently by every test, re-running ethtool discovery over the tuned pod each time. checkDeviceSetWithReservedCPU then polled that same rediscovery every 5s while waiting for convergence. A slow update and a broken feature produced the same timeout -> Skip. - Split checkDeviceSupport into discoverMultiQueueNICs (per-node NIC enumeration, now via nodes.GetNodeInterfaces/node_inspector instead of tuned-pod exec) and getCombinedChannels (single NIC -> combined channel count). One does discovery, one does the ethtool read; each is reusable on its own. - discoverMultiQueueNICs runs once in BeforeAll into baselineMultiQueueNICs (map[string]map[nodes.NodeInterface]int); BeforeAll Skips the whole suite upfront if none are found. Tests read from this baseline instead of rediscovering. - checkDeviceSetWithReservedCPU -> waitForNICsToMatchReservedCPU: no longer mutates a caller map or rediscovers, just polls the baseline. Also fixes a real bug -- the old function returned success on the first NIC that matched reservedCPUCount, even for tests asserting all supported NICs converge. - getRandomNodeDevice updated for the new map type, returns a NodeInterface instead of a bare string. Drops its defensive empty-name check -- discoverMultiQueueNICs never inserts zero-value entries, unlike the old checkDeviceSupport. - 40543/40545/72051/40668 now pull their target device from the baseline instead of calling checkDeviceSupport themselves. profile updates use `var err error` explicitly to avoid re-shadowing the Describe-scoped profile var now that device is a struct, not a string, everywhere. AI Attribution: AIA Human-AI blend, Content edits, New content, Human-initiated, Reviewed, opus 4.6 high v1.0 --- .../functests/1_performance/netqueues.go | 252 +++++++++--------- 1 file changed, 127 insertions(+), 125 deletions(-) diff --git a/test/e2e/performanceprofile/functests/1_performance/netqueues.go b/test/e2e/performanceprofile/functests/1_performance/netqueues.go index ecbe81a64e..705b855254 100644 --- a/test/e2e/performanceprofile/functests/1_performance/netqueues.go +++ b/test/e2e/performanceprofile/functests/1_performance/netqueues.go @@ -36,6 +36,7 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s var profile, initialProfile *performancev2.PerformanceProfile var tunedConfPath, performanceProfileName string var reservedCPUCount int + var baselineMultiQueueNICs map[string]map[nodes.NodeInterface]int BeforeAll(func() { if discovery.Enabled() && testutils.ProfileNotFound { @@ -79,6 +80,12 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s tunedConfPath = filepath.Join(tunedprofilesDirectory, tunedPaoProfile, "tuned.conf") + By("Discovering multi-queue capable NICs before any profile changes") + baselineMultiQueueNICs = discoverMultiQueueNICs(context.TODO(), workerRTNodes) + if len(baselineMultiQueueNICs) == 0 { + Skip("No multi-queue capable NICs found on worker nodes") + } + By("Ensuring a baseline of UserLevelNetworking=true, no device filter") desiredNet := &performancev2.Net{UserLevelNetworking: ptr.To(true)} if !equality.Semantic.DeepEqual(profile.Spec.Net, desiredNet) { @@ -100,30 +107,26 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s Context("Updating performance profile for netqueues", func() { It("[test_id:40308][crit:high][vendor:cnf-qe@redhat.com][level:acceptance] Network device queues Should be set to the profile's reserved CPUs count", func() { - nodesDevices := make(map[string]map[string]int) By("To all non virtual network devices when no devices are specified under profile.Spec.Net.Devices") - err := checkDeviceSetWithReservedCPU(context.TODO(), workerRTNodes, nodesDevices, reservedCPUCount) + err := waitForNICsToMatchReservedCPU(context.TODO(), workerRTNodes, baselineMultiQueueNICs, reservedCPUCount) if err != nil { Skip("Skipping Test: Unable to set Network queue size to reserved cpu count") } }) It("[test_id:40543] Add interfaceName and verify the interface netqueues are equal to reserved cpus count.", func() { - nodesDevices := make(map[string]map[string]int) - deviceSupport, err := checkDeviceSupport(context.TODO(), workerRTNodes, nodesDevices) - Expect(err).ToNot(HaveOccurred()) - if !deviceSupport { - Skip("Skipping Test: There are no supported Network Devices") - } - nodeName, device := getRandomNodeDevice(nodesDevices) + nodeName, device := getRandomNodeDevice(baselineMultiQueueNICs) + + var err error profile, err = profiles.GetByNodeLabels(testutils.NodeSelectorLabels) Expect(err).ToNot(HaveOccurred()) + By("Enable UserLevelNetworking and add Devices in Profile") profile.Spec.Net = &performancev2.Net{ UserLevelNetworking: ptr.To(true), Devices: []performancev2.Device{ { - InterfaceName: &device, + InterfaceName: &device.Name, }, }, } @@ -143,28 +146,23 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s if err != nil { return false } - return strings.Contains(string(out), device) + return strings.Contains(string(out), device.Name) }, cluster.ComputeTestTimeout(2*time.Minute, RunningOnSingleNode), 5*time.Second).Should(BeTrue(), "could not get a tuned profile set with devices_udev_regex") - nodesDevices = make(map[string]map[string]int) - err = checkDeviceSetWithReservedCPU(context.TODO(), workerRTNodes, nodesDevices, reservedCPUCount) + err = waitForNICsToMatchReservedCPU(context.TODO(), workerRTNodes, baselineMultiQueueNICs, reservedCPUCount) if err != nil { Skip("Skipping Test: Unable to set Network queue size to reserved cpu count") } }) It("[test_id:40545] Verify reserved cpus count is applied to specific supported networking devices using wildcard matches", func() { - nodesDevices := make(map[string]map[string]int) - var device, devicePattern string - deviceSupport, err := checkDeviceSupport(context.TODO(), workerRTNodes, nodesDevices) - Expect(err).ToNot(HaveOccurred()) - if !deviceSupport { - Skip("Skipping Test: There are no supported Network Devices") - } - nodeName, device := getRandomNodeDevice(nodesDevices) - devicePattern = device[:len(device)-1] + "*" + nodeName, device := getRandomNodeDevice(baselineMultiQueueNICs) + devicePattern := device.Name[:len(device.Name)-1] + "*" + + var err error profile, err = profiles.GetByNodeLabels(testutils.NodeSelectorLabels) Expect(err).ToNot(HaveOccurred()) + By("Enable UserLevelNetworking and add Devices in Profile") profile.Spec.Net = &performancev2.Net{ UserLevelNetworking: ptr.To(true), @@ -189,37 +187,35 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s if err != nil { return false } - return strings.Contains(string(out), device) + return strings.Contains(string(out), device.Name) }, cluster.ComputeTestTimeout(2*time.Minute, RunningOnSingleNode), 5*time.Second).Should(BeTrue(), "could not get a tuned profile set with devices_udev_regex") - nodesDevices = make(map[string]map[string]int) - err = checkDeviceSetWithReservedCPU(context.TODO(), workerRTNodes, nodesDevices, reservedCPUCount) + err = waitForNICsToMatchReservedCPU(context.TODO(), workerRTNodes, baselineMultiQueueNICs, reservedCPUCount) if err != nil { Skip("Skipping Test: Unable to set Network queue size to reserved cpu count") } }) It("[test_id:72051] Verify reserved cpus count is applied to all but specific supported networking device using a negative match", func() { - nodesDevices := make(map[string]map[string]int) - var device, devicePattern string - deviceSupport, err := checkDeviceSupport(context.TODO(), workerRTNodes, nodesDevices) - Expect(err).ToNot(HaveOccurred()) - if !deviceSupport { - Skip("Skipping Test: There are no supported Network Devices") - } - // Remove nodes with only one NIC as that cannot be used to check this behavior // this is done by removing the NIC entries to avoid deleting from the map while iterating - for node, nics := range nodesDevices { - if len(nics) < 2 { - nodesDevices[node] = nil + nodesWithMultipleNICs := make(map[string]map[nodes.NodeInterface]int) + for node, nics := range baselineMultiQueueNICs { + if len(nics) >= 2 { + nodesWithMultipleNICs[node] = nics } } + if len(nodesWithMultipleNICs) == 0 { + Skip("No nodes with multiple NICs available to test negative device match") + } + + nodeName, device := getRandomNodeDevice(baselineMultiQueueNICs) + devicePattern := "!" + device.Name - nodeName, device := getRandomNodeDevice(nodesDevices) - devicePattern = "!" + device + var err error profile, err = profiles.GetByNodeLabels(testutils.NodeSelectorLabels) Expect(err).ToNot(HaveOccurred()) + By("Enable UserLevelNetworking and add Devices in Profile") profile.Spec.Net = &performancev2.Net{ UserLevelNetworking: ptr.To(true), @@ -244,34 +240,23 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s if err != nil { return false } - return strings.Contains(string(out), device) + return strings.Contains(string(out), device.Name) }, cluster.ComputeTestTimeout(2*time.Minute, RunningOnSingleNode), 5*time.Second).Should(BeTrue(), "could not get a tuned profile set with devices_udev_regex") - nodesDevices = make(map[string]map[string]int) - err = checkDeviceSetWithReservedCPU(context.TODO(), workerRTNodes, nodesDevices, reservedCPUCount) + err = waitForNICsToMatchReservedCPU(context.TODO(), workerRTNodes, baselineMultiQueueNICs, reservedCPUCount) if err != nil { Skip("Skipping Test: Unable to set Network queue size to reserved cpu count") } - - // After at least one NIC was configured, make sure that the selected NIC was NOT it - - Expect(nodesDevices).To(HaveKey(node.Name)) - Expect(nodesDevices[node.Name]).To(HaveKey(device)) - Expect(nodesDevices[node.Name][device]).ToNot(Equal(reservedCPUCount)) }) It("[test_id:40668] Verify reserved cpu count is added to networking devices matched with vendor and Device id", func() { - nodesDevices := make(map[string]map[string]int) - deviceSupport, err := checkDeviceSupport(context.TODO(), workerRTNodes, nodesDevices) - Expect(err).ToNot(HaveOccurred()) - if !deviceSupport { - Skip("Skipping Test: There are no supported Network Devices") - } - nodeName, device := getRandomNodeDevice(nodesDevices) + nodeName, device := getRandomNodeDevice(baselineMultiQueueNICs) node, err := nodes.GetByName(nodeName) Expect(err).ToNot(HaveOccurred()) - vid := getVendorID(context.TODO(), *node, device) - did := getDeviceID(context.TODO(), *node, device) + + vid := getVendorID(context.TODO(), *node, device.Name) + did := getDeviceID(context.TODO(), *node, device.Name) + profile, err = profiles.GetByNodeLabels(testutils.NodeSelectorLabels) Expect(err).ToNot(HaveOccurred()) @@ -280,7 +265,7 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s UserLevelNetworking: ptr.To(true), Devices: []performancev2.Device{ { - InterfaceName: &device, + InterfaceName: &device.Name, }, { VendorID: &vid, @@ -302,11 +287,10 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s if err != nil { return false } - return strings.Contains(string(out), device) + return strings.Contains(string(out), device.Name) }, cluster.ComputeTestTimeout(2*time.Minute, RunningOnSingleNode), 5*time.Second).Should(BeTrue(), "could not get a tuned profile set with devices_udev_regex") - nodesDevices = make(map[string]map[string]int) - err = checkDeviceSetWithReservedCPU(context.TODO(), workerRTNodes, nodesDevices, reservedCPUCount) + err = waitForNICsToMatchReservedCPU(context.TODO(), workerRTNodes, baselineMultiQueueNICs, reservedCPUCount) if err != nil { Skip("Skipping Test: Unable to set Network queue size to reserved cpu count") } @@ -314,71 +298,94 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s }) }) -// Check a device that supports multiple queues and set with with reserved CPU size exists -func checkDeviceSetWithReservedCPU(ctx context.Context, workerRTNodes []corev1.Node, nodesDevices map[string]map[string]int, reservedCPUCount int) error { - return wait.PollUntilContextTimeout(ctx, 5*time.Second, 90*time.Second, true, func(ctx context.Context) (bool, error) { - deviceSupport, err := checkDeviceSupport(ctx, workerRTNodes, nodesDevices) - Expect(err).ToNot(HaveOccurred()) - if !deviceSupport { - return false, nil - } - for _, devices := range nodesDevices { - for _, size := range devices { - if size == reservedCPUCount { - return true, nil +// waitForNICsToMatchReservedCPU polls the pre-discovered multi-queue NICs until all +// have their combined channel count equal to reservedCPUCount, indicating TuneD has +// applied the net queue configuration. Returns an error on timeout. +func waitForNICsToMatchReservedCPU(ctx context.Context, workerRTNodes []corev1.Node, baselineMultiQueueNICs map[string]map[nodes.NodeInterface]int, reservedCPUCount int) error { + nodesByName := make(map[string]corev1.Node, len(workerRTNodes)) + for _, workerNode := range workerRTNodes { + nodesByName[workerNode.Name] = workerNode + } + err := wait.PollUntilContextTimeout(ctx, 5*time.Second, 3*time.Minute, true, + func(ctx context.Context) (bool, error) { + for nodeName, nodeSupportedNics := range baselineMultiQueueNICs { + node := nodesByName[nodeName] + for supportedNic := range nodeSupportedNics { + channels, err := getCombinedChannels(ctx, node, supportedNic) + if err != nil || channels == 0 { + testlog.Warningf("NIC %s on %s: unexpected error or unavailable: %v", supportedNic.Name, nodeName, err) + return false, nil + } + if channels != reservedCPUCount { + testlog.Infof("not all NICs ready - %s combined(%d) != reserved(%d), retrying (%s)", supportedNic.Name, channels, reservedCPUCount, nodeName) + return false, nil + } } } - } - return false, nil - }) + return true, nil + }) + + return err } -// Check if the device support multiple queues -func checkDeviceSupport(ctx context.Context, workernodes []corev1.Node, nodesDevices map[string]map[string]int) (bool, error) { - cmdGetPhysicalDevices := []string{"find", "/sys/class/net", "-type", "l", "-not", "-lname", "*virtual*", "-printf", "%f "} - var channelCurrentCombined int - var noSupportedDevices = true - var err error +// getCombinedChannels returns the current combined channel count for a NIC, or 0 if unsupported. +func getCombinedChannels(ctx context.Context, node corev1.Node, iface nodes.NodeInterface) (int, error) { + if !iface.Physical { + return 0, nil + } + cmdCombinedChannelsCurrent := []string{"bash", "-c", + fmt.Sprintf("ethtool -l %s | sed -n '/Current hardware settings:/,/Combined:/{s/^Combined:\\s*//p}'", iface.Name)} + out, err := nodes.ExecCommand(ctx, &node, cmdCombinedChannelsCurrent) + if err != nil { + testlog.Warningf("failed to get combined: exec error: %v", err) + return 0, fmt.Errorf("ethtool exec failed: %w", err) + } + // sed extracts the Combined value: either "n/a" (unsupported) or a numeric string + if strings.Contains(string(out), "n/a") { + return 0, nil + } + combinedChannels, err := strconv.Atoi(strings.TrimSpace(string(out))) + if err != nil { + testlog.Warningf("failed to get combined: failed to parse combined channels: %v", err) + return 0, fmt.Errorf("failed to parse combined channels: %w", err) + } + if combinedChannels <= 1 { + testlog.Infof("combined not supported: (combined<=1)") + return 0, nil + } + return combinedChannels, nil +} + +// discoverMultiQueueNICs returns a snapshot of all NICs with combined channels > 1 +// across the given nodes. Result is keyed by node name → NodeInterface → combined channel count. +// Returns an empty map if no qualifying NICs are found; errors are logged and the NIC is skipped. +func discoverMultiQueueNICs(ctx context.Context, workernodes []corev1.Node) map[string]map[nodes.NodeInterface]int { + multiQueueNICs := make(map[string]map[nodes.NodeInterface]int) for _, node := range workernodes { - if nodesDevices[node.Name] == nil { - nodesDevices[node.Name] = make(map[string]int) + interfaces, err := nodes.GetNodeInterfaces(ctx, node) + if err != nil { + testlog.Warningf("Failed to get interfaces on %s: %v", node.Name, err) + continue } - tunedPod := nodes.TunedForNode(&node, RunningOnSingleNode) - phyDevs, err := pods.WaitForPodOutput(ctx, testclient.K8sClient, tunedPod, cmdGetPhysicalDevices) - Expect(err).ToNot(HaveOccurred()) - for _, d := range strings.Split(string(phyDevs), " ") { - if d == "" { + testlog.Infof("Discovering multi-queue NICs on %s", node.Name) + nodeNICs := make(map[nodes.NodeInterface]int) + for _, iface := range interfaces { + channels, err := getCombinedChannels(ctx, node, iface) + if err != nil { + testlog.Warningf("%s: Couldn't get combined, skipping: %v", iface.Name, err) continue } - _, err := pods.WaitForPodOutput(ctx, testclient.K8sClient, tunedPod, []string{"ethtool", "-l", d}) - if err == nil { - cmdCombinedChannelsCurrent := []string{"bash", "-c", - fmt.Sprintf("ethtool -l %s | sed -n '/Current hardware settings:/,/Combined:/{s/^Combined:\\s*//p}'", d)} - out, err := pods.WaitForPodOutput(ctx, testclient.K8sClient, tunedPod, cmdCombinedChannelsCurrent) - Expect(err).ToNot(HaveOccurred()) - if strings.Contains(string(out), "n/a") { - fmt.Printf("Device %s doesn't support multiple queues\n", d) - } else { - channelCurrentCombined, err = strconv.Atoi(strings.TrimSpace(string(out))) - if err != nil { - testlog.Warningf(fmt.Sprintf("unable to retrieve current multi-purpose channels hardware settings for device %s on %s", - d, node.Name)) - } - if channelCurrentCombined == 1 { - fmt.Printf("Device %s doesn't support multiple queues\n", d) - } else { - fmt.Printf("Device %s supports multiple queues\n", d) - nodesDevices[node.Name][d] = channelCurrentCombined - noSupportedDevices = false - } - } + if channels == 0 { + continue } + testlog.Infof("Discovered %s: multi-queue (combined=%d)", iface.Name, channels) + nodeNICs[iface] = channels + } + if len(nodeNICs) > 0 { + multiQueueNICs[node.Name] = nodeNICs } } - if noSupportedDevices { - return false, err - } - return true, err + return multiQueueNICs } func getVendorID(ctx context.Context, node corev1.Node, device string) string { @@ -399,17 +406,12 @@ func getDeviceID(ctx context.Context, node corev1.Node, device string) string { return stdout } -func getRandomNodeDevice(nodesDevices map[string]map[string]int) (string, string) { - node := "" - device := "" - for n := range nodesDevices { - node = n - for d := range nodesDevices[node] { - if d != "" { - device = d - return node, device - } +func getRandomNodeDevice(multiQueueNICs map[string]map[nodes.NodeInterface]int) (string, nodes.NodeInterface) { + Expect(multiQueueNICs).ToNot(BeEmpty(), "getRandomNodeDevice: multiQueueNICs map is empty") + for node, nics := range multiQueueNICs { + for iface := range nics { + return node, iface } } - return node, device + return "", nodes.NodeInterface{} } From d159a8ef586c1c215ef067fba185307f74be18c7 Mon Sep 17 00:00:00 2001 From: oblau Date: Thu, 2 Jul 2026 13:29:30 +0300 Subject: [PATCH 4/5] refactor(e2e): rework netqueue tests around a pre-discovered NIC baseline - checkDeviceSupport split into discoverMultiQueueNICs (enumeration) and getCombinedChannels (per-NIC ethtool parsing). Discovery now runs once in BeforeAll into baselineMultiQueueNICs, Skipping the whole suite upfront if none are found, instead of each test re-discovering and skipping individually. - checkDeviceSetWithReservedCPU -> waitForNICsToMatchReservedCPU: takes a NIC map to poll instead of rediscovering. Also fixes a bug where it returned success on the first matching NIC instead of waiting for all of them to converge. - getRandomNodeDevice: updated for the new map type, returns a NodeInterface, drops its now-unneeded empty-name guard. - Each test builds its own scoped NIC map (targetNICs/matchedNICs/ expectedNICs) instead of polling the full baseline, so convergence checks validate only the NICs that test actually touches. Per-test checkDeviceSupport+Skip guards removed, BeforeAll already guarantees them. - Skip-on-error replaced with Expect(err).ToNot(HaveOccurred()) throughout. - profile.Spec.Net setup now only sets .Devices (UserLevelNetworking=true comes from the BeforeAll baseline). - var err error added before some profile fetches so profile, err = ... assigns the Describe-scoped var instead of shadowing it. - nodes.GetByName moved before scoped-map-building/profile update in every test, to keep the exec call out of the post-update ethtool blackout window. - 40545: wildcard check now searches for the derived udev-regex form ("iface.*") instead of the literal device name, since tuned stores wildcards as regex in devices_udev_regex. - 72051: pick device only from nodes with >=2 NICs, a single-NIC node can't demonstrate negation. Restored the negative assertion (pre/post combined-channel comparison on the negated NIC) lost when checkDeviceSupport was removed. Tuned-config check now searches for the negated pattern ("!iface") instead of the bare device name, to avoid a false positive from a leftover config. - 40668: tuned-config check upgraded to a 3-way match (interface name + vendor ID + device ID); it previously only checked the interface name. AI Attribution: AIA Human-AI blend, Content edits, New content, Human-initiated, Reviewed, opus 4.6 high v1.0 --- .../functests/1_performance/netqueues.go | 180 +++++++++++------- 1 file changed, 112 insertions(+), 68 deletions(-) diff --git a/test/e2e/performanceprofile/functests/1_performance/netqueues.go b/test/e2e/performanceprofile/functests/1_performance/netqueues.go index 705b855254..07e94c34e5 100644 --- a/test/e2e/performanceprofile/functests/1_performance/netqueues.go +++ b/test/e2e/performanceprofile/functests/1_performance/netqueues.go @@ -109,9 +109,7 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s It("[test_id:40308][crit:high][vendor:cnf-qe@redhat.com][level:acceptance] Network device queues Should be set to the profile's reserved CPUs count", func() { By("To all non virtual network devices when no devices are specified under profile.Spec.Net.Devices") err := waitForNICsToMatchReservedCPU(context.TODO(), workerRTNodes, baselineMultiQueueNICs, reservedCPUCount) - if err != nil { - Skip("Skipping Test: Unable to set Network queue size to reserved cpu count") - } + Expect(err).ToNot(HaveOccurred(), "no NIC matched reserved CPU count %d within timeout", reservedCPUCount) }) It("[test_id:40543] Add interfaceName and verify the interface netqueues are equal to reserved cpus count.", func() { @@ -121,15 +119,19 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s profile, err = profiles.GetByNodeLabels(testutils.NodeSelectorLabels) Expect(err).ToNot(HaveOccurred()) - By("Enable UserLevelNetworking and add Devices in Profile") - profile.Spec.Net = &performancev2.Net{ - UserLevelNetworking: ptr.To(true), - Devices: []performancev2.Device{ - { - InterfaceName: &device.Name, - }, - }, + node, err := nodes.GetByName(nodeName) + Expect(err).ToNot(HaveOccurred()) + + By("Building target NIC map for selected device") + targetNICs := map[string]map[nodes.NodeInterface]int{ + nodeName: {device: baselineMultiQueueNICs[nodeName][device]}, + } + + By("Adding device filter to profile") + profile.Spec.Net.Devices = []performancev2.Device{ + {InterfaceName: &device.Name}, } + By("Updating the performance profile") profiles.UpdateWithRetry(profile) @@ -137,8 +139,6 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s tunedCmd := []string{"bash", "-c", fmt.Sprintf("grep devices_udev_regex %s", tunedConfPath)} - node, err := nodes.GetByName(nodeName) - Expect(err).ToNot(HaveOccurred()) tunedPod := nodes.TunedForNode(node, RunningOnSingleNode) Eventually(func() bool { @@ -149,37 +149,50 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s return strings.Contains(string(out), device.Name) }, cluster.ComputeTestTimeout(2*time.Minute, RunningOnSingleNode), 5*time.Second).Should(BeTrue(), "could not get a tuned profile set with devices_udev_regex") - err = waitForNICsToMatchReservedCPU(context.TODO(), workerRTNodes, baselineMultiQueueNICs, reservedCPUCount) - if err != nil { - Skip("Skipping Test: Unable to set Network queue size to reserved cpu count") - } + err = waitForNICsToMatchReservedCPU(context.TODO(), workerRTNodes, targetNICs, reservedCPUCount) + Expect(err).ToNot(HaveOccurred(), "NIC %s on %s: combined channels did not converge to %d", device.Name, nodeName, reservedCPUCount) }) It("[test_id:40545] Verify reserved cpus count is applied to specific supported networking devices using wildcard matches", func() { nodeName, device := getRandomNodeDevice(baselineMultiQueueNICs) + devicePattern := device.Name[:len(device.Name)-1] + "*" + expectedUdevRegex := device.Name[:len(device.Name)-1] + ".*" var err error profile, err = profiles.GetByNodeLabels(testutils.NodeSelectorLabels) Expect(err).ToNot(HaveOccurred()) - By("Enable UserLevelNetworking and add Devices in Profile") - profile.Spec.Net = &performancev2.Net{ - UserLevelNetworking: ptr.To(true), - Devices: []performancev2.Device{ - { - InterfaceName: &devicePattern, - }, - }, + node, err := nodes.GetByName(nodeName) + Expect(err).ToNot(HaveOccurred()) + + By("Building matched NIC map for wildcard pattern") + matchedNICs := make(map[string]map[nodes.NodeInterface]int) + for nodeKey, nics := range baselineMultiQueueNICs { + for nic, channels := range nics { + matched, err := filepath.Match(devicePattern, nic.Name) + Expect(err).ToNot(HaveOccurred()) + if matched { + if matchedNICs[nodeKey] == nil { + matchedNICs[nodeKey] = make(map[nodes.NodeInterface]int) + } + matchedNICs[nodeKey][nic] = channels + } + } + } + Expect(matchedNICs).ToNot(BeEmpty(), "Unexpected: no baseline NICs match pattern %q", devicePattern) + + By("Adding wildcard device filter to profile") + profile.Spec.Net.Devices = []performancev2.Device{ + {InterfaceName: &devicePattern}, } + profiles.UpdateWithRetry(profile) //Verify the tuned profile is created on the worker-cnf nodes: tunedCmd := []string{"bash", "-c", fmt.Sprintf("grep devices_udev_regex %s", tunedConfPath)} - node, err := nodes.GetByName(nodeName) - Expect(err).ToNot(HaveOccurred()) tunedPod := nodes.TunedForNode(node, RunningOnSingleNode) Eventually(func() bool { @@ -187,13 +200,11 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s if err != nil { return false } - return strings.Contains(string(out), device.Name) - }, cluster.ComputeTestTimeout(2*time.Minute, RunningOnSingleNode), 5*time.Second).Should(BeTrue(), "could not get a tuned profile set with devices_udev_regex") + return strings.Contains(string(out), expectedUdevRegex) + }, cluster.ComputeTestTimeout(2*time.Minute, RunningOnSingleNode), 5*time.Second).Should(BeTrue(), "tuned config does not contain %q", expectedUdevRegex) - err = waitForNICsToMatchReservedCPU(context.TODO(), workerRTNodes, baselineMultiQueueNICs, reservedCPUCount) - if err != nil { - Skip("Skipping Test: Unable to set Network queue size to reserved cpu count") - } + err = waitForNICsToMatchReservedCPU(context.TODO(), workerRTNodes, matchedNICs, reservedCPUCount) + Expect(err).ToNot(HaveOccurred(), "NICs matching %q did not converge to reserved CPU count %d", devicePattern, reservedCPUCount) }) It("[test_id:72051] Verify reserved cpus count is applied to all but specific supported networking device using a negative match", func() { @@ -209,21 +220,38 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s Skip("No nodes with multiple NICs available to test negative device match") } - nodeName, device := getRandomNodeDevice(baselineMultiQueueNICs) + nodeName, device := getRandomNodeDevice(nodesWithMultipleNICs) devicePattern := "!" + device.Name var err error profile, err = profiles.GetByNodeLabels(testutils.NodeSelectorLabels) Expect(err).ToNot(HaveOccurred()) + node, err := nodes.GetByName(nodeName) + Expect(err).ToNot(HaveOccurred()) + + By("Building expected NIC map excluding negated device") + expectedNICs := make(map[string]map[nodes.NodeInterface]int) + for nodeKey, nics := range baselineMultiQueueNICs { + for nic, channels := range nics { + if nic.Name == device.Name { + continue + } + if expectedNICs[nodeKey] == nil { + expectedNICs[nodeKey] = make(map[nodes.NodeInterface]int) + } + expectedNICs[nodeKey][nic] = channels + } + } + Expect(expectedNICs).ToNot(BeEmpty(), "Unexpected: no NICs remain after excluding %s", device.Name) + + By("Recording pre-update combined channels for the negated NIC") + preUpdateCombined, err := getCombinedChannels(context.TODO(), *node, device) + Expect(err).ToNot(HaveOccurred()) + By("Enable UserLevelNetworking and add Devices in Profile") - profile.Spec.Net = &performancev2.Net{ - UserLevelNetworking: ptr.To(true), - Devices: []performancev2.Device{ - { - InterfaceName: &devicePattern, - }, - }, + profile.Spec.Net.Devices = []performancev2.Device{ + {InterfaceName: &devicePattern}, } profiles.UpdateWithRetry(profile) @@ -231,8 +259,6 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s tunedCmd := []string{"bash", "-c", fmt.Sprintf("grep devices_udev_regex %s", tunedConfPath)} - node, err := nodes.GetByName(nodeName) - Expect(err).ToNot(HaveOccurred()) tunedPod := nodes.TunedForNode(node, RunningOnSingleNode) Eventually(func() bool { @@ -240,13 +266,19 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s if err != nil { return false } - return strings.Contains(string(out), device.Name) - }, cluster.ComputeTestTimeout(2*time.Minute, RunningOnSingleNode), 5*time.Second).Should(BeTrue(), "could not get a tuned profile set with devices_udev_regex") - - err = waitForNICsToMatchReservedCPU(context.TODO(), workerRTNodes, baselineMultiQueueNICs, reservedCPUCount) - if err != nil { - Skip("Skipping Test: Unable to set Network queue size to reserved cpu count") - } + return strings.Contains(string(out), devicePattern) + }, cluster.ComputeTestTimeout(2*time.Minute, RunningOnSingleNode), 5*time.Second).Should(BeTrue(), "tuned config does not contain %q", devicePattern) + + err = waitForNICsToMatchReservedCPU(context.TODO(), workerRTNodes, expectedNICs, reservedCPUCount) + Expect(err).ToNot(HaveOccurred(), "NICs (excluding %s) did not converge to reserved CPU count %d", device.Name, reservedCPUCount) + + By("Verifying the negated NIC's channels were left untouched by the update") + var combined int + Eventually(func() (err error) { + combined, err = getCombinedChannels(context.TODO(), *node, device) + return err + }, 120*time.Second, 5*time.Second).Should(Succeed(), "failed to read combined channels for %s - retrying", device.Name) + Expect(combined).To(Equal(preUpdateCombined), "%s should remain unchanged at pre-update combined=%d, got %d", device.Name, preUpdateCombined, combined) }) It("[test_id:40668] Verify reserved cpu count is added to networking devices matched with vendor and Device id", func() { @@ -260,17 +292,33 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s profile, err = profiles.GetByNodeLabels(testutils.NodeSelectorLabels) Expect(err).ToNot(HaveOccurred()) - By("Enable UserLevelNetworking and add DeviceID, VendorID and Interface in Profile") - profile.Spec.Net = &performancev2.Net{ - UserLevelNetworking: ptr.To(true), - Devices: []performancev2.Device{ - { - InterfaceName: &device.Name, - }, - { - VendorID: &vid, - DeviceID: &did, - }, + node, err = nodes.GetByName(nodeName) + Expect(err).ToNot(HaveOccurred()) + + By("Building vendor+device ID matched NIC map before profile update (exec calls need node reachable)") + matchedNICs := make(map[string]map[nodes.NodeInterface]int) + for _, wn := range workerRTNodes { + for nic, channels := range baselineMultiQueueNICs[wn.Name] { + nicVid := getVendorID(context.TODO(), wn, nic.Name) + nicDid := getDeviceID(context.TODO(), wn, nic.Name) + if nicVid == vid && nicDid == did { + if matchedNICs[wn.Name] == nil { + matchedNICs[wn.Name] = make(map[nodes.NodeInterface]int) + } + matchedNICs[wn.Name][nic] = channels + } + } + } + Expect(matchedNICs).ToNot(BeEmpty(), "no baseline NICs match vendorID=%s deviceID=%s", vid, did) + + By("Adding DeviceID, VendorID and Interface in Profile") + profile.Spec.Net.Devices = []performancev2.Device{ + { + InterfaceName: &device.Name, + }, + { + VendorID: &vid, + DeviceID: &did, }, } profiles.UpdateWithRetry(profile) @@ -279,21 +327,17 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s tunedCmd := []string{"bash", "-c", fmt.Sprintf("grep devices_udev_regex %s", tunedConfPath)} - node, err = nodes.GetByName(nodeName) - Expect(err).ToNot(HaveOccurred()) tunedPod := nodes.TunedForNode(node, RunningOnSingleNode) Eventually(func() bool { out, err := pods.WaitForPodOutput(context.TODO(), testclient.K8sClient, tunedPod, tunedCmd) if err != nil { return false } - return strings.Contains(string(out), device.Name) + return strings.Contains(string(out), device.Name) && strings.Contains(string(out), vid) && strings.Contains(string(out), did) }, cluster.ComputeTestTimeout(2*time.Minute, RunningOnSingleNode), 5*time.Second).Should(BeTrue(), "could not get a tuned profile set with devices_udev_regex") - err = waitForNICsToMatchReservedCPU(context.TODO(), workerRTNodes, baselineMultiQueueNICs, reservedCPUCount) - if err != nil { - Skip("Skipping Test: Unable to set Network queue size to reserved cpu count") - } + err = waitForNICsToMatchReservedCPU(context.TODO(), workerRTNodes, matchedNICs, reservedCPUCount) + Expect(err).ToNot(HaveOccurred(), "NICs matching vendorID=%s deviceID=%s did not converge to reserved CPU count %d", vid, did, reservedCPUCount) }) }) }) From f21c07faa1c74dad7f9affa3c04b312693bb898b Mon Sep 17 00:00:00 2001 From: oblau Date: Thu, 2 Jul 2026 14:18:21 +0300 Subject: [PATCH 5/5] refactor(e2e): unify Ginkgo step labels and logging across netqueue tests - Converted remaining plain `//` comments to By() calls; unified the 4 tests' tuned-config-check block (identical By text, no blank lines) -- they run the same grep, differing only in what they search for. - Added a By() before each test's final convergence check (previously unlabeled); all now end in "converged to reserved CPU count". - Added testlog.Infof after each getRandomNodeDevice call, logging which NIC/node the run picked. - 40545/40668: added missing By("Updating the performance profile"), matching 40543. 72051: split its stale "Enable ULN..." By into the same two-step shape as the other tests. - 40668: removed a redundant duplicate nodes.GetByName call. - AfterAll now logs the profile diff before reverting, mirroring BeforeAll's equivalent branch. - Added a top-of-file comment on the ARM ethtool -L blackout window, and a doc comment on getRandomNodeDevice explaining its random-pick trick. AI Attribution: AIA Human-AI blend, Content edits, New content, Human-initiated, Reviewed, opus 4.6 high v1.0 --- .../functests/1_performance/netqueues.go | 54 +++++++++++-------- 1 file changed, 33 insertions(+), 21 deletions(-) diff --git a/test/e2e/performanceprofile/functests/1_performance/netqueues.go b/test/e2e/performanceprofile/functests/1_performance/netqueues.go index 07e94c34e5..76bdccd547 100644 --- a/test/e2e/performanceprofile/functests/1_performance/netqueues.go +++ b/test/e2e/performanceprofile/functests/1_performance/netqueues.go @@ -31,6 +31,10 @@ import ( const tunedprofilesDirectory string = "/var/lib/ocp-tuned/profiles" +// When TuneD applies ethtool -L on the br-ex uplink NIC, the node loses +// connectivity for ~30s. Any exec into the node during that window will +// time out, so node-reaching calls must either be inside a poll/Eventually +// loop or placed before the profile update that triggers the blackout. var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(string(label.Tier1)), func() { var workerRTNodes []corev1.Node var profile, initialProfile *performancev2.PerformanceProfile @@ -65,7 +69,7 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s reservedCPUCount = reservedCPUs.Size() tunedPaoProfile := fmt.Sprintf("openshift-node-performance-%s", performanceProfileName) - //Verify the tuned profile is created on the worker-cnf nodes: + By("Verify the tuned profile is created on the worker-cnf nodes") // direct the error to /dev/null on purpose because tuneD always shows the following error: // "Cannot talk to TuneD daemon via DBus. Is TuneD daemon running?" // Which causes the test to fail, but it's a false-positive @@ -99,6 +103,7 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s currentProfile, err := profiles.GetByNodeLabels(testutils.NodeSelectorLabels) Expect(err).ToNot(HaveOccurred()) if !equality.Semantic.DeepEqual(currentProfile.Spec, initialProfile.Spec) { + testlog.Infof("current profile.Spec differs from initial, reverting: current=%+v", currentProfile.Spec) By("Reverting to initial Profile") currentProfile.Spec = initialProfile.Spec profiles.UpdateWithRetry(currentProfile) @@ -107,13 +112,14 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s Context("Updating performance profile for netqueues", func() { It("[test_id:40308][crit:high][vendor:cnf-qe@redhat.com][level:acceptance] Network device queues Should be set to the profile's reserved CPUs count", func() { - By("To all non virtual network devices when no devices are specified under profile.Spec.Net.Devices") + By("Verifying all non-virtual NICs converged to reserved CPU count") err := waitForNICsToMatchReservedCPU(context.TODO(), workerRTNodes, baselineMultiQueueNICs, reservedCPUCount) Expect(err).ToNot(HaveOccurred(), "no NIC matched reserved CPU count %d within timeout", reservedCPUCount) }) It("[test_id:40543] Add interfaceName and verify the interface netqueues are equal to reserved cpus count.", func() { nodeName, device := getRandomNodeDevice(baselineMultiQueueNICs) + testlog.Infof("Selected NIC %s on node %s (combined=%d)", device.Name, nodeName, baselineMultiQueueNICs[nodeName][device]) var err error profile, err = profiles.GetByNodeLabels(testutils.NodeSelectorLabels) @@ -135,12 +141,10 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s By("Updating the performance profile") profiles.UpdateWithRetry(profile) - //Verify the tuned profile is created on the worker-cnf nodes: + By("Verify the tuned profile is created on the worker-cnf nodes") tunedCmd := []string{"bash", "-c", fmt.Sprintf("grep devices_udev_regex %s", tunedConfPath)} - tunedPod := nodes.TunedForNode(node, RunningOnSingleNode) - Eventually(func() bool { out, err := pods.WaitForPodOutput(context.TODO(), testclient.K8sClient, tunedPod, tunedCmd) if err != nil { @@ -149,15 +153,16 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s return strings.Contains(string(out), device.Name) }, cluster.ComputeTestTimeout(2*time.Minute, RunningOnSingleNode), 5*time.Second).Should(BeTrue(), "could not get a tuned profile set with devices_udev_regex") + By("Verifying the configured NIC's combined channels converged to reserved CPU count") err = waitForNICsToMatchReservedCPU(context.TODO(), workerRTNodes, targetNICs, reservedCPUCount) Expect(err).ToNot(HaveOccurred(), "NIC %s on %s: combined channels did not converge to %d", device.Name, nodeName, reservedCPUCount) }) It("[test_id:40545] Verify reserved cpus count is applied to specific supported networking devices using wildcard matches", func() { nodeName, device := getRandomNodeDevice(baselineMultiQueueNICs) - devicePattern := device.Name[:len(device.Name)-1] + "*" expectedUdevRegex := device.Name[:len(device.Name)-1] + ".*" + testlog.Infof("Selected NIC %s on node %s, wildcard pattern %q, expected udev regex %q", device.Name, nodeName, devicePattern, expectedUdevRegex) var err error profile, err = profiles.GetByNodeLabels(testutils.NodeSelectorLabels) @@ -187,14 +192,13 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s {InterfaceName: &devicePattern}, } + By("Updating the performance profile") profiles.UpdateWithRetry(profile) - //Verify the tuned profile is created on the worker-cnf nodes: + By("Verify the tuned profile is created on the worker-cnf nodes") tunedCmd := []string{"bash", "-c", fmt.Sprintf("grep devices_udev_regex %s", tunedConfPath)} - tunedPod := nodes.TunedForNode(node, RunningOnSingleNode) - Eventually(func() bool { out, err := pods.WaitForPodOutput(context.TODO(), testclient.K8sClient, tunedPod, tunedCmd) if err != nil { @@ -203,6 +207,7 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s return strings.Contains(string(out), expectedUdevRegex) }, cluster.ComputeTestTimeout(2*time.Minute, RunningOnSingleNode), 5*time.Second).Should(BeTrue(), "tuned config does not contain %q", expectedUdevRegex) + By("Verifying all NICs matching the wildcard converged to reserved CPU count") err = waitForNICsToMatchReservedCPU(context.TODO(), workerRTNodes, matchedNICs, reservedCPUCount) Expect(err).ToNot(HaveOccurred(), "NICs matching %q did not converge to reserved CPU count %d", devicePattern, reservedCPUCount) }) @@ -222,6 +227,7 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s nodeName, device := getRandomNodeDevice(nodesWithMultipleNICs) devicePattern := "!" + device.Name + testlog.Infof("Selected NIC %s on node %s, negated pattern %q", device.Name, nodeName, devicePattern) var err error profile, err = profiles.GetByNodeLabels(testutils.NodeSelectorLabels) @@ -249,18 +255,18 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s preUpdateCombined, err := getCombinedChannels(context.TODO(), *node, device) Expect(err).ToNot(HaveOccurred()) - By("Enable UserLevelNetworking and add Devices in Profile") + By("Adding negated device filter to profile") profile.Spec.Net.Devices = []performancev2.Device{ {InterfaceName: &devicePattern}, } + + By("Updating the performance profile") profiles.UpdateWithRetry(profile) - //Verify the tuned profile is created on the worker-cnf nodes: + By("Verify the tuned profile is created on the worker-cnf nodes") tunedCmd := []string{"bash", "-c", fmt.Sprintf("grep devices_udev_regex %s", tunedConfPath)} - tunedPod := nodes.TunedForNode(node, RunningOnSingleNode) - Eventually(func() bool { out, err := pods.WaitForPodOutput(context.TODO(), testclient.K8sClient, tunedPod, tunedCmd) if err != nil { @@ -269,6 +275,7 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s return strings.Contains(string(out), devicePattern) }, cluster.ComputeTestTimeout(2*time.Minute, RunningOnSingleNode), 5*time.Second).Should(BeTrue(), "tuned config does not contain %q", devicePattern) + By("Verifying all NICs except the negated one converged to reserved CPU count") err = waitForNICsToMatchReservedCPU(context.TODO(), workerRTNodes, expectedNICs, reservedCPUCount) Expect(err).ToNot(HaveOccurred(), "NICs (excluding %s) did not converge to reserved CPU count %d", device.Name, reservedCPUCount) @@ -283,18 +290,18 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s It("[test_id:40668] Verify reserved cpu count is added to networking devices matched with vendor and Device id", func() { nodeName, device := getRandomNodeDevice(baselineMultiQueueNICs) - node, err := nodes.GetByName(nodeName) - Expect(err).ToNot(HaveOccurred()) - - vid := getVendorID(context.TODO(), *node, device.Name) - did := getDeviceID(context.TODO(), *node, device.Name) + var err error profile, err = profiles.GetByNodeLabels(testutils.NodeSelectorLabels) Expect(err).ToNot(HaveOccurred()) - node, err = nodes.GetByName(nodeName) + node, err := nodes.GetByName(nodeName) Expect(err).ToNot(HaveOccurred()) + vid := getVendorID(context.TODO(), *node, device.Name) + did := getDeviceID(context.TODO(), *node, device.Name) + testlog.Infof("Selected NIC %s on node %s, vendorID=%s, deviceID=%s", device.Name, nodeName, vid, did) + By("Building vendor+device ID matched NIC map before profile update (exec calls need node reachable)") matchedNICs := make(map[string]map[nodes.NodeInterface]int) for _, wn := range workerRTNodes { @@ -321,12 +328,13 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s DeviceID: &did, }, } + + By("Updating the performance profile") profiles.UpdateWithRetry(profile) - //Verify the tuned profile is created on the worker-cnf nodes: + By("Verify the tuned profile is created on the worker-cnf nodes") tunedCmd := []string{"bash", "-c", fmt.Sprintf("grep devices_udev_regex %s", tunedConfPath)} - tunedPod := nodes.TunedForNode(node, RunningOnSingleNode) Eventually(func() bool { out, err := pods.WaitForPodOutput(context.TODO(), testclient.K8sClient, tunedPod, tunedCmd) @@ -336,6 +344,7 @@ var _ = Describe("[ref_id: 40307][pao]Resizing Network Queues", Ordered, Label(s return strings.Contains(string(out), device.Name) && strings.Contains(string(out), vid) && strings.Contains(string(out), did) }, cluster.ComputeTestTimeout(2*time.Minute, RunningOnSingleNode), 5*time.Second).Should(BeTrue(), "could not get a tuned profile set with devices_udev_regex") + By("Verifying all NICs matching vendor+device ID converged to reserved CPU count") err = waitForNICsToMatchReservedCPU(context.TODO(), workerRTNodes, matchedNICs, reservedCPUCount) Expect(err).ToNot(HaveOccurred(), "NICs matching vendorID=%s deviceID=%s did not converge to reserved CPU count %d", vid, did, reservedCPUCount) }) @@ -450,6 +459,9 @@ func getDeviceID(ctx context.Context, node corev1.Node, device string) string { return stdout } +// getRandomNodeDevice returns an arbitrary (node name, NIC) pair from a multi-queue +// NIC map. Go randomizes map iteration order, so returning on the first entry is +// an easy way to pick a random one without importing math/rand. func getRandomNodeDevice(multiQueueNICs map[string]map[nodes.NodeInterface]int) (string, nodes.NodeInterface) { Expect(multiQueueNICs).ToNot(BeEmpty(), "getRandomNodeDevice: multiQueueNICs map is empty") for node, nics := range multiQueueNICs {