diff --git a/test/e2e/performanceprofile/functests/6_mustgather_testing/mustgather.go b/test/e2e/performanceprofile/functests/6_mustgather_testing/mustgather.go index 706b6bf7d2..80a6aa644e 100644 --- a/test/e2e/performanceprofile/functests/6_mustgather_testing/mustgather.go +++ b/test/e2e/performanceprofile/functests/6_mustgather_testing/mustgather.go @@ -10,6 +10,7 @@ import ( "strings" machineconfigv1 "github.com/openshift/api/machineconfiguration/v1" + "github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/controller/performanceprofile/components" testutils "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils" testclient "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/client" hypershiftutils "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/hypershift" @@ -61,16 +62,18 @@ var _ = Describe("[rfe_id: 50649] Performance Addon Operator Must Gather", Label if profile == nil { Skip("No Performance Profile found") } - //replace peformance.yaml for profile.Name when data is generated in the node + profileName := profile.Name + kubeletConfigName := components.GetComponentName(profileName, components.ComponentNamePrefix) + tunedName := components.GetComponentName(profileName, components.ProfileNamePerformance) ClusterSpecificFiles := []string{ - "cluster-scoped-resources/performance.openshift.io/performanceprofiles/performance.yaml", - "cluster-scoped-resources/machineconfiguration.openshift.io/kubeletconfigs/performance-performance.yaml", - "namespaces/openshift-cluster-node-tuning-operator/tuned.openshift.io/tuneds/openshift-node-performance-performance.yaml", + fmt.Sprintf("cluster-scoped-resources/performance.openshift.io/performanceprofiles/%s.yaml", profileName), + fmt.Sprintf("cluster-scoped-resources/machineconfiguration.openshift.io/kubeletconfigs/%s.yaml", kubeletConfigName), + fmt.Sprintf("namespaces/openshift-cluster-node-tuning-operator/tuned.openshift.io/tuneds/%s.yaml", tunedName), } // On a hypershift env, the tuned file name has an indentifier in the end if hypershiftutils.IsHypershiftCluster() { ClusterSpecificFiles = []string{ - "namespaces/openshift-cluster-node-tuning-operator/tuned.openshift.io/tuneds/openshift-node-performance-performance-*.yaml", + fmt.Sprintf("namespaces/openshift-cluster-node-tuning-operator/tuned.openshift.io/tuneds/%s-*.yaml", tunedName), } } By(fmt.Sprintf("Checking Folder: %q\n", mgContentFolder)) diff --git a/test/e2e/performanceprofile/functests/7_performance_kubelet_node/cgroups.go b/test/e2e/performanceprofile/functests/7_performance_kubelet_node/cgroups.go index 029acc1ceb..b1131d156f 100644 --- a/test/e2e/performanceprofile/functests/7_performance_kubelet_node/cgroups.go +++ b/test/e2e/performanceprofile/functests/7_performance_kubelet_node/cgroups.go @@ -3,7 +3,6 @@ package __performance_kubelet_node_test import ( "bytes" "context" - "encoding/json" "fmt" "path/filepath" "strconv" @@ -33,9 +32,7 @@ import ( testlog "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/log" "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/nodes" "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/pods" - "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/poolname" "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/profiles" - "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/profilesupdate" "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/systemd" ) @@ -58,8 +55,7 @@ var _ = Describe("[performance] Cgroups and affinity", Ordered, Label(string(lab isolatedCPUSet cpuset.CPUSet workerRTNode *corev1.Node workerRTNodes []corev1.Node - profile, initialProfile *performancev2.PerformanceProfile - poolName string + profile *performancev2.PerformanceProfile ovsSliceCgroup string ctx context.Context = context.Background() ovsSystemdServices []string @@ -89,8 +85,6 @@ var _ = Describe("[performance] Cgroups and affinity", Ordered, Label(string(lab profile, err = profiles.GetByNodeLabels(testutils.NodeSelectorLabels) Expect(err).ToNot(HaveOccurred()) - poolName = poolname.GetByProfile(ctx, profile) - isCgroupV2, err = cgroup.IsVersion2(ctx, testclient.DataPlaneClient) Expect(err).ToNot(HaveOccurred()) @@ -160,56 +154,19 @@ var _ = Describe("[performance] Cgroups and affinity", Ordered, Label(string(lab }) - Context("[Performance Profile Modified]", Label(string(label.Tier1)), func() { - BeforeEach(func() { - initialProfile = profile.DeepCopy() - }) + Context("[Node Reboot]", Label(string(label.Tier1)), func() { It("[test_id:64099] Activation file doesn't get deleted", func() { - policy := "best-effort" - // Need to make some changes to pp , causing system reboot - // and check if activation files is modified or deleted - profile, err := profiles.GetByNodeLabels(testutils.NodeSelectorLabels) - Expect(err).ToNot(HaveOccurred(), "Unable to fetch latest performance profile") - currentPolicy := profile.Spec.NUMA.TopologyPolicy - if *currentPolicy == "best-effort" { - policy = "restricted" - } - profile.Spec.NUMA = &performancev2.NUMA{ - TopologyPolicy: &policy, - } - By("Updating the performance profile") - profiles.UpdateWithRetry(profile) - - By(fmt.Sprintf("Applying changes in performance profile and waiting until %s will start updating", poolName)) - profilesupdate.WaitForTuningUpdating(ctx, profile) - - By(fmt.Sprintf("Waiting when %s finishes updates", poolName)) - profilesupdate.WaitForTuningUpdated(ctx, profile) + By(fmt.Sprintf("Rebooting the worker node %q", workerRTNode.Name)) + _, _ = nodes.ExecCommand(ctx, workerRTNode, []string{"sh", "-c", "chroot /rootfs systemctl reboot"}) + nodes.WaitForNotReadyOrFail("Reboot", workerRTNode.Name, 10*time.Minute, 30*time.Second) + nodes.WaitForReadyOrFail("Reboot", workerRTNode.Name, 10*time.Minute, 30*time.Second) By("Checking Activation file") cmd := []string{"ls", activation_file} - for _, node := range workerRTNodes { - output, err := nodes.ExecCommand(context.TODO(), &node, cmd) - Expect(err).ToNot(HaveOccurred(), "file %s doesn't exist ", activation_file) - out := testutils.ToString(output) - Expect(out).To(Equal(activation_file)) - } - }) - AfterEach(func() { - By("Reverting the Profile") - profile, err := profiles.GetByNodeLabels(testutils.NodeSelectorLabels) - Expect(err).ToNot(HaveOccurred()) - currentSpec, _ := json.Marshal(profile.Spec) - spec, _ := json.Marshal(initialProfile.Spec) - if !bytes.Equal(currentSpec, spec) { - profiles.UpdateWithRetry(initialProfile) - - By(fmt.Sprintf("Applying changes in performance profile and waiting until %s will start updating", poolName)) - profilesupdate.WaitForTuningUpdating(ctx, profile) - - By(fmt.Sprintf("Waiting when %s finishes updates", poolName)) - profilesupdate.WaitForTuningUpdated(ctx, profile) - } + output, err := nodes.ExecCommand(context.TODO(), workerRTNode, cmd) + Expect(err).ToNot(HaveOccurred(), "file %s doesn't exist", activation_file) + out := testutils.ToString(output) + Expect(out).To(Equal(activation_file)) }) }) }) diff --git a/test/e2e/performanceprofile/functests/7_performance_kubelet_node/kubelet.go b/test/e2e/performanceprofile/functests/7_performance_kubelet_node/kubelet.go index 6f37f2724f..cdc2a1ba3f 100644 --- a/test/e2e/performanceprofile/functests/7_performance_kubelet_node/kubelet.go +++ b/test/e2e/performanceprofile/functests/7_performance_kubelet_node/kubelet.go @@ -2,7 +2,6 @@ package __performance_kubelet_node_test import ( "context" - "encoding/json" "fmt" "strconv" "strings" @@ -24,6 +23,7 @@ import ( testutils "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils" testclient "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/client" "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/hypershift" + "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/infrastructure" "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/label" "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/nodes" "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/poolname" @@ -41,13 +41,11 @@ var _ = Describe("[ref_id: 45487][performance]additional kubelet arguments", Ord ) testutils.CustomBeforeAll(func() { - // TODO: The code here is broken as it masks workerRTNodes defined above - // and the value of workerRTNodes is never used. Keep the linter happy for now - // and remove the ineffectual assignment of workerRTNodes, err = nodes.MatchingOptionalSelector(workerRTNodes) below. - workerRTNodes, err := nodes.GetByLabels(testutils.NodeSelectorLabels) + var err error + workerRTNodes, err = nodes.GetByLabels(testutils.NodeSelectorLabels) Expect(err).ToNot(HaveOccurred()) - _, err = nodes.MatchingOptionalSelector(workerRTNodes) + workerRTNodes, err = nodes.MatchingOptionalSelector(workerRTNodes) Expect(err).ToNot(HaveOccurred()) profile, err = profiles.GetByNodeLabels(testutils.NodeSelectorLabels) @@ -58,6 +56,12 @@ var _ = Describe("[ref_id: 45487][performance]additional kubelet arguments", Ord }) Context("Additional kubelet arguments", Label(string(label.Tier2)), func() { + BeforeEach(func() { + var err error + profile, err = profiles.GetByNodeLabels(testutils.NodeSelectorLabels) + Expect(err).ToNot(HaveOccurred()) + }) + It("[test_id:45488]Test performance profile annotation for changing multiple kubelet settings", func() { sysctls := "{\"allowedUnsafeSysctls\":[\"net.core.somaxconn\",\"kernel.msg*\"],\"systemReserved\":{\"memory\":\"300Mi\"},\"kubeReserved\":{\"memory\":\"768Mi\"},\"imageMinimumGCAge\":\"3m\"}" profile.Annotations = updateKubeletConfigOverrideAnnotations(profile.Annotations, sysctls) @@ -77,19 +81,20 @@ var _ = Describe("[ref_id: 45487][performance]additional kubelet arguments", Ord sysctlsValue := kubeletConfig.AllowedUnsafeSysctls Expect(sysctlsValue).Should(ContainElements("net.core.somaxconn", "kernel.msg*")) Expect(kubeletConfig.KubeReserved["memory"]).To(Equal("768Mi")) - Expect(kubeletConfig.ImageMinimumGCAge.Seconds()).To(Equal(180)) + Expect(kubeletConfig.ImageMinimumGCAge.Seconds()).To(BeNumerically("==", 180)) } - kubeletArguments := []string{"/bin/bash", "-c", "ps -ef | grep kubelet | grep config"} + + autoSizingCmd := []string{"cat", "/rootfs/etc/openshift/kubelet.conf.d/20-auto-sizing.conf"} for _, node := range workerRTNodes { - out, err := nodes.ExecCommand(context.TODO(), &node, kubeletArguments) + out, err := nodes.ExecCommand(context.TODO(), &node, autoSizingCmd) Expect(err).ToNot(HaveOccurred()) stdout := testutils.ToString(out) - Expect(strings.Contains(stdout, "300Mi")).To(BeTrue()) + Expect(stdout).To(ContainSubstring("300Mi")) } }) Context("When setting cpu manager related parameters", func() { It("[test_id:45493]Should not override performance-addon-operator values", func() { - paoValues := "{\"cpuManagerPolicy\":\"static\",\"cpuManagerReconcilePeriod\":\"5s\"}" + paoValues := "{\"cpuManagerPolicy\":\"none\",\"cpuManagerReconcilePeriod\":\"10s\"}" profile.Annotations = updateKubeletConfigOverrideAnnotations(profile.Annotations, paoValues) By("updating Performance profile") @@ -105,11 +110,11 @@ var _ = Describe("[ref_id: 45487][performance]additional kubelet arguments", Ord kubeletConfig, err := nodes.GetKubeletConfig(context.TODO(), &node) Expect(err).ToNot(HaveOccurred()) Expect(kubeletConfig.CPUManagerPolicy).Should(Equal("static")) - Expect(kubeletConfig.CPUManagerReconcilePeriod.Seconds()).To(Equal(5)) + Expect(kubeletConfig.CPUManagerReconcilePeriod.Seconds()).To(BeNumerically("==", 5)) } }) }) - It("[test_id:45490]Test memory reservation changes", func() { + It("[test_id:45490]Test memory reservation changes", Label("test1"), func() { // In this test case we check if after applying reserving memory for // systemReserved and KubeReserved, the allocatable is reduced and Allocatable // Verify that Allocatable = Node capacity - (kubereserved + systemReserved + EvictionMemory) @@ -144,10 +149,18 @@ var _ = Describe("[ref_id: 45487][performance]additional kubelet arguments", Ord Expect(kubeletConfigString).To(ContainSubstring(`"kubeReserved":{"memory":"768Mi"}`)) Expect(kubeletConfigString).To(ContainSubstring(`"systemReserved":{"memory":"300Mi"}`)) - for _, node := range workerRTNodes { + // Re-fetch nodes to get current allocatable and capacity after + // the tuning update, since workerRTNodes was populated before the + // annotation was applied and its Status values are stale. + updatedNodes, err := nodes.GetByLabels(testutils.NodeSelectorLabels) + Expect(err).ToNot(HaveOccurred()) + updatedNodes, err = nodes.MatchingOptionalSelector(updatedNodes) + Expect(err).ToNot(HaveOccurred()) + + for _, node := range updatedNodes { kubeletConfig, err := nodes.GetKubeletConfig(context.TODO(), &node) Expect(err).ToNot(HaveOccurred()) - totalCapactity := node.Status.Capacity.Memory().MilliValue() + totalCapacity := node.Status.Capacity.Memory().MilliValue() evictionMemory := kubeletConfig.EvictionHard["memory.available"] kubeReserved := kubeletConfig.KubeReserved["memory"] evictionMemoryInt, err := strconv.ParseInt(strings.TrimSuffix(evictionMemory, "Mi"), 10, 64) @@ -158,12 +171,35 @@ var _ = Describe("[ref_id: 45487][performance]additional kubelet arguments", Ord kubeReservedMemoryResource := resource.NewQuantity(kubeReservedMemoryInt*1024*1024, resource.BinarySI) evictionMemoryResource := resource.NewQuantity(evictionMemoryInt*1024*1024, resource.BinarySI) totalKubeMemory := systemReservedResource.MilliValue() + kubeReservedMemoryResource.MilliValue() + evictionMemoryResource.MilliValue() - calculatedAllocatable := totalCapactity - totalKubeMemory + + // Pre-allocated hugepages are subtracted from allocatable memory by the + // kubelet but are still included in node capacity. The standard formula + // Allocatable = Capacity - kubeReserved - systemReserved - evictionHard + // does not account for this, so we must subtract hugepages to match the + // actual allocatable reported by the node. + var totalHugepages int64 + for resourceName, quantity := range node.Status.Capacity { + if strings.HasPrefix(string(resourceName), corev1.ResourceHugePagesPrefix) { + totalHugepages += quantity.MilliValue() + } + } + + calculatedAllocatable := totalCapacity - totalKubeMemory - totalHugepages currentAllocatable := node.Status.Allocatable.Memory().MilliValue() Expect(calculatedAllocatable).To(Equal(currentAllocatable)) } }) + It("[test_id:45495] Test setting PAO managed parameters", func() { + cnfNodes, err := nodes.GetByLabels(testutils.NodeSelectorLabels) + Expect(err).ToNot(HaveOccurred()) + Expect(len(cnfNodes)).To(BeNumerically(">", 0), "expected at least one node to match the selector") + isArm, err := infrastructure.IsARM(ctx, &cnfNodes[0]) + Expect(err).ToNot(HaveOccurred()) + if isArm { + Skip("Changing topologyManagerPolicy is not supported on ARM architecture") + } + var paoParameters string if *profile.Spec.NUMA.TopologyPolicy == "single-numa-node" { paoParameters = "{\"topologyManagerPolicy\":\"restricted\"}" @@ -196,19 +232,26 @@ var _ = Describe("[ref_id: 45487][performance]additional kubelet arguments", Ord By("Reverting the Profile") profiles.UpdateWithRetry(initialProfile) - kubeletArguments := []string{"/bin/bash", "-c", "ps -ef | grep kubelet | grep config"} + By(fmt.Sprintf("Applying changes in performance profile and waiting until %s will start updating", poolName)) + profilesupdate.WaitForTuningUpdating(ctx, initialProfile) + + By(fmt.Sprintf("Waiting when %s finishes updates", poolName)) + profilesupdate.WaitForTuningUpdated(ctx, initialProfile) + for _, node := range workerRTNodes { kubeletConfig, err := nodes.GetKubeletConfig(context.TODO(), &node) Expect(err).ToNot(HaveOccurred()) - Expect(kubeletConfig.AllowedUnsafeSysctls).To(Equal(nil)) + Expect(kubeletConfig.AllowedUnsafeSysctls).To(BeEmpty()) Expect(kubeletConfig.KubeReserved["memory"]).ToNot(Equal("768Mi")) Expect(kubeletConfig.ImageMinimumGCAge.Seconds()).ToNot(Equal(180)) } + + autoSizingCmd := []string{"cat", "/rootfs/etc/openshift/kubelet.conf.d/20-auto-sizing.conf"} for _, node := range workerRTNodes { - out, err := nodes.ExecCommand(context.TODO(), &node, kubeletArguments) + out, err := nodes.ExecCommand(context.TODO(), &node, autoSizingCmd) Expect(err).ToNot(HaveOccurred()) stdout := testutils.ToString(out) - Expect(strings.Contains(stdout, "300Mi")).To(BeTrue()) + Expect(stdout).ToNot(ContainSubstring("300Mi")) } }) @@ -216,10 +259,8 @@ var _ = Describe("[ref_id: 45487][performance]additional kubelet arguments", Ord By("Reverting the Profile") profile, err := profiles.GetByNodeLabels(testutils.NodeSelectorLabels) Expect(err).ToNot(HaveOccurred()) - currentSpec, _ := json.Marshal(profile.Spec) - spec, _ := json.Marshal(initialProfile.Spec) // revert only if the profile changes. - if !equality.Semantic.DeepEqual(currentSpec, spec) { + if !equality.Semantic.DeepEqual(profile.Spec, initialProfile.Spec) || !equality.Semantic.DeepEqual(profile.Annotations, initialProfile.Annotations) { profiles.UpdateWithRetry(initialProfile) By(fmt.Sprintf("Applying changes in performance profile and waiting until %s will start updating", poolName))