OCPBUGS-98060 BUG-FIX Latency Test

docs/performanceprofile/performance_controller.md

@@ -117,6 +117,7 @@ You can run the container with different ENV variables, but the bare minimum is
 
 - `LATENCY_TEST_DELAY` indicates an (optional) delay in seconds to be used between the container is created and the tests actually start. Default is zero (start immediately).
 - `LATENCY_TEST_RUNTIME` the amount of time in seconds that the latency test should run.
+- `LATENCY_TEST_SETUP_DELAY` overhead seconds included in the per-test timeout to Pod to reach Succeded.
 - `LATENCY_TEST_IMAGE` the image that used under the latency test.
 - `LATECNY_TEST_CPUS` the amount of CPUs the pod which run the latency test should request
 - `OSLAT_MAXIMUM_LATENCY` the expected maximum latency for all buckets in us in the oslat test.

test/e2e/performanceprofile/functests/4_latency/latency.go

@@ -42,12 +42,12 @@ const (
 	hwlatdetectTestName = "hwlatdetect"
 
 	//default values
-	defaultTestDelay   = 0
-	defaultTestRuntime = "300"
-	defaultMaxLatency  = -1
-	defaultTestCpus    = -1
-	defaultTestMemory  = "1Gi"
-
+	defaultTestDelay      = 0
+	defaultTestRuntime    = "300"
+	defaultMaxLatency     = -1
+	defaultTestCpus       = -1
+	defaultTestMemory     = "1Gi"
+	defaultTestSetupDelay = 150
 	//dynamic memory mode values
 	// 32Mi per requested CPU should be reasonable for the test
 	perCpuMemoryFactor = 32
@@ -57,11 +57,12 @@ const (
 )
 
 var (
-	latencyTestDelay   = defaultTestDelay
-	latencyTestRuntime = defaultTestRuntime
-	maximumLatency     = defaultMaxLatency
-	latencyTestCpus    = defaultTestCpus
-	latencyTestMemory  = defaultTestMemory
+	latencyTestDelay      = defaultTestDelay
+	latencyTestRuntime    = defaultTestRuntime
+	latencyTestSetupDelay = defaultTestSetupDelay
+	maximumLatency        = defaultMaxLatency
+	latencyTestCpus       = defaultTestCpus
+	latencyTestMemory     = defaultTestMemory
 )
 
 // LATENCY_TEST_DELAY delay the run of the binary, can be useful to give time to the CPU manager reconcile loop
@@ -79,6 +80,9 @@ var _ = Describe("[performance] Latency Test", Ordered, func() {
 		latencyTestDelay, err = getLatencyTestDelay()
 		Expect(err).ToNot(HaveOccurred())
 
+		latencyTestSetupDelay, err = getLatencyTestSetupDelay()
+		Expect(err).ToNot(HaveOccurred())
+
 		latencyTestCpus, err = getLatencyTestCpus()
 		Expect(err).ToNot(HaveOccurred())
 
@@ -275,6 +279,23 @@ func getLatencyTestDelay() (int, error) {
 	return defaultTestDelay, nil
 }
 
+func getLatencyTestSetupDelay() (int, error) {
+	if latencyTestSetupDelayEnv, ok := os.LookupEnv("LATENCY_TEST_SETUP_DELAY"); ok {
+		val, err := strconv.Atoi(latencyTestSetupDelayEnv)
+		if err != nil {
+			return val, fmt.Errorf("the environment variable LATENCY_TEST_SETUP_DELAY has incorrect value %q, it must be a non-negative integer with maximum value of %d: %w", latencyTestSetupDelayEnv, math.MaxInt32, err)
+		}
+		if val < 0 || val > math.MaxInt32 {
+			return val, fmt.Errorf("the environment variable LATENCY_TEST_SETUP_DELAY has an invalid number %q, it must be a non-negative integer with maximum value of %d", latencyTestSetupDelayEnv, math.MaxInt32)
+		}
+		if val < defaultTestSetupDelay {
+			testlog.Warningf("LATENCY_TEST_SETUP_DELAY=%d is below %d; for safe execution set it to %d or higher, as a lower value may cause timeouts", val, defaultTestSetupDelay, defaultTestSetupDelay)
+		}
+		return val, nil
+	}
+	return defaultTestSetupDelay, nil
+}
+
 func getLatencyTestCpus() (int, error) {
 	if latencyTestCpusEnv, ok := os.LookupEnv("LATENCY_TEST_CPUS"); ok {
 		val, err := strconv.Atoi(latencyTestCpusEnv)
@@ -490,12 +511,12 @@ func createLatencyTestPod(testPod *corev1.Pod) {
 	}
 
 	By("Waiting another two minutes to give enough time for the cluster to move the pod to Succeeded phase")
-	podTimeout := time.Duration(timeout + latencyTestDelay + 120)
+	podTimeout := time.Duration(timeout + latencyTestDelay + latencyTestSetupDelay)
 	testPod, err = pods.WaitForPhase(context.TODO(), client.ObjectKeyFromObject(testPod), corev1.PodSucceeded, podTimeout*time.Second)
 	if err != nil {
 		logEventsForPod(testPod)
 	}
-	Expect(err).ToNot(HaveOccurred(), "pod %q did not reach %q phase; error: %v", podKey, corev1.PodSucceeded, err)
+	Expect(err).ToNot(HaveOccurred(), "pod %q did not reach %q phase; error: %v. Please Increase LATENCY_TEST_SETUP_DELAY to allow the cluster to move the pod to Succeeded phase and run again.", podKey, corev1.PodSucceeded, err)
 }
 
 func extractLatencyValues(exp string, pod *corev1.Pod) []int {