From cbb32a2c06a290643b713940cf6a5c45f006ede3 Mon Sep 17 00:00:00 2001 From: Tomer Avital Date: Tue, 11 Aug 2026 12:56:01 +0300 Subject: [PATCH] Fix latency e2e timeout on large-CPU nodes If LATENCY_TEST_CPUS is not set, the test pod is given almost all of the node's isolated CPUs. On large machines that CPU setup inside the pod can take so long that the old fixed 120-second wait buffer runs out, and the test fails waiting for the pod to finish even though the latency tool itself succeeded. Replace the hard-coded 120s with LATENCY_TEST_TIMEOUT_BUFFER (default 150s) and return a clear error asking to increase it when the wait still times out. Signed-off-by: Tomer Avital --- .../performance_controller.md | 1 + .../functests/4_latency/latency.go | 51 ++++++++++++++----- .../5_latency_testing/latency_testing.go | 28 +++++++--- 3 files changed, 59 insertions(+), 21 deletions(-) diff --git a/docs/performanceprofile/performance_controller.md b/docs/performanceprofile/performance_controller.md index 5453bb5649..b1fe9109c0 100644 --- a/docs/performanceprofile/performance_controller.md +++ b/docs/performanceprofile/performance_controller.md @@ -117,6 +117,7 @@ You can run the container with different ENV variables, but the bare minimum is - `LATENCY_TEST_DELAY` indicates an (optional) delay in seconds to be used between the container is created and the tests actually start. Default is zero (start immediately). - `LATENCY_TEST_RUNTIME` the amount of time in seconds that the latency test should run. +- `LATENCY_TEST_TIMEOUT_BUFFER` overhead seconds (not per-tool runtime) for in-pod CPU setup/init and for the pod to reach Succeeded after the tool finishes. - `LATENCY_TEST_IMAGE` the image that used under the latency test. - `LATECNY_TEST_CPUS` the amount of CPUs the pod which run the latency test should request - `OSLAT_MAXIMUM_LATENCY` the expected maximum latency for all buckets in us in the oslat test. diff --git a/test/e2e/performanceprofile/functests/4_latency/latency.go b/test/e2e/performanceprofile/functests/4_latency/latency.go index b8d938fe2d..c8069c081d 100644 --- a/test/e2e/performanceprofile/functests/4_latency/latency.go +++ b/test/e2e/performanceprofile/functests/4_latency/latency.go @@ -42,12 +42,12 @@ const ( hwlatdetectTestName = "hwlatdetect" //default values - defaultTestDelay = 0 - defaultTestRuntime = "300" - defaultMaxLatency = -1 - defaultTestCpus = -1 - defaultTestMemory = "1Gi" - + defaultTestDelay = 0 + defaultTestRuntime = "300" + defaultMaxLatency = -1 + defaultTestCpus = -1 + defaultTestMemory = "1Gi" + defaultTestTimeoutBuffer = 150 //dynamic memory mode values // 32Mi per requested CPU should be reasonable for the test perCpuMemoryFactor = 32 @@ -57,15 +57,17 @@ const ( ) var ( - latencyTestDelay = defaultTestDelay - latencyTestRuntime = defaultTestRuntime - maximumLatency = defaultMaxLatency - latencyTestCpus = defaultTestCpus - latencyTestMemory = defaultTestMemory + latencyTestDelay = defaultTestDelay + latencyTestRuntime = defaultTestRuntime + latencyTestTimeoutBuffer = defaultTestTimeoutBuffer + maximumLatency = defaultMaxLatency + latencyTestCpus = defaultTestCpus + latencyTestMemory = defaultTestMemory ) // LATENCY_TEST_DELAY delay the run of the binary, can be useful to give time to the CPU manager reconcile loop // to update the default CPU pool +// LATENCY_TEST_TIMEOUT_BUFFER: extra seconds for pod CPU setup and Succeeded wait (not per-tool runtime) // LATENCY_TEST_RUNTIME: the amount of time in seconds that the latency test should run // LATENCY_TEST_CPUS: the amount of CPUs the pod which run the latency test should request // LATENCY_TEST_MEMORY: the amount of memory the pod which run the latency test should request @@ -79,6 +81,9 @@ var _ = Describe("[performance] Latency Test", Ordered, func() { latencyTestDelay, err = getLatencyTestDelay() Expect(err).ToNot(HaveOccurred()) + latencyTestTimeoutBuffer, err = getLatencyTestTimeoutBuffer() + Expect(err).ToNot(HaveOccurred()) + latencyTestCpus, err = getLatencyTestCpus() Expect(err).ToNot(HaveOccurred()) @@ -283,6 +288,23 @@ func getLatencyTestDelay() (int, error) { return defaultTestDelay, nil } +func getLatencyTestTimeoutBuffer() (int, error) { + if latencyTestTimeoutBufferEnv, ok := os.LookupEnv("LATENCY_TEST_TIMEOUT_BUFFER"); ok { + val, err := strconv.Atoi(latencyTestTimeoutBufferEnv) + if err != nil { + return val, fmt.Errorf("the environment variable LATENCY_TEST_TIMEOUT_BUFFER has incorrect value %q, it must be a non-negative integer with maximum value of %d: %w", latencyTestTimeoutBufferEnv, math.MaxInt32, err) + } + if val < 0 || val > math.MaxInt32 { + return val, fmt.Errorf("the environment variable LATENCY_TEST_TIMEOUT_BUFFER has an invalid number %q, it must be a non-negative integer with maximum value of %d", latencyTestTimeoutBufferEnv, math.MaxInt32) + } + if val < defaultTestTimeoutBuffer { + testlog.Warningf("LATENCY_TEST_TIMEOUT_BUFFER=%d is below %d; for safe execution set it to %d or higher, as a lower value may cause timeouts", val, defaultTestTimeoutBuffer, defaultTestTimeoutBuffer) + } + return val, nil + } + return defaultTestTimeoutBuffer, nil +} + func getLatencyTestCpus() (int, error) { if latencyTestCpusEnv, ok := os.LookupEnv("LATENCY_TEST_CPUS"); ok { val, err := strconv.Atoi(latencyTestCpusEnv) @@ -497,13 +519,14 @@ func createLatencyTestPod(testPod *corev1.Pod) { Expect(isEqual(RequestsCpusQuantity, latencyTestCpus)).To(BeTrue(), fmt.Sprintf("actual requests of cpus number used for the latency pod is not as set in LATENCY_TEST_CPUS, actual number is: %s", RequestsCpusQuantity)) } - By("Waiting another two minutes to give enough time for the cluster to move the pod to Succeeded phase") - podTimeout := time.Duration(timeout + latencyTestDelay + 120) + podTimeout := time.Duration(timeout + latencyTestDelay + latencyTestTimeoutBuffer) + By(fmt.Sprintf("Waiting up to %d seconds (runtime=%d, delay=%d, timeoutBuffer=%d) for the pod to reach Succeeded phase", int(podTimeout), timeout, latencyTestDelay, latencyTestTimeoutBuffer)) + testPod, err = pods.WaitForPhase(context.TODO(), client.ObjectKeyFromObject(testPod), corev1.PodSucceeded, podTimeout*time.Second) if err != nil { logEventsForPod(testPod) } - Expect(err).ToNot(HaveOccurred(), "pod %q did not reach %q phase; error: %v", podKey, corev1.PodSucceeded, err) + Expect(err).ToNot(HaveOccurred(), "pod %q did not reach %q phase; error: %v. Please Increase LATENCY_TEST_TIMEOUT_BUFFER to allow the cluster to move the pod to Succeeded phase and run again.", podKey, corev1.PodSucceeded, err) } func extractLatencyValues(exp string, pod *corev1.Pod) []int { diff --git a/test/e2e/performanceprofile/functests/5_latency_testing/latency_testing.go b/test/e2e/performanceprofile/functests/5_latency_testing/latency_testing.go index 24d2b7e5f3..7a8d80ce28 100644 --- a/test/e2e/performanceprofile/functests/5_latency_testing/latency_testing.go +++ b/test/e2e/performanceprofile/functests/5_latency_testing/latency_testing.go @@ -23,13 +23,14 @@ const ( cyclictest = "cyclictest" hwlatdetect = "hwlatdetect" //Environment variables names - latencyTestDelay = "LATENCY_TEST_DELAY" - latencyTestRuntime = "LATENCY_TEST_RUNTIME" - maximumLatency = "MAXIMUM_LATENCY" - oslatMaxLatency = "OSLAT_MAXIMUM_LATENCY" - hwlatdetecMaxLatency = "HWLATDETECT_MAXIMUM_LATENCY" - cyclictestMaxLatency = "CYCLICTEST_MAXIMUM_LATENCY" - latencyTestCpus = "LATENCY_TEST_CPUS" + latencyTestDelay = "LATENCY_TEST_DELAY" + latencyTestRuntime = "LATENCY_TEST_RUNTIME" + latencyTestTimeoutBuffer = "LATENCY_TEST_TIMEOUT_BUFFER" + maximumLatency = "MAXIMUM_LATENCY" + oslatMaxLatency = "OSLAT_MAXIMUM_LATENCY" + hwlatdetecMaxLatency = "HWLATDETECT_MAXIMUM_LATENCY" + cyclictestMaxLatency = "CYCLICTEST_MAXIMUM_LATENCY" + latencyTestCpus = "LATENCY_TEST_CPUS" //invalid values error messages unexpectedError = "Unexpected error" //incorrect values error messages @@ -44,6 +45,8 @@ const ( invalidCpuNumber = incorrectMsgPart1 + latencyTestCpus + invalidNumber + mustBePositiveInt incorrectDelay = incorrectMsgPart1 + latencyTestDelay + incorrectMsgPart2 + mustBeNonNegativeInt invalidNumberDelay = incorrectMsgPart1 + latencyTestDelay + invalidNumber + mustBeNonNegativeInt + incorrectTimeoutBuffer = incorrectMsgPart1 + latencyTestTimeoutBuffer + incorrectMsgPart2 + mustBeNonNegativeInt + invalidNumberTimeoutBuffer = incorrectMsgPart1 + latencyTestTimeoutBuffer + invalidNumber + mustBeNonNegativeInt incorrectMaxLatency = incorrectMsgPart1 + maximumLatency + incorrectMsgPart2 + mustBeNonNegativeInt invalidNumberMaxLatency = incorrectMsgPart1 + maximumLatency + invalidNumber + mustBeNonNegativeInt incorrectOslatMaxLatency = incorrectMsgPart1 + "\"" + oslatMaxLatency + "\"" + incorrectMsgPart2 + mustBeNonNegativeInt @@ -82,6 +85,7 @@ const ( type latencyTest struct { testDelay string testRuntime string + testTimeoutBuffer string testMaxLatency string oslatMaxLatency string cyclictestMaxLatency string @@ -178,6 +182,9 @@ func setEnvAndGetDescription(tst latencyTest) string { if tst.testDelay != "" { setEnvWriteDescription(latencyTestDelay, tst.testDelay, sb, &nonDefaultValues) } + if tst.testTimeoutBuffer != "" { + setEnvWriteDescription(latencyTestTimeoutBuffer, tst.testTimeoutBuffer, sb, &nonDefaultValues) + } if tst.testRuntime != "" { setEnvWriteDescription(latencyTestRuntime, tst.testRuntime, sb, &nonDefaultValues) } @@ -211,6 +218,7 @@ func setEnvWriteDescription(envVar string, val string, sb *bytes.Buffer, flag *b func clearEnv() { os.Unsetenv(latencyTestDelay) + os.Unsetenv(latencyTestTimeoutBuffer) os.Unsetenv(latencyTestRuntime) os.Unsetenv(maximumLatency) os.Unsetenv(oslatMaxLatency) @@ -237,6 +245,7 @@ func getValidValuesTests(toolToTest string) []latencyTest { testSet = append(testSet, latencyTest{testDelay: "1", testRuntime: successRuntime, testMaxLatency: untunedLatencyThreshold, outputMsgs: []string{success}, toolToTest: toolToTest, ginkgoTimeout: successGinkgoTimeout}) testSet = append(testSet, latencyTest{testDelay: "60", testRuntime: successRuntime, testMaxLatency: untunedLatencyThreshold, outputMsgs: []string{success}, toolToTest: toolToTest, ginkgoTimeout: successGinkgoTimeout}) testSet = append(testSet, latencyTest{testRuntime: "2", testCpus: "5", testMaxLatency: untunedLatencyThreshold, outputMsgs: []string{skip, skipOddCpuNumber}, toolToTest: toolToTest, ginkgoTimeout: successGinkgoTimeout}) + testSet = append(testSet, latencyTest{testTimeoutBuffer: "155", testRuntime: successRuntime, testMaxLatency: untunedLatencyThreshold, testCpus: "4", outputMsgs: []string{success}, toolToTest: toolToTest, ginkgoTimeout: successGinkgoTimeout}) if toolToTest != hwlatdetect { testSet = append(testSet, latencyTest{testRuntime: "1", outputMsgs: []string{skip, skipMaxLatency}, toolToTest: toolToTest, ginkgoTimeout: successGinkgoTimeout}) @@ -279,6 +288,11 @@ func getNegativeTests(toolToTest string) []latencyTest { testSet = append(testSet, latencyTest{testRuntime: "2", testCpus: "-1", outputMsgs: []string{invalidCpuNumber, fail}, toolToTest: toolToTest}) testSet = append(testSet, latencyTest{testRuntime: "2", testCpus: "0", outputMsgs: []string{invalidCpuNumber, fail}, toolToTest: toolToTest}) + // LATENCY_TEST_TIMEOUT_BUFFER must be a valid integer, reject non-numeric, and negative values. + testSet = append(testSet, latencyTest{testTimeoutBuffer: "J", outputMsgs: []string{incorrectTimeoutBuffer, fail}, toolToTest: toolToTest}) + testSet = append(testSet, latencyTest{testTimeoutBuffer: fmt.Sprint(math.MaxInt32 + 1), outputMsgs: []string{invalidNumberTimeoutBuffer, fail}, toolToTest: toolToTest}) + testSet = append(testSet, latencyTest{testTimeoutBuffer: "-5", outputMsgs: []string{invalidNumberTimeoutBuffer, fail}, toolToTest: toolToTest}) + if toolToTest == oslat { testSet = append(testSet, latencyTest{testRuntime: "2", oslatMaxLatency: "&", outputMsgs: []string{incorrectOslatMaxLatency, fail}, toolToTest: toolToTest}) testSet = append(testSet, latencyTest{testRuntime: "2", oslatMaxLatency: fmt.Sprint(math.MaxInt32 + 1), outputMsgs: []string{invalidNumberOslatMaxLatency, fail}, toolToTest: toolToTest})