-
Notifications
You must be signed in to change notification settings - Fork 135
e2e: Fix broken assertions #1571
New issue
Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.
By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.
Already on GitHub? Sign in to your account
base: main
Are you sure you want to change the base?
Changes from all commits
File filter
Filter by extension
Conversations
Jump to
Diff view
Diff view
There are no files selected for viewing
| Original file line number | Diff line number | Diff line change |
|---|---|---|
|
|
@@ -50,15 +50,10 @@ const ( | |
| kubeletMixedCPUsConfigFile = "/etc/kubernetes/openshift-workload-mixed-cpus" | ||
| crioRuntimesConfigFile = "/etc/crio/crio.conf.d/99-runtimes.conf" | ||
| sharedCpusResource = "workload.openshift.io/enable-shared-cpus" | ||
| // the minimal number of cores for running the test is as follows: | ||
| // reserved = one core, shared = one core, infra workload = one core, test pod = one core - 4 in total | ||
| // smt alignment won't allow us to run the test pod with a single core, hence we should cancel it. | ||
| numberOfCoresThatRequiredCancelingSMTAlignment = 4 | ||
| restartCooldownTime = 1 * time.Minute | ||
| isolatedCpusEnv = "OPENSHIFT_ISOLATED_CPUS" | ||
| sharedCpusEnv = "OPENSHIFT_SHARED_CPUS" | ||
| // DeploymentName contains the name of the deployment | ||
| DeploymentName = "test-deployment" | ||
| restartCooldownTime = 1 * time.Minute | ||
| isolatedCpusEnv = "OPENSHIFT_ISOLATED_CPUS" | ||
| sharedCpusEnv = "OPENSHIFT_SHARED_CPUS" | ||
| DeploymentName = "test-deployment" // DeploymentName contains the name of the deployment | ||
| ) | ||
|
|
||
| var _ = Describe("Mixedcpus", Ordered, Label(string(label.MixedCPUs)), func() { | ||
|
|
@@ -151,7 +146,7 @@ var _ = Describe("Mixedcpus", Ordered, Label(string(label.MixedCPUs)), func() { | |
| When("workloads requests access for shared cpus", func() { | ||
| It("verify cpu load balancing still works with mixed cpus", func() { | ||
| rl := &corev1.ResourceList{ | ||
| corev1.ResourceCPU: resource.MustParse("1"), | ||
| corev1.ResourceCPU: resource.MustParse("2"), | ||
|
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. 🎯 Functional Correctness | 🟡 Minor | ⚡ Quick win 🧩 Analysis chain🏁 Script executed: #!/bin/bash
set -euo pipefail
ast-grep outline test/e2e/performanceprofile/functests/11_mixedcpus/mixedcpus.go --items all
rg -n -C 5 \
'ResourceCPU:\s+resource\.MustParse\("2"\)|func setup\(|CPU\.Isolated|isolated.*Size\(\)|createPod\(' \
test/e2e/performanceprofile/functests/11_mixedcpus/mixedcpus.go
fd -e go . test/e2e/performanceprofile | while IFS= read -r file; do
rg -n -C 3 'MixedCPUs|CPU\.Isolated|isolated.*cpus|isolated.*CPU' "$file" || true
doneRepository: openshift/cluster-node-tuning-operator Length of output: 50394 🏁 Script executed: #!/bin/bash
set -euo pipefail
printf '%s\n' '--- test profile and suite setup ---'
sed -n '55,155p' test/e2e/performanceprofile/functests/11_mixedcpus/mixedcpus.go
sed -n '810,875p' test/e2e/performanceprofile/functests/11_mixedcpus/mixedcpus.go
printf '%s\n' '--- all direct uses of the seven workloads ---'
sed -n '140,270p' test/e2e/performanceprofile/functests/11_mixedcpus/mixedcpus.go
sed -n '286,355p' test/e2e/performanceprofile/functests/11_mixedcpus/mixedcpus.go
sed -n '428,470p' test/e2e/performanceprofile/functests/11_mixedcpus/mixedcpus.go
printf '%s\n' '--- resource and scheduling helpers ---'
sed -n '900,1035p' test/e2e/performanceprofile/functests/11_mixedcpus/mixedcpus.go
rg -n -C 4 'enable-shared-cpus|sharedCpusResource|MixedCpus|mixed cpus|mixed CPUs|ResourceCPU' \
pkg test/e2e/performanceprofile/functests/utils | head -n 300Repository: openshift/cluster-node-tuning-operator Length of output: 50394 🏁 Script executed: #!/bin/bash
set -euo pipefail
printf '%s\n' '--- profile construction and test entry points ---'
sed -n '95,155p' test/e2e/performanceprofile/functests/11_mixedcpus/mixedcpus.go
rg -n -C 5 'testProfile\(\)|MixedCPUs|NodeSelectorLabels|ProfileNotFound' \
test/e2e/performanceprofile/functests/11_mixedcpus/mixedcpus.go \
test/e2e/performanceprofile/functests/utils
printf '%s\n' '--- mixed-CPU resource implementation ---'
rg -n -C 8 --glob '*.go' \
'enable-shared-cpus|shared_cpus|shared cpus|SharedCPUs|MixedCpus|MixedCPUs' \
. | head -n 500
printf '%s\n' '--- current change context for mixedcpus.go ---'
git diff --stat -- test/e2e/performanceprofile/functests/11_mixedcpus/mixedcpus.go
git diff --unified=12 -- test/e2e/performanceprofile/functests/11_mixedcpus/mixedcpus.go | sed -n '1,260p'Repository: openshift/cluster-node-tuning-operator Length of output: 50394 🏁 Script executed: #!/bin/bash
set -euo pipefail
printf '%s\n' '--- files defining the shared CPU resource ---'
git ls-files | rg -i 'mixed|shared|cpu.*manager|kubelet|crio'
printf '%s\n' '--- exact resource-name references ---'
rg -n --glob '*.go' --glob '*.yaml' --glob '*.yml' --glob '*.json' \
'workload\.openshift\.io/enable-shared-cpus' . || true
printf '%s\n' '--- test profile defaults and discovery behavior ---'
rg -n -C 8 --glob '*.go' \
'IsolatedCPUs|SharedCPUs|NewPerformanceProfile|GetDiscoveryPerformanceProfile|PerformanceProfileName' \
pkg test e2e | head -n 400
printf '%s\n' '--- concise static invariant check ---'
python3 - <<'PY'
from pathlib import Path
import re
p = Path("test/e2e/performanceprofile/functests/11_mixedcpus/mixedcpus.go")
s = p.read_text()
resource_blocks = re.findall(
r'rl := &corev1\.ResourceList\{(.*?)\n\s*\}',
s,
re.S,
)
target = []
for i, block in enumerate(resource_blocks, 1):
if 'resource.MustParse("2")' in block and 'sharedCpusResource' in block:
target.append(i)
setup = s[s.index('func setup('):s.index('func cpuSetToPerformanceCPUSet(')]
print(f"target resource-list blocks: {len(target)}")
print(f"target blocks contain CPU=2 and shared resource: {target}")
print("setup changes isolated CPUs only under !profileutil.IsMixedCPUsEnabled(profile):",
'!profileutil.IsMixedCPUsEnabled(profile)' in setup)
print("setup has no isolated-size guard:",
'Size()' not in setup and 'List()' not in setup)
PYRepository: openshift/cluster-node-tuning-operator Length of output: 50394 🏁 Script executed: #!/bin/bash
set -euo pipefail
printf '%s\n' '--- generated mixed-CPU configuration ---'
cat assets/performanceprofile/configs/openshift-workload-mixed-cpus
sed -n '1,220p' docs/performanceprofile/kubelet-config-snippets.md | rg -n -C 8 \
'mixed|shared|reservedSystemCPUs|cpu-shared'
printf '%s\n' '--- E2E profile defaults and discovery ---'
rg -n -C 6 --glob '*.go' \
'IsolatedCPUs|SharedCPUs|NewPerformanceProfile|GetDiscoveryPerformanceProfile|PerformanceProfileName' \
pkg test | head -n 300
printf '%s\n' '--- static invariant check ---'
python3 - <<'PY'
from pathlib import Path
import re
p = Path("test/e2e/performanceprofile/functests/11_mixedcpus/mixedcpus.go")
s = p.read_text()
blocks = re.findall(r'rl := &corev1\.ResourceList\{(.*?)\n\s*\}', s, re.S)
targets = [
(i, 'CPU=2' in b, 'shared=1' in b)
for i, b in enumerate(blocks, 1)
if 'resource.MustParse("2")' in b and 'sharedCpusResource' in b
]
setup = s[s.index('func setup('):s.index('func cpuSetToPerformanceCPUSet(')]
print("target resource-list blocks:", len(targets))
print("all target blocks request CPU=2 and shared resource=1:",
all(cpu and shared for _, cpu, shared in targets))
print("setup changes the profile only when mixed CPUs are disabled:",
'!profileutil.IsMixedCPUsEnabled(profile)' in setup)
print("setup checks isolated CPU count:", 'Size()' in setup)
print("setup checks isolated CPU list length:", 'List()' in setup)
PYRepository: openshift/cluster-node-tuning-operator Length of output: 371 🏁 Script executed: #!/bin/bash
set -u
python3 - <<'PY'
from pathlib import Path
import re
p = Path("test/e2e/performanceprofile/functests/11_mixedcpus/mixedcpus.go")
s = p.read_text()
blocks = re.findall(r'rl := &corev1\.ResourceList\{(.*?)\n\s*\}', s, re.S)
targets = [
b for b in blocks
if 'resource.MustParse("2")' in b and 'sharedCpusResource' in b
]
setup = s[s.index('func setup('):s.index('func cpuSetToPerformanceCPUSet(')]
print("target resource-list blocks:", len(targets))
print("all target blocks request CPU=2:", all('corev1.ResourceCPU' in b for b in targets))
print("all target blocks request shared resource=1:",
all('sharedCpusResource' in b and 'resource.MustParse("1")' in b for b in targets))
print("setup changes the profile only when mixed CPUs are disabled:",
'!profileutil.IsMixedCPUsEnabled(profile)' in setup)
print("setup checks isolated CPU count:", 'Size()' in setup or 'len(isolated.List())' in setup)
PYRepository: openshift/cluster-node-tuning-operator Length of output: 400 🌐 Web query:
💡 Result: The annotation workload.openshift.io/enable-shared-cpus is part of the Mixed-CPUs feature in OpenShift, which allows Guaranteed Quality of Service (QoS) pods to utilize both isolated and shared CPU pools [1][2][3]. Typically, Guaranteed QoS pods with integer CPU requests are assigned exclusive (isolated) CPUs, meaning they cannot access the shared CPU pool [4]. The Mixed-CPUs feature changes this behavior by permitting such pods to request access to the shared CPU pool via this specific resource annotation [1][2]. Key aspects of this functionality include: 1. Admission Validation: To use this feature, a pod must be in the Guaranteed QoS class [1][2][3]. The admission controller also verifies that the pod is deployed in a namespace explicitly allowed to use mixed-CPU workloads, typically indicated by the annotation workload.mixedcpus.openshift.io/allowed [2][3]. 2. Runtime Hinting: Because container runtimes (like CRI-O) are not natively aware of extended OpenShift resources, the admission controller injects an annotation (e.g., cpu-shared.crio.io/) into the pod definition [2][3]. This annotation informs the runtime that the specific container has been granted access to shared CPUs [2]. 3. Constraint: A pod can request no more than a single instance of the workload.openshift.io/enable-shared-cpus resource [1][3]. This feature is distinct from standard workload partitioning (enabled via cpuPartitioningMode: AllNodes), which separates cluster management pods into a reserved CPU pool while isolating customer workloads on separate cores [5][6][7]. The Mixed-CPUs feature specifically addresses the flexibility for customer workloads that require both isolation and access to shared resources within a Guaranteed QoS context [1][3]. Citations:
Guard the two-isolated-CPU requirement. These workloads request two whole CPUs. The shared-CPU resource only grants shared-CPU access. 📍 Affects 1 file
🤖 Prompt for AI Agents |
||
| corev1.ResourceMemory: resource.MustParse("100Mi"), | ||
| sharedCpusResource: resource.MustParse("1"), | ||
| } | ||
|
|
@@ -183,7 +178,7 @@ var _ = Describe("Mixedcpus", Ordered, Label(string(label.MixedCPUs)), func() { | |
| }) | ||
| It("should have the shared cpus under its cgroups", func() { | ||
| rl := &corev1.ResourceList{ | ||
| corev1.ResourceCPU: resource.MustParse("1"), | ||
| corev1.ResourceCPU: resource.MustParse("2"), | ||
| corev1.ResourceMemory: resource.MustParse("100Mi"), | ||
| sharedCpusResource: resource.MustParse("1"), | ||
| } | ||
|
|
@@ -203,7 +198,7 @@ var _ = Describe("Mixedcpus", Ordered, Label(string(label.MixedCPUs)), func() { | |
| }) | ||
| It("should be able to disable cfs_quota", func() { | ||
| rl := &corev1.ResourceList{ | ||
| corev1.ResourceCPU: resource.MustParse("1"), | ||
| corev1.ResourceCPU: resource.MustParse("2"), | ||
| corev1.ResourceMemory: resource.MustParse("100Mi"), | ||
| sharedCpusResource: resource.MustParse("1"), | ||
| } | ||
|
|
@@ -220,7 +215,7 @@ var _ = Describe("Mixedcpus", Ordered, Label(string(label.MixedCPUs)), func() { | |
| }) | ||
| It("should have OPENSHIFT_ISOLATED_CPUS and OPENSHIFT_SHARED_CPUS env variables under the container", func() { | ||
| rl := &corev1.ResourceList{ | ||
| corev1.ResourceCPU: resource.MustParse("1"), | ||
| corev1.ResourceCPU: resource.MustParse("2"), | ||
| corev1.ResourceMemory: resource.MustParse("100Mi"), | ||
| sharedCpusResource: resource.MustParse("1"), | ||
| } | ||
|
|
@@ -251,7 +246,7 @@ var _ = Describe("Mixedcpus", Ordered, Label(string(label.MixedCPUs)), func() { | |
| }) | ||
| It("should contains the shared cpus after Kubelet restarts", func() { | ||
| rl := &corev1.ResourceList{ | ||
| corev1.ResourceCPU: resource.MustParse("1"), | ||
| corev1.ResourceCPU: resource.MustParse("2"), | ||
| corev1.ResourceMemory: resource.MustParse("100Mi"), | ||
| sharedCpusResource: resource.MustParse("1"), | ||
| } | ||
|
|
@@ -320,14 +315,14 @@ var _ = Describe("Mixedcpus", Ordered, Label(string(label.MixedCPUs)), func() { | |
| By(fmt.Sprintf("Waiting when %s finishes updates", poolName)) | ||
| profilesupdate.WaitForTuningUpdated(context.TODO(), profile) | ||
|
|
||
| Expect(testclient.ControlPlaneClient.Get(ctx, client.ObjectKeyFromObject(profile), profile)) | ||
| testlog.Infof("new isolated CPU set=%q\nnew shared CPU set=%q", string(*profile.Spec.CPU.Isolated), string(*profile.Spec.CPU.Isolated)) | ||
| Expect(testclient.ControlPlaneClient.Get(ctx, client.ObjectKeyFromObject(profile), profile)).To(Succeed()) | ||
| testlog.Infof("new isolated CPU set=%q\nnew shared CPU set=%q", string(*profile.Spec.CPU.Isolated), string(*profile.Spec.CPU.Shared)) | ||
| // we do not bother to revert the profile at the end of the test, since its irrelevant which of the cpus are shared | ||
| }) | ||
|
|
||
| It("should contains the updated values under the container", func() { | ||
| rl := &corev1.ResourceList{ | ||
| corev1.ResourceCPU: resource.MustParse("1"), | ||
| corev1.ResourceCPU: resource.MustParse("2"), | ||
| corev1.ResourceMemory: resource.MustParse("100Mi"), | ||
| sharedCpusResource: resource.MustParse("1"), | ||
| } | ||
|
|
@@ -439,7 +434,7 @@ var _ = Describe("Mixedcpus", Ordered, Label(string(label.MixedCPUs)), func() { | |
|
|
||
| By("Creating a deployment with one pod asking for a shared cpu") | ||
| rl := &corev1.ResourceList{ | ||
| corev1.ResourceCPU: resource.MustParse("1"), | ||
| corev1.ResourceCPU: resource.MustParse("2"), | ||
| corev1.ResourceMemory: resource.MustParse("100Mi"), | ||
| sharedCpusResource: resource.MustParse("1"), | ||
| } | ||
|
|
@@ -502,7 +497,7 @@ var _ = Describe("Mixedcpus", Ordered, Label(string(label.MixedCPUs)), func() { | |
| Expect(pod.Status.Phase).To(Equal(corev1.PodPending), "Pod %s is not in the pending state", pod.Name) | ||
|
|
||
| By("Reverting the cluster to previous state") | ||
| Expect(testclient.ControlPlaneClient.Get(ctx, client.ObjectKeyFromObject(profile), profile)) | ||
| Expect(testclient.ControlPlaneClient.Get(ctx, client.ObjectKeyFromObject(profile), profile)).To(Succeed()) | ||
| profile.Spec.CPU.Shared = cpuSetToPerformanceCPUSet(ppShared) | ||
| profile.Spec.WorkloadHints.MixedCpus = ptr.To(true) | ||
| profiles.UpdateWithRetry(profile) | ||
|
|
@@ -577,13 +572,16 @@ var _ = Describe("Mixedcpus", Ordered, Label(string(label.MixedCPUs)), func() { | |
|
|
||
| coreSiblings, err := nodes.GetCoreSiblings(ctx, workerRTNode) | ||
| Expect(err).ToNot(HaveOccurred()) | ||
| // When Shared already has 1 CPU and we need 2, we replace Shared with a new pair from | ||
|
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. But with the new logic we always have 2 shared CPUs by default, so why the comment says Besides the wrong comment, we don't even need to change anything, since now 2 CPUs is the default.
Member
Author
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more. Thanks for flagging this, I think there's a small mix-up, my bad if the earlier phrasing wasn't clear. The cpu:1 -> cpu:2 change in this PR was to the pods' resource requests, for SMT alignment reasons. It's unrelated to profile.Spec.CPU.Shared, which is set separately by setup() at the top of the file, and still defaults to 1 CPU there. So updatedShared.Size() < 2 is still true in the normal case, and this code (with the oldShared/Union fix) still runs. Worth noting too: even if we changed setup() to assign 2 shared CPUs, that only covers the case where setup() itself enables mixed CPUs. If a cluster arrives with mixed CPUs already enabled and only 1 shared CPU, setup() skips that assignment entirely, so we'd still need this fallback to grow Shared to 2. On the comment wording: Shared could in theory be 0 as well as 1, but that's actually unreachable in practice, since IsMixedCPUsEnabled requires Shared to already be non-empty. So the comment holds as written, only the "1 CPU" case ever really happens here.
Contributor
There was a problem hiding this comment. Choose a reason for hiding this commentThe reason will be displayed to describe this comment to others. Learn more.
This is a well defined testing environment, we're not expecting prior state and fully control it. IOW, if we're moving from a single CPU for testing mixed-cpus feature, lets do it properly and get rid of the other ugly workarounds we had in the code, (which increasing from 1 -> 2 was one of them). |
||
| // Isolated. Put the old shared CPU back into Isolated so it is not left unassigned. | ||
| oldShared := updatedShared | ||
| updatedShared, err = nodes.GetTwoSiblingsFromCPUSet(coreSiblings, updatedIsolated) | ||
| if err != nil { | ||
| testlog.Info("no two siblings found in the given CPU set, looks like the initial profile does not respect hyperthreading; proceed then with this state and pick first two isolated CPUs as the shared CPUs") | ||
| updatedShared = cpuset.New(updatedIsolated.List()[0], updatedIsolated.List()[1]) | ||
| } | ||
|
|
||
| updatedIsolated = updatedIsolated.Difference(updatedShared) | ||
| updatedIsolated = updatedIsolated.Difference(updatedShared).Union(oldShared) | ||
|
|
||
| testlog.Infof("CPU update:shared cpu %q isolated cpus %q", updatedShared.String(), updatedIsolated.String()) | ||
| profile.Spec.CPU.Isolated = cpuSetToPerformanceCPUSet(&updatedIsolated) | ||
|
|
@@ -843,20 +841,6 @@ func setup(ctx context.Context) func(ctx2 context.Context) { | |
| testlog.Infof("mixed cpus already enabled for profile %q", profile.Name) | ||
| } | ||
|
|
||
| workers, err := nodes.GetByLabels(testutils.NodeSelectorLabels) | ||
| Expect(err).ToNot(HaveOccurred()) | ||
| for _, worker := range workers { | ||
| //node cpu numbers are integral | ||
| numOfCores, _ := worker.Status.Capacity.Cpu().AsInt64() | ||
| if numOfCores <= numberOfCoresThatRequiredCancelingSMTAlignment { | ||
| profile.Annotations = map[string]string{ | ||
| "kubeletconfig.experimental": "{\"cpuManagerPolicyOptions\": {\"full-pcpus-only\": \"false\"}}", | ||
| } | ||
| testlog.Infof("canceling SMT alignment for nodes under profile %q", profile.Name) | ||
| updateNeeded = true | ||
| } | ||
| } | ||
|
|
||
| if !updateNeeded { | ||
| return func(ctx context.Context) { | ||
| By(fmt.Sprintf("skipping teardown - no changes to profile %q were applied", profile.Name)) | ||
|
|
@@ -872,7 +856,7 @@ func setup(ctx context.Context) func(ctx2 context.Context) { | |
|
|
||
| teardown := func(ctx2 context.Context) { | ||
| By(fmt.Sprintf("executing teardown - revert profile %q back to its initial state", profile.Name)) | ||
| Expect(testclient.ControlPlaneClient.Get(ctx2, client.ObjectKeyFromObject(initialProfile), profile)) | ||
| Expect(testclient.ControlPlaneClient.Get(ctx2, client.ObjectKeyFromObject(initialProfile), profile)).To(Succeed()) | ||
| profiles.UpdateWithRetry(initialProfile) | ||
|
|
||
| // do not wait if nothing has changed | ||
|
|
||
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
That's a change in the behavior. it makes the test more restrict. is that what we want here?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
This is changing behavior - but isn't this the intention here?
If we are not asserting on
okthen i fail to see why we need theerrStringand
regexp.MatchString(errString, string(output))to begin with.Maybe im missing something here.
Is the goal to make sure the correct error message appears, or that PPC script fails in general?
There was a problem hiding this comment.
Choose a reason for hiding this comment
The reason will be displayed to describe this comment to others. Learn more.
After reevaluation I think this change is ok.
the only change I would add is in the message to make it more clear: