From daa09ea02a9bec84ee1c33468cdc03dd3843f6ac Mon Sep 17 00:00:00 2001 From: oblau Date: Thu, 23 Jul 2026 13:13:47 +0300 Subject: [PATCH 1/3] E2E: add missing matcher to offlined CPU Expect() calls (5 sites) Expect(bool) without .To() is a Gomega no-op; these tests never fail on these assertions. --- .../functests/2_performance_update/updating_profile.go | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) diff --git a/test/e2e/performanceprofile/functests/2_performance_update/updating_profile.go b/test/e2e/performanceprofile/functests/2_performance_update/updating_profile.go index ce53fbd91..d28c71333 100644 --- a/test/e2e/performanceprofile/functests/2_performance_update/updating_profile.go +++ b/test/e2e/performanceprofile/functests/2_performance_update/updating_profile.go @@ -637,7 +637,7 @@ var _ = Describe("[rfe_id:28761][performance] Updating parameters in performance Expect(err).ToNot(HaveOccurred()) offlinedCPUSetProfile, err := cpuset.Parse(string(offlined)) Expect(err).ToNot(HaveOccurred()) - Expect(offlinedCPUSet.Equals(offlinedCPUSetProfile)) + Expect(offlinedCPUSet.Equals(offlinedCPUSetProfile)).To(BeTrue(), "offlined CPUs mismatch: expected %q, got %q", offlinedCPUSetProfile, offlinedCPUSet) } }) @@ -707,7 +707,7 @@ var _ = Describe("[rfe_id:28761][performance] Updating parameters in performance Expect(err).ToNot(HaveOccurred()) offlinedCPUSetProfile, err := cpuset.Parse(string(offlinedSet)) Expect(err).ToNot(HaveOccurred()) - Expect(offlinedCPUSet.Equals(offlinedCPUSetProfile)) + Expect(offlinedCPUSet.Equals(offlinedCPUSetProfile)).To(BeTrue(), "offlined CPUs mismatch: expected %q, got %q", offlinedCPUSetProfile, offlinedCPUSet) } }) @@ -770,7 +770,7 @@ var _ = Describe("[rfe_id:28761][performance] Updating parameters in performance Expect(err).ToNot(HaveOccurred()) offlinedCPUSetProfile, err := cpuset.Parse(string(offlinedSet)) Expect(err).ToNot(HaveOccurred()) - Expect(offlinedCPUSet.Equals(offlinedCPUSetProfile)) + Expect(offlinedCPUSet.Equals(offlinedCPUSetProfile)).To(BeTrue(), "offlined CPUs mismatch: expected %q, got %q", offlinedCPUSetProfile, offlinedCPUSet) } }) @@ -840,7 +840,7 @@ var _ = Describe("[rfe_id:28761][performance] Updating parameters in performance Expect(err).ToNot(HaveOccurred()) offlinedCPUSetProfile, err := cpuset.Parse(string(offlinedSet)) Expect(err).ToNot(HaveOccurred()) - Expect(offlinedCPUSet.Equals(offlinedCPUSetProfile)) + Expect(offlinedCPUSet.Equals(offlinedCPUSetProfile)).To(BeTrue(), "offlined CPUs mismatch: expected %q, got %q", offlinedCPUSetProfile, offlinedCPUSet) } }) @@ -966,7 +966,7 @@ var _ = Describe("[rfe_id:28761][performance] Updating parameters in performance Expect(err).ToNot(HaveOccurred()) offlinedCPUSetProfile, err := cpuset.Parse(string(offlinedSet)) Expect(err).ToNot(HaveOccurred()) - Expect(offlinedCPUSet.Equals(offlinedCPUSetProfile)) + Expect(offlinedCPUSet.Equals(offlinedCPUSetProfile)).To(BeTrue(), "offlined CPUs mismatch: expected %q, got %q", offlinedCPUSetProfile, offlinedCPUSet) } }) From 4ecdb76cb7608b3218e2b21596b7ebf168f96de5 Mon Sep 17 00:00:00 2001 From: oblau Date: Thu, 23 Jul 2026 13:16:51 +0300 Subject: [PATCH 2/3] E2E: mixedcpus Get() checks, SMT-safe CPU requests, and shared-set expand fix - Add .To(Succeed()) / err checks on three ControlPlaneClient.Get() calls. Bare Expect(Get(...)) is a Gomega no-op, so API errors were ignored. - Change scheduling Guaranteed pods from cpu:1 to cpu:2, and remove the setup() path that set full-pcpus-only=false on small nodes. With default SMT alignment, cpu:1 cannot schedule; CI has enough CPUs for cpu:2, so the cancel-SMT workaround is unnecessary. - When exec-cpu-affinity expands Shared from 1 CPU to a new 2-CPU set, Union the previous shared CPU into Isolated. Replacing Shared without that left the old shared CPU in neither set and broke later assertions. --- .../functests/11_mixedcpus/mixedcpus.go | 54 +++++++------------ 1 file changed, 19 insertions(+), 35 deletions(-) diff --git a/test/e2e/performanceprofile/functests/11_mixedcpus/mixedcpus.go b/test/e2e/performanceprofile/functests/11_mixedcpus/mixedcpus.go index fcd5914c3..7dd19f4da 100644 --- a/test/e2e/performanceprofile/functests/11_mixedcpus/mixedcpus.go +++ b/test/e2e/performanceprofile/functests/11_mixedcpus/mixedcpus.go @@ -50,15 +50,10 @@ const ( kubeletMixedCPUsConfigFile = "/etc/kubernetes/openshift-workload-mixed-cpus" crioRuntimesConfigFile = "/etc/crio/crio.conf.d/99-runtimes.conf" sharedCpusResource = "workload.openshift.io/enable-shared-cpus" - // the minimal number of cores for running the test is as follows: - // reserved = one core, shared = one core, infra workload = one core, test pod = one core - 4 in total - // smt alignment won't allow us to run the test pod with a single core, hence we should cancel it. - numberOfCoresThatRequiredCancelingSMTAlignment = 4 - restartCooldownTime = 1 * time.Minute - isolatedCpusEnv = "OPENSHIFT_ISOLATED_CPUS" - sharedCpusEnv = "OPENSHIFT_SHARED_CPUS" - // DeploymentName contains the name of the deployment - DeploymentName = "test-deployment" + restartCooldownTime = 1 * time.Minute + isolatedCpusEnv = "OPENSHIFT_ISOLATED_CPUS" + sharedCpusEnv = "OPENSHIFT_SHARED_CPUS" + DeploymentName = "test-deployment" // DeploymentName contains the name of the deployment ) var _ = Describe("Mixedcpus", Ordered, Label(string(label.MixedCPUs)), func() { @@ -151,7 +146,7 @@ var _ = Describe("Mixedcpus", Ordered, Label(string(label.MixedCPUs)), func() { When("workloads requests access for shared cpus", func() { It("verify cpu load balancing still works with mixed cpus", func() { rl := &corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceCPU: resource.MustParse("2"), corev1.ResourceMemory: resource.MustParse("100Mi"), sharedCpusResource: resource.MustParse("1"), } @@ -183,7 +178,7 @@ var _ = Describe("Mixedcpus", Ordered, Label(string(label.MixedCPUs)), func() { }) It("should have the shared cpus under its cgroups", func() { rl := &corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceCPU: resource.MustParse("2"), corev1.ResourceMemory: resource.MustParse("100Mi"), sharedCpusResource: resource.MustParse("1"), } @@ -203,7 +198,7 @@ var _ = Describe("Mixedcpus", Ordered, Label(string(label.MixedCPUs)), func() { }) It("should be able to disable cfs_quota", func() { rl := &corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceCPU: resource.MustParse("2"), corev1.ResourceMemory: resource.MustParse("100Mi"), sharedCpusResource: resource.MustParse("1"), } @@ -220,7 +215,7 @@ var _ = Describe("Mixedcpus", Ordered, Label(string(label.MixedCPUs)), func() { }) It("should have OPENSHIFT_ISOLATED_CPUS and OPENSHIFT_SHARED_CPUS env variables under the container", func() { rl := &corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceCPU: resource.MustParse("2"), corev1.ResourceMemory: resource.MustParse("100Mi"), sharedCpusResource: resource.MustParse("1"), } @@ -251,7 +246,7 @@ var _ = Describe("Mixedcpus", Ordered, Label(string(label.MixedCPUs)), func() { }) It("should contains the shared cpus after Kubelet restarts", func() { rl := &corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceCPU: resource.MustParse("2"), corev1.ResourceMemory: resource.MustParse("100Mi"), sharedCpusResource: resource.MustParse("1"), } @@ -320,14 +315,14 @@ var _ = Describe("Mixedcpus", Ordered, Label(string(label.MixedCPUs)), func() { By(fmt.Sprintf("Waiting when %s finishes updates", poolName)) profilesupdate.WaitForTuningUpdated(context.TODO(), profile) - Expect(testclient.ControlPlaneClient.Get(ctx, client.ObjectKeyFromObject(profile), profile)) - testlog.Infof("new isolated CPU set=%q\nnew shared CPU set=%q", string(*profile.Spec.CPU.Isolated), string(*profile.Spec.CPU.Isolated)) + Expect(testclient.ControlPlaneClient.Get(ctx, client.ObjectKeyFromObject(profile), profile)).To(Succeed()) + testlog.Infof("new isolated CPU set=%q\nnew shared CPU set=%q", string(*profile.Spec.CPU.Isolated), string(*profile.Spec.CPU.Shared)) // we do not bother to revert the profile at the end of the test, since its irrelevant which of the cpus are shared }) It("should contains the updated values under the container", func() { rl := &corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceCPU: resource.MustParse("2"), corev1.ResourceMemory: resource.MustParse("100Mi"), sharedCpusResource: resource.MustParse("1"), } @@ -439,7 +434,7 @@ var _ = Describe("Mixedcpus", Ordered, Label(string(label.MixedCPUs)), func() { By("Creating a deployment with one pod asking for a shared cpu") rl := &corev1.ResourceList{ - corev1.ResourceCPU: resource.MustParse("1"), + corev1.ResourceCPU: resource.MustParse("2"), corev1.ResourceMemory: resource.MustParse("100Mi"), sharedCpusResource: resource.MustParse("1"), } @@ -502,7 +497,7 @@ var _ = Describe("Mixedcpus", Ordered, Label(string(label.MixedCPUs)), func() { Expect(pod.Status.Phase).To(Equal(corev1.PodPending), "Pod %s is not in the pending state", pod.Name) By("Reverting the cluster to previous state") - Expect(testclient.ControlPlaneClient.Get(ctx, client.ObjectKeyFromObject(profile), profile)) + Expect(testclient.ControlPlaneClient.Get(ctx, client.ObjectKeyFromObject(profile), profile)).To(Succeed()) profile.Spec.CPU.Shared = cpuSetToPerformanceCPUSet(ppShared) profile.Spec.WorkloadHints.MixedCpus = ptr.To(true) profiles.UpdateWithRetry(profile) @@ -577,13 +572,16 @@ var _ = Describe("Mixedcpus", Ordered, Label(string(label.MixedCPUs)), func() { coreSiblings, err := nodes.GetCoreSiblings(ctx, workerRTNode) Expect(err).ToNot(HaveOccurred()) + // When Shared already has 1 CPU and we need 2, we replace Shared with a new pair from + // Isolated. Put the old shared CPU back into Isolated so it is not left unassigned. + oldShared := updatedShared updatedShared, err = nodes.GetTwoSiblingsFromCPUSet(coreSiblings, updatedIsolated) if err != nil { testlog.Info("no two siblings found in the given CPU set, looks like the initial profile does not respect hyperthreading; proceed then with this state and pick first two isolated CPUs as the shared CPUs") updatedShared = cpuset.New(updatedIsolated.List()[0], updatedIsolated.List()[1]) } - updatedIsolated = updatedIsolated.Difference(updatedShared) + updatedIsolated = updatedIsolated.Difference(updatedShared).Union(oldShared) testlog.Infof("CPU update:shared cpu %q isolated cpus %q", updatedShared.String(), updatedIsolated.String()) profile.Spec.CPU.Isolated = cpuSetToPerformanceCPUSet(&updatedIsolated) @@ -843,20 +841,6 @@ func setup(ctx context.Context) func(ctx2 context.Context) { testlog.Infof("mixed cpus already enabled for profile %q", profile.Name) } - workers, err := nodes.GetByLabels(testutils.NodeSelectorLabels) - Expect(err).ToNot(HaveOccurred()) - for _, worker := range workers { - //node cpu numbers are integral - numOfCores, _ := worker.Status.Capacity.Cpu().AsInt64() - if numOfCores <= numberOfCoresThatRequiredCancelingSMTAlignment { - profile.Annotations = map[string]string{ - "kubeletconfig.experimental": "{\"cpuManagerPolicyOptions\": {\"full-pcpus-only\": \"false\"}}", - } - testlog.Infof("canceling SMT alignment for nodes under profile %q", profile.Name) - updateNeeded = true - } - } - if !updateNeeded { return func(ctx context.Context) { By(fmt.Sprintf("skipping teardown - no changes to profile %q were applied", profile.Name)) @@ -872,7 +856,7 @@ func setup(ctx context.Context) func(ctx2 context.Context) { teardown := func(ctx2 context.Context) { By(fmt.Sprintf("executing teardown - revert profile %q back to its initial state", profile.Name)) - Expect(testclient.ControlPlaneClient.Get(ctx2, client.ObjectKeyFromObject(initialProfile), profile)) + Expect(testclient.ControlPlaneClient.Get(ctx2, client.ObjectKeyFromObject(initialProfile), profile)).To(Succeed()) profiles.UpdateWithRetry(initialProfile) // do not wait if nothing has changed From 3cafd463e039d1148135da3e0528b75890159382 Mon Sep 17 00:00:00 2001 From: oblau Date: Wed, 29 Jul 2026 16:08:51 +0300 Subject: [PATCH 3/3] E2E: PPC: replace no-op log-on-match with actual assertions (4 sites) The `if ok { testlog.Info(...) }` pattern after regexp.MatchString never failed the test when the expected error was absent from output. Replace with Wait + gexec.Exit and ContainSubstring (not regexp) so a missing error message actually fails the test case. Match leaf error text only; outer wraps have changed and are not the intent of these cases: - 41405: https://github.com/openshift/cluster-node-tuning-operator/blob/a9d25d502ca894272f88753f1bd7ecef82fb188e/pkg/performanceprofile/profilecreator/cmd/root.go#L221 - 41419: https://github.com/openshift/cluster-node-tuning-operator/blob/a9d25d502ca894272f88753f1bd7ecef82fb188e/pkg/performanceprofile/profilecreator/profilecreator.go#L363 - 41420: https://github.com/openshift/cluster-node-tuning-operator/blob/a9d25d502ca894272f88753f1bd7ecef82fb188e/pkg/performanceprofile/profilecreator/profilecreator.go#L128 ([1,%d] is TotalThreads-1; format at runtime via maxReservedCPUCountFromMustGather from the must-gather MCP node) Also move gexec.Exit checks to right after Wait (before content asserts), and fix success-path "marshal" -> "unmarshal" messages. --- .../functests/10_performance_ppc/ppc.go | 88 ++++++++++++------- 1 file changed, 55 insertions(+), 33 deletions(-) diff --git a/test/e2e/performanceprofile/functests/10_performance_ppc/ppc.go b/test/e2e/performanceprofile/functests/10_performance_ppc/ppc.go index 51c988623..abef72188 100644 --- a/test/e2e/performanceprofile/functests/10_performance_ppc/ppc.go +++ b/test/e2e/performanceprofile/functests/10_performance_ppc/ppc.go @@ -3,16 +3,16 @@ package __performance_ppc import ( "fmt" "os/exec" - "regexp" + "path/filepath" "strings" . "github.com/onsi/ginkgo/v2" . "github.com/onsi/gomega" "github.com/onsi/gomega/gexec" performancev2 "github.com/openshift/cluster-node-tuning-operator/pkg/apis/performanceprofile/v2" + "github.com/openshift/cluster-node-tuning-operator/pkg/performanceprofile/profilecreator" testutils "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils" "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/label" - testlog "github.com/openshift/cluster-node-tuning-operator/test/e2e/performanceprofile/functests/utils/log" "k8s.io/utils/cpuset" "sigs.k8s.io/yaml" ) @@ -89,9 +89,12 @@ var _ = Describe("[rfe_id: 38968] PerformanceProfile setup helper and platform a podmanArgs := append(defaultArgs, cmdArgs...) session, err := ppcIntgTest.PodmanAsUserBase(podmanArgs, false, false) Expect(err).ToNot(HaveOccurred(), "Podman command failed") + output := session.Wait(20).Out.Contents() + Expect(session).Should(gexec.Exit(0)) + err = yaml.Unmarshal(output, pp) - Expect(err).ToNot(HaveOccurred(), "Unable to marshal the ppc output") + Expect(err).ToNot(HaveOccurred(), "Unable to unmarshal the ppc output") reservedCpus, err := cpuset.Parse(string(*pp.Spec.CPU.Reserved)) Expect(err).ToNot(HaveOccurred(), "Unable to parse cpus") totalReservedCpus := reservedCpus.Size() @@ -99,8 +102,8 @@ var _ = Describe("[rfe_id: 38968] PerformanceProfile setup helper and platform a Expect(*pp.Spec.RealTimeKernel.Enabled).To(BeTrue()) Expect(*pp.Spec.WorkloadHints.RealTime).To(BeTrue()) Expect(*pp.Spec.NUMA.TopologyPolicy).To(Equal("restricted")) - Eventually(session).Should(gexec.Exit(0)) }) + It("[test_id:41405] Verify PPC script fails when the splitting of reserved cpus and single numa-node policy is specified", func() { cmdArgs := []string{ fmt.Sprintf("%s:%s:z", mustgatherDir, mustgatherDir), @@ -116,14 +119,12 @@ var _ = Describe("[rfe_id: 38968] PerformanceProfile setup helper and platform a podmanArgs := append(defaultArgs, cmdArgs...) session, err := ppcIntgTest.PodmanAsUserBase(podmanArgs, false, false) Expect(err).ToNot(HaveOccurred(), "Podman command failed") + output := session.Wait(20).Err.Contents() - errString := "Error: failed to obtain data from flags not appropriate to split reserved CPUs in case of topology-manager-policy: single-numa-node" - ok, err := regexp.MatchString(errString, string(output)) - Expect(err).ToNot(HaveOccurred()) - if ok { - testlog.Info(errString) - } - Eventually(session).Should(gexec.Exit(1)) + Expect(session).Should(gexec.Exit(1)) + + errString := "not appropriate to split reserved CPUs in case of topology-manager-policy: single-numa-node" + Expect(string(output)).To(ContainSubstring(errString), "expected error:\n%q\ngot:\n%s", errString, output) }) It("[test_id:41419] Verify PPC script fails when reserved cpu count is 2 and requires to split across numa nodes", func() { @@ -140,14 +141,12 @@ var _ = Describe("[rfe_id: 38968] PerformanceProfile setup helper and platform a podmanArgs := append(defaultArgs, cmdArgs...) session, err := ppcIntgTest.PodmanAsUserBase(podmanArgs, false, false) Expect(err).ToNot(HaveOccurred(), "Podman command failed") + output := session.Wait(20).Err.Contents() - errString := "Error: failed to compute the reserved and isolated CPUs: can't allocate odd number of CPUs from a NUMA Node" - ok, err := regexp.MatchString(errString, string(output)) - Expect(err).ToNot(HaveOccurred(), "did not fail with Expected:%s failure", errString) - if ok { - testlog.Info(errString) - } - Eventually(session).Should(gexec.Exit(1)) + Expect(session).Should(gexec.Exit(1)) + + errString := "can't allocate odd number of CPUs from a NUMA Node" + Expect(string(output)).To(ContainSubstring(errString), "expected error:\n%q\ngot:\n%s", errString, output) }) It("[test_id:41420] Verify PPC script fails when reserved cpu count is more than available cpus", func() { @@ -165,14 +164,13 @@ var _ = Describe("[rfe_id: 38968] PerformanceProfile setup helper and platform a podmanArgs := append(defaultArgs, cmdArgs...) session, err := ppcIntgTest.PodmanAsUserBase(podmanArgs, false, false) Expect(err).ToNot(HaveOccurred(), "Podman command failed") + output := session.Wait(20).Err.Contents() - errString := "Error: failed to compute the reserved and isolated CPUs: please specify the reserved CPU count in the range [1,3]" - ok, err := regexp.MatchString(errString, string(output)) - Expect(err).ToNot(HaveOccurred(), "did not fail with Expected:%s failure", errString) - if ok { - testlog.Info(errString) - } - Eventually(session).Should(gexec.Exit(1)) + Expect(session).Should(gexec.Exit(1)) + + errString := fmt.Sprintf("please specify the reserved CPU count in the range [1,%d]", + maxReservedCPUCountFromMustGather(mustgatherDir, mcpName)) + Expect(string(output)).To(ContainSubstring(errString), "expected error:\n%q\ngot:\n%s", errString, output) }) It("[test_id: 54187] PPC generates profile with PerPodPowerManagement workload hint", func() { @@ -191,9 +189,12 @@ var _ = Describe("[rfe_id: 38968] PerformanceProfile setup helper and platform a podmanArgs := append(defaultArgs, cmdArgs...) session, err := ppcIntgTest.PodmanAsUserBase(podmanArgs, false, false) Expect(err).ToNot(HaveOccurred(), "Podman command failed") + output := session.Wait(20).Out.Contents() + Expect(session).Should(gexec.Exit(0)) + err = yaml.Unmarshal(output, pp) - Expect(err).ToNot(HaveOccurred(), "Unable to marshal the ppc output") + Expect(err).ToNot(HaveOccurred(), "Unable to unmarshal the ppc output") Expect(*pp.Spec.WorkloadHints.PerPodPowerManagement).To(BeTrue()) Expect(*pp.Spec.WorkloadHints.HighPowerConsumption).To(BeFalse()) }) @@ -213,14 +214,35 @@ var _ = Describe("[rfe_id: 38968] PerformanceProfile setup helper and platform a podmanArgs := append(defaultArgs, cmdArgs...) session, err := ppcIntgTest.PodmanAsUserBase(podmanArgs, false, false) Expect(err).ToNot(HaveOccurred(), "Podman command failed") + output := session.Wait(20).Err.Contents() - errString := `please use one of \[default low-latency\] power consumption modes together with the perPodPowerManagement` - ok, err := regexp.MatchString(errString, string(output)) - Expect(err).ToNot(HaveOccurred(), "did not fail with Expected:%s failure", errString) - if ok { - testlog.Info(errString) - } - Eventually(session).Should(gexec.Exit(1)) + Expect(session).Should(gexec.Exit(1)) + + errString := "please use one of [default low-latency] power consumption modes together with the perPodPowerManagement" + Expect(string(output)).To(ContainSubstring(errString), "expected error:\n%q\ngot:\n%s", errString, output) }) }) }) + +// maxReservedCPUCountFromMustGather returns TotalThreads-1 from one node in mcpName +// (the upper bound in PPC's "reserved CPU count in the range [1,%d]" error). +func maxReservedCPUCountFromMustGather(mustGatherDir, mcpName string) int { + GinkgoHelper() + dir, err := filepath.Abs(mustGatherDir) + Expect(err).ToNot(HaveOccurred()) + nodes, err := profilecreator.GetNodeList(dir) + Expect(err).ToNot(HaveOccurred()) + mcps, err := profilecreator.GetMCPList(dir) + Expect(err).ToNot(HaveOccurred()) + mcp, err := profilecreator.GetMCP(dir, mcpName) + Expect(err).ToNot(HaveOccurred()) + poolNodes, err := profilecreator.GetNodesForPool(mcp, mcps, nodes) + Expect(err).ToNot(HaveOccurred()) + Expect(poolNodes).ToNot(BeEmpty()) + h, err := profilecreator.NewGHWHandler(dir, poolNodes[0]) + Expect(err).ToNot(HaveOccurred()) + DeferCleanup(h.Cleanup) + cpu, err := h.CPU() + Expect(err).ToNot(HaveOccurred()) + return int(cpu.TotalThreads) - 1 +}