From b72ef42dd4e9be48981995976156e6e516c47bf4 Mon Sep 17 00:00:00 2001 From: Jiri Mencak Date: Thu, 13 Aug 2026 09:25:01 +0200 Subject: [PATCH] Use Add() instead of AddRateLimited() for routine Profile enqueues Change 1: workqueue.DefaultTypedControllerRateLimiter's AddRateLimited() draws from a single 10qps/burst-100 token bucket shared by every item in the queue, regardless of kind or key. Cascade/trigger enqueues (Pod/Node change to Profile update, per-Profile bootstrap loop, initial bootstrap event) used AddRateLimited() for ordinary enqueuing rather than error retries. During a burst of ~100 nodes joining, this exhausted the shared bucket, delaying later Profile syncs by minutes even though they were never dropped, just queued behind the backlog. Switch these call sites to Add(), which is unthrottled and still deduped by key via the delaying queue's dirty set. Keep AddRateLimited() only for its intended purpose: the retry-after- sync()-error path, where per-key exponential backoff is correct. Change 2: validateTunedCRs() called UpdateStatus() on every Tuned CR on every invocation, even when the computed status conditions were identical to what's already stored. Compare the newly computed status against the cached copy with reflect.DeepEqual() and skip the UpdateStatus() call when nothing semantically changed. Resolves: OCPBUGS-105458 --- pkg/operator/controller.go | 10 +++++----- pkg/operator/validator.go | 9 +++++++++ 2 files changed, 14 insertions(+), 5 deletions(-) diff --git a/pkg/operator/controller.go b/pkg/operator/controller.go index 4b0ca75d11..ca56e29129 100644 --- a/pkg/operator/controller.go +++ b/pkg/operator/controller.go @@ -125,7 +125,7 @@ func NewController() (*Controller, error) { controller.mcLabelsAcrossMCP = map[string]bool{} // Initial event to bootstrap CR if it doesn't exist. - controller.workqueue.AddRateLimited(wqKey{kind: wqKindTuned, name: tunedv1.TunedDefaultResourceName}) + controller.workqueue.Add(wqKey{kind: wqKindTuned, name: tunedv1.TunedDefaultResourceName}) controller.clients.Kube, err = kubeset.NewForConfig(controller.kubeconfig) if err != nil { @@ -280,7 +280,7 @@ func (c *Controller) sync(key wqKey) error { if change { klog.V(2).Infof("sync(): Pod %s/%s label(s) change is Node %s wide", key.namespace, key.name, nodeName) // Trigger a Profile update - c.workqueue.AddRateLimited(wqKey{kind: wqKindProfile, namespace: ntoconfig.WatchNamespace(), name: nodeName}) + c.workqueue.Add(wqKey{kind: wqKindProfile, namespace: ntoconfig.WatchNamespace(), name: nodeName}) } return nil @@ -291,7 +291,7 @@ func (c *Controller) sync(key wqKey) error { if err != nil { if errors.IsNotFound(err) { // Trigger Profile update/deletion for this node; syncProfile() will handle the deletion and also update internal data structures - c.workqueue.AddRateLimited(wqKey{kind: wqKindProfile, namespace: ntoconfig.WatchNamespace(), name: key.name}) + c.workqueue.Add(wqKey{kind: wqKindProfile, namespace: ntoconfig.WatchNamespace(), name: key.name}) return nil } return fmt.Errorf("failed to process Node %s change: %v", key.name, err) @@ -300,7 +300,7 @@ func (c *Controller) sync(key wqKey) error { // We need to update Profile associated with the Node klog.V(2).Infof("sync(): Node %s label(s) changed", key.name) // Trigger a Profile update - c.workqueue.AddRateLimited(wqKey{kind: wqKindProfile, namespace: ntoconfig.WatchNamespace(), name: key.name}) + c.workqueue.Add(wqKey{kind: wqKindProfile, namespace: ntoconfig.WatchNamespace(), name: key.name}) } return nil @@ -472,7 +472,7 @@ func (c *Controller) enqueueProfileUpdates() error { } for _, profile := range profileList { // Enqueue Profile updates into the operator's workqueue - c.workqueue.AddRateLimited(wqKey{kind: wqKindProfile, namespace: ntoconfig.WatchNamespace(), name: profile.Name}) + c.workqueue.Add(wqKey{kind: wqKindProfile, namespace: ntoconfig.WatchNamespace(), name: profile.Name}) } return nil } diff --git a/pkg/operator/validator.go b/pkg/operator/validator.go index d124f4ba0d..a16d75ac7e 100644 --- a/pkg/operator/validator.go +++ b/pkg/operator/validator.go @@ -3,6 +3,7 @@ package operator import ( "context" "fmt" + "reflect" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" @@ -60,6 +61,8 @@ func (c *Controller) validateTunedCRs() error { } } + oldStatus := tuned.Status + tuned = tuned.DeepCopy() // Make sure we do not modify objects in cache if len(tuned.Status.Conditions) == 0 { @@ -67,6 +70,12 @@ func (c *Controller) validateTunedCRs() error { } tuned.Status.Conditions = computeStatusConditions(tunedValid, message, tuned.Status.Conditions) + // Update the status subresource only if there is a semantic change in it. + if reflect.DeepEqual(oldStatus, tuned.Status) { + klog.V(2).Infof("validateTunedCRs(): no need to update status of Tuned %s", tuned.Name) + continue + } + _, err = c.clients.Tuned.TunedV1().Tuneds(ntoconfig.WatchNamespace()).UpdateStatus(context.TODO(), tuned, metav1.UpdateOptions{}) if err != nil { return fmt.Errorf("failed to update Tuned %s status: %v", tuned.Name, err)