From 4d92ada7373b4ede36a30f7c49d9b2326ca7bb98 Mon Sep 17 00:00:00 2001 From: Chai Bot Date: Tue, 4 Aug 2026 01:42:20 +0000 Subject: [PATCH 1/4] [ROSAENG-62134] fix: add rate-limit backoff guard for fleet mode HTTP 429 When the OCM API responds with HTTP 429 (Too Many Requests) during service log sends, ocm-agent was retrying indefinitely on every AlertManager webhook delivery, creating a retry storm. This change: - Adds a RateLimitError type in pkg/ocm that wraps 429 responses from SendServiceLog, allowing callers to distinguish rate limits from other errors. - Adds an in-memory per-notification:cluster backoff map in the RHOBS webhook handler. On a 429 response the timestamp is recorded and subsequent firing alerts for that pair are silently skipped for 30 minutes, breaking the retry loop. - Clears the backoff entry on a successful send so normal operation resumes immediately once the rate limit lifts. Co-Authored-By: Claude Opus 4.6 --- pkg/handlers/webhookrhobsreceiver.go | 35 ++++++++++++++++++++++++++++ pkg/ocm/ocm.go | 19 +++++++++++++++ 2 files changed, 54 insertions(+) diff --git a/pkg/handlers/webhookrhobsreceiver.go b/pkg/handlers/webhookrhobsreceiver.go index 7874b476..2b995b72 100644 --- a/pkg/handlers/webhookrhobsreceiver.go +++ b/pkg/handlers/webhookrhobsreceiver.go @@ -3,9 +3,11 @@ package handlers import ( "context" "encoding/json" + stderrors "errors" "fmt" "net/http" "strings" + "sync" "time" "github.com/prometheus/alertmanager/template" @@ -41,6 +43,12 @@ var ( } customIs409 = func(err error) bool { return errors.IsConflict(err) || errors.IsAlreadyExists(err) } + + // rateLimitBackoffs tracks per-notification:cluster backoff timestamps after + // an OCM API 429 rate-limit response. The key is "notification:clusterID" + // and the value is the time.Time when the 429 was received. + rateLimitBackoffs sync.Map + rateLimitRetryInterval = 30 * time.Minute ) type WebhookRHOBSReceiverHandler struct { @@ -380,6 +388,21 @@ func (h *WebhookRHOBSReceiverHandler) processAlert(alert template.Alert, isCurre return nil } + // Skip firing alerts that are within the rate-limit backoff window. + if isCurrentlyFiring { + rateLimitKey := alert.Labels[AMLabelTemplateName] + ":" + alert.Labels[AMLabelAlertHCID] + if backoffTime, ok := rateLimitBackoffs.Load(rateLimitKey); ok { + if time.Since(backoffTime.(time.Time)) < rateLimitRetryInterval { + log.WithFields(log.Fields{ + LogFieldNotificationName: alert.Labels[AMLabelTemplateName], + "hosted_cluster_id": alert.Labels[AMLabelAlertHCID], + }).Warn("skipping alert due to OCM API rate-limit backoff") + return nil + } + rateLimitBackoffs.Delete(rateLimitKey) + } + } + var c *fleetNotificationContext canSend := false err = retryOnConflictOrAlreadyExists(retryConfig, func() error { @@ -413,6 +436,15 @@ func (h *WebhookRHOBSReceiverHandler) processAlert(alert template.Alert, isCurre } if err != nil { + var rateLimitErr *ocm.RateLimitError + if stderrors.As(err, &rateLimitErr) { + rateLimitKey := alert.Labels[AMLabelTemplateName] + ":" + alert.Labels[AMLabelAlertHCID] + rateLimitBackoffs.Store(rateLimitKey, time.Now()) + log.WithFields(log.Fields{ + LogFieldNotificationName: fleetNotification.Name, + "hosted_cluster_id": alert.Labels[AMLabelAlertHCID], + }).Warn("OCM API rate limit hit (HTTP 429), backing off for 30 minutes") + } if fleetNotification.LimitedSupport { // Limited support case metrics.IncrementFailedLimitedSupportSend(fleetNotification.Name) } else { // Service log case @@ -423,6 +455,9 @@ func (h *WebhookRHOBSReceiverHandler) processAlert(alert template.Alert, isCurre return err } + // Clear any rate-limit backoff for this notification:cluster on success. + rateLimitBackoffs.Delete(alert.Labels[AMLabelTemplateName] + ":" + alert.Labels[AMLabelAlertHCID]) + if fleetNotification.LimitedSupport { // Limited support case metrics.IncrementLimitedSupportSentCount(fleetNotification.Name) } else { // Service log case diff --git a/pkg/ocm/ocm.go b/pkg/ocm/ocm.go index 94dcde93..a3247cbe 100644 --- a/pkg/ocm/ocm.go +++ b/pkg/ocm/ocm.go @@ -20,6 +20,19 @@ const ( ServiceLogResolvePrefix = "Issue Resolution" ) +// RateLimitError indicates that an OCM API call was rejected with HTTP 429. +type RateLimitError struct { + Err error +} + +func (e *RateLimitError) Error() string { + return e.Err.Error() +} + +func (e *RateLimitError) Unwrap() error { + return e.Err +} + type ServiceLogBuilder struct { wrappedBuilder *slv1.LogEntryBuilder summary string @@ -242,10 +255,16 @@ func (o *ocmClientImpl) SendServiceLog(logEntry *slv1.LogEntry) error { // Send the request to the OCM API. response, err := request.Send() if err != nil { + if response != nil && response.Status() == http.StatusTooManyRequests { + return &RateLimitError{Err: fmt.Errorf("can't post service log: rate limited (HTTP 429): %v", err)} + } return fmt.Errorf("can't post service log: %v", err) } // Check the response status code. + if response.Status() == http.StatusTooManyRequests { + return &RateLimitError{Err: fmt.Errorf("can't post service log: rate limited (HTTP 429)")} + } if response.Status() != http.StatusCreated { // Extract error details from the response and return an appropriate error. return fmt.Errorf("unexpected status: %d", response.Status()) From bf841c0003d3c7873389adba7371bb240eba1a19 Mon Sep 17 00:00:00 2001 From: Chai Bot Date: Tue, 4 Aug 2026 01:56:49 +0000 Subject: [PATCH 2/4] [ROSAENG-62134] test: add unit tests for rate-limit backoff logic Adds unit tests for both the RateLimitError type and the rate-limit backoff behavior in the fleet webhook handler: - RateLimitError: Error(), Unwrap(), errors.As() traversal - SendServiceLog: returns *RateLimitError on 429, plain error on 500 - processAlert: backoff map populated on 429, subsequent alerts skipped within window, normal operation after expiry, entry cleared on success Also addresses CodeRabbit review feedback: - Remove hosted_cluster_id from warning logs (customer data concern) - Use %w instead of %v to preserve original error in RateLimitError - Remove eager Delete of expired entries (race-safe: stale expired entries are harmless since the pre-send check only blocks within the 30-minute window) Co-Authored-By: Claude Opus 4.6 --- pkg/handlers/webhookrhobsreceiver.go | 6 +- pkg/handlers/webhookrhobsreceiver_test.go | 121 ++++++++++++++++++++++ pkg/ocm/ocm.go | 2 +- pkg/ocm/ocm_test.go | 90 ++++++++++++++++ 4 files changed, 214 insertions(+), 5 deletions(-) diff --git a/pkg/handlers/webhookrhobsreceiver.go b/pkg/handlers/webhookrhobsreceiver.go index 2b995b72..33a55574 100644 --- a/pkg/handlers/webhookrhobsreceiver.go +++ b/pkg/handlers/webhookrhobsreceiver.go @@ -395,11 +395,9 @@ func (h *WebhookRHOBSReceiverHandler) processAlert(alert template.Alert, isCurre if time.Since(backoffTime.(time.Time)) < rateLimitRetryInterval { log.WithFields(log.Fields{ LogFieldNotificationName: alert.Labels[AMLabelTemplateName], - "hosted_cluster_id": alert.Labels[AMLabelAlertHCID], }).Warn("skipping alert due to OCM API rate-limit backoff") return nil } - rateLimitBackoffs.Delete(rateLimitKey) } } @@ -442,7 +440,6 @@ func (h *WebhookRHOBSReceiverHandler) processAlert(alert template.Alert, isCurre rateLimitBackoffs.Store(rateLimitKey, time.Now()) log.WithFields(log.Fields{ LogFieldNotificationName: fleetNotification.Name, - "hosted_cluster_id": alert.Labels[AMLabelAlertHCID], }).Warn("OCM API rate limit hit (HTTP 429), backing off for 30 minutes") } if fleetNotification.LimitedSupport { // Limited support case @@ -455,7 +452,8 @@ func (h *WebhookRHOBSReceiverHandler) processAlert(alert template.Alert, isCurre return err } - // Clear any rate-limit backoff for this notification:cluster on success. + // Clear any rate-limit backoff for this notification:cluster pair + // after a successful send so normal operation resumes immediately. rateLimitBackoffs.Delete(alert.Labels[AMLabelTemplateName] + ":" + alert.Labels[AMLabelAlertHCID]) if fleetNotification.LimitedSupport { // Limited support case diff --git a/pkg/handlers/webhookrhobsreceiver_test.go b/pkg/handlers/webhookrhobsreceiver_test.go index 571f126b..985613c7 100644 --- a/pkg/handlers/webhookrhobsreceiver_test.go +++ b/pkg/handlers/webhookrhobsreceiver_test.go @@ -664,3 +664,124 @@ func (f *FailingResponseWriter) Write([]byte) (int, error) { func (f *FailingResponseWriter) WriteHeader(statusCode int) { f.statusCode = statusCode } + +func clearRateLimitBackoffs() { + rateLimitBackoffs.Range(func(k, v any) bool { + rateLimitBackoffs.Delete(k) + return true + }) +} + +var _ = Describe("Rate-limit backoff behavior", func() { + var ( + mockCtrl *gomock.Controller + mockClient *clientmocks.MockClient + mockOCMClient *webhookreceivermock.MockOCMClient + testHandler *WebhookRHOBSReceiverHandler + testAlertFiring template.Alert + mockStatusWriter *clientmocks.MockStatusWriter + serviceLog *ocm.ServiceLog + managedFleetNotification *ocmagentv1alpha1.ManagedFleetNotification + managedFleetNotificationRecord *ocmagentv1alpha1.ManagedFleetNotificationRecord + ) + + BeforeEach(func() { + clearRateLimitBackoffs() + + mockCtrl = gomock.NewController(GinkgoT()) + mockClient = clientmocks.NewMockClient(mockCtrl) + mockStatusWriter = clientmocks.NewMockStatusWriter(mockCtrl) + mockOCMClient = webhookreceivermock.NewMockOCMClient(mockCtrl) + testHandler = &WebhookRHOBSReceiverHandler{ + c: mockClient, + ocm: mockOCMClient, + } + testAlertFiring = testconst.NewTestAlert(false, true) + + defaultMFN := testconst.NewManagedFleetNotification(false) + defaultMFN.Spec.FleetNotification.ResendWait = 1 + managedFleetNotification = &defaultMFN + + defaultRecord := testconst.NewManagedFleetNotificationRecordWithStatus() + managedFleetNotificationRecord = &defaultRecord + + serviceLog = testconst.NewTestServiceLog( + ocm.ServiceLogActivePrefix+": "+testconst.ServiceLogSummary, + testconst.ServiceLogFleetDesc, + testconst.TestHostedClusterID, + testconst.TestNotification.Severity, + "", + testconst.TestNotification.References) + + // Setup k8s mocks for ManagedFleetNotification and Record retrieval + mockClient.EXPECT().Get(gomock.Any(), client.ObjectKey{ + Namespace: OCMAgentNamespaceName, + Name: managedFleetNotification.ObjectMeta.Name, + }, gomock.Any()).DoAndReturn( + func(ctx context.Context, key client.ObjectKey, res *ocmagentv1alpha1.ManagedFleetNotification, opts ...client.GetOption) error { + *res = *managedFleetNotification + return nil + }).AnyTimes() + + mockClient.EXPECT().Get(gomock.Any(), client.ObjectKey{ + Namespace: OCMAgentNamespaceName, + Name: managedFleetNotificationRecord.ObjectMeta.Name, + }, gomock.Any()).DoAndReturn( + func(ctx context.Context, key client.ObjectKey, res *ocmagentv1alpha1.ManagedFleetNotificationRecord, opts ...client.GetOption) error { + *res = *managedFleetNotificationRecord + return nil + }).AnyTimes() + + mockClient.EXPECT().Status().Return(mockStatusWriter).AnyTimes() + mockStatusWriter.EXPECT().Update(gomock.Any(), gomock.Any(), gomock.Any()).Return(nil).AnyTimes() + }) + + AfterEach(func() { + clearRateLimitBackoffs() + mockCtrl.Finish() + }) + + It("populates rateLimitBackoffs when SendServiceLog returns RateLimitError", func() { + rateLimitErr := &ocm.RateLimitError{Err: fmt.Errorf("rate limited")} + mockOCMClient.EXPECT().SendServiceLog(serviceLog).Return(rateLimitErr) + + err := testHandler.processAlert(testAlertFiring, true) + Expect(err).To(HaveOccurred()) + + key := testconst.TestNotificationName + ":" + testconst.TestHostedClusterID + _, ok := rateLimitBackoffs.Load(key) + Expect(ok).To(BeTrue()) + }) + + It("skips sending when within the rate-limit backoff window", func() { + key := testconst.TestNotificationName + ":" + testconst.TestHostedClusterID + rateLimitBackoffs.Store(key, time.Now()) + + // SendServiceLog should NOT be called because the backoff guard returns early + err := testHandler.processAlert(testAlertFiring, true) + Expect(err).ShouldNot(HaveOccurred()) + }) + + It("proceeds normally after the backoff window expires", func() { + key := testconst.TestNotificationName + ":" + testconst.TestHostedClusterID + rateLimitBackoffs.Store(key, time.Now().Add(-rateLimitRetryInterval-time.Minute)) + + mockOCMClient.EXPECT().SendServiceLog(serviceLog).Return(nil) + + err := testHandler.processAlert(testAlertFiring, true) + Expect(err).ShouldNot(HaveOccurred()) + }) + + It("clears backoff entry on successful send", func() { + key := testconst.TestNotificationName + ":" + testconst.TestHostedClusterID + rateLimitBackoffs.Store(key, time.Now().Add(-rateLimitRetryInterval-time.Minute)) + + mockOCMClient.EXPECT().SendServiceLog(serviceLog).Return(nil) + + err := testHandler.processAlert(testAlertFiring, true) + Expect(err).ShouldNot(HaveOccurred()) + + _, ok := rateLimitBackoffs.Load(key) + Expect(ok).To(BeFalse()) + }) +}) diff --git a/pkg/ocm/ocm.go b/pkg/ocm/ocm.go index a3247cbe..a3ffd1cc 100644 --- a/pkg/ocm/ocm.go +++ b/pkg/ocm/ocm.go @@ -256,7 +256,7 @@ func (o *ocmClientImpl) SendServiceLog(logEntry *slv1.LogEntry) error { response, err := request.Send() if err != nil { if response != nil && response.Status() == http.StatusTooManyRequests { - return &RateLimitError{Err: fmt.Errorf("can't post service log: rate limited (HTTP 429): %v", err)} + return &RateLimitError{Err: fmt.Errorf("can't post service log: rate limited (HTTP 429): %w", err)} } return fmt.Errorf("can't post service log: %v", err) } diff --git a/pkg/ocm/ocm_test.go b/pkg/ocm/ocm_test.go index 51cf5b29..e6878084 100644 --- a/pkg/ocm/ocm_test.go +++ b/pkg/ocm/ocm_test.go @@ -1,6 +1,7 @@ package ocm import ( + "errors" "fmt" "net/http" "testing" @@ -312,6 +313,95 @@ var _ = Describe("OCM client Handler", func() { }) }) + + Context("RateLimitError type", func() { + It("Error() returns the wrapped error message", func() { + inner := fmt.Errorf("some error") + rle := &RateLimitError{Err: inner} + Expect(rle.Error()).To(Equal("some error")) + }) + + It("Unwrap() returns the inner error", func() { + inner := fmt.Errorf("some error") + rle := &RateLimitError{Err: inner} + Expect(rle.Unwrap()).To(Equal(inner)) + }) + + It("errors.As() detects RateLimitError from a wrapped error chain", func() { + inner := fmt.Errorf("sdk error") + rle := &RateLimitError{Err: inner} + wrapped := fmt.Errorf("outer context: %w", rle) + + var target *RateLimitError + Expect(errors.As(wrapped, &target)).To(BeTrue()) + Expect(target.Err.Error()).To(Equal("sdk error")) + }) + + It("errors.As() does not match a plain error as RateLimitError", func() { + plainErr := fmt.Errorf("just an error") + var target *RateLimitError + Expect(errors.As(plainErr, &target)).To(BeFalse()) + }) + }) + + Context("SendServiceLog returns RateLimitError on 429", func() { + It("should return a RateLimitError when the API responds with HTTP 429", func() { + mockServer.SetHandler(0, CombineHandlers( + VerifyRequest("POST", "/api/service_logs/v1/cluster_logs"), + RespondWith( + http.StatusTooManyRequests, + `{"kind": "Error", "id": "429", "href": "/api/service_logs/v1/errors/429", "code": "SERVICE-LOGS-429", "reason": "Rate limit exceeded"}`, + http.Header{"Content-Type": []string{"application/json"}}, + ), + )) + // The OCM SDK may retry on certain errors, so add extra handlers + for i := 0; i < 5; i++ { + mockServer.AppendHandlers(CombineHandlers( + VerifyRequest("POST", "/api/service_logs/v1/cluster_logs"), + RespondWith( + http.StatusTooManyRequests, + `{"kind": "Error", "id": "429", "href": "/api/service_logs/v1/errors/429", "code": "SERVICE-LOGS-429", "reason": "Rate limit exceeded"}`, + http.Header{"Content-Type": []string{"application/json"}}, + ), + )) + } + + err := ocmClient.SendServiceLog(serviceLog) + Expect(err).To(HaveOccurred()) + + var rateLimitErr *RateLimitError + Expect(errors.As(err, &rateLimitErr)).To(BeTrue()) + Expect(rateLimitErr.Error()).To(ContainSubstring("rate limited")) + }) + + It("should return a non-RateLimitError for other failure statuses", func() { + mockServer.SetHandler(0, CombineHandlers( + VerifyRequest("POST", "/api/service_logs/v1/cluster_logs"), + RespondWith( + http.StatusInternalServerError, + `{"kind": "Error", "id": "500", "href": "/api/service_logs/v1/errors/500", "code": "SERVICE-LOGS-500", "reason": "Internal server error"}`, + http.Header{"Content-Type": []string{"application/json"}}, + ), + )) + for i := 0; i < 5; i++ { + mockServer.AppendHandlers(CombineHandlers( + VerifyRequest("POST", "/api/service_logs/v1/cluster_logs"), + RespondWith( + http.StatusInternalServerError, + `{"kind": "Error", "id": "500", "href": "/api/service_logs/v1/errors/500", "code": "SERVICE-LOGS-500", "reason": "Internal server error"}`, + http.Header{"Content-Type": []string{"application/json"}}, + ), + )) + } + + err := ocmClient.SendServiceLog(serviceLog) + Expect(err).To(HaveOccurred()) + + var rateLimitErr *RateLimitError + Expect(errors.As(err, &rateLimitErr)).To(BeFalse()) + }) + }) + Context("Limit support", func() { It("should not return an error on successful post", func() { mockServer.SetHandler(0, CombineHandlers( From 7a10962332a232387a8bc6f211b9d403a92da7bd Mon Sep 17 00:00:00 2001 From: Chai Bot Date: Tue, 4 Aug 2026 02:18:15 +0000 Subject: [PATCH 3/4] [ROSAENG-62134] fix: make success-path backoff cleanup race-safe Co-Authored-By: Claude Opus 4.6 --- pkg/handlers/webhookrhobsreceiver.go | 10 +++++++++- pkg/handlers/webhookrhobsreceiver_test.go | 22 ++++++++++++++++++++++ 2 files changed, 31 insertions(+), 1 deletion(-) diff --git a/pkg/handlers/webhookrhobsreceiver.go b/pkg/handlers/webhookrhobsreceiver.go index 33a55574..9b9d283d 100644 --- a/pkg/handlers/webhookrhobsreceiver.go +++ b/pkg/handlers/webhookrhobsreceiver.go @@ -424,6 +424,7 @@ func (h *WebhookRHOBSReceiverHandler) processAlert(alert template.Alert, isCurre if isCurrentlyFiring { if canSend { + sendStartTime := time.Now() err := c.sendNotification(h.ocm, alert) var logService string @@ -454,7 +455,14 @@ func (h *WebhookRHOBSReceiverHandler) processAlert(alert template.Alert, isCurre // Clear any rate-limit backoff for this notification:cluster pair // after a successful send so normal operation resumes immediately. - rateLimitBackoffs.Delete(alert.Labels[AMLabelTemplateName] + ":" + alert.Labels[AMLabelAlertHCID]) + // Only delete if the stored timestamp predates this send attempt, + // so a concurrent 429 that arrived while we were sending is preserved. + rateLimitKey := alert.Labels[AMLabelTemplateName] + ":" + alert.Labels[AMLabelAlertHCID] + if val, ok := rateLimitBackoffs.Load(rateLimitKey); ok { + if val.(time.Time).Before(sendStartTime) { + rateLimitBackoffs.Delete(rateLimitKey) + } + } if fleetNotification.LimitedSupport { // Limited support case metrics.IncrementLimitedSupportSentCount(fleetNotification.Name) diff --git a/pkg/handlers/webhookrhobsreceiver_test.go b/pkg/handlers/webhookrhobsreceiver_test.go index 985613c7..a3b9103f 100644 --- a/pkg/handlers/webhookrhobsreceiver_test.go +++ b/pkg/handlers/webhookrhobsreceiver_test.go @@ -784,4 +784,26 @@ var _ = Describe("Rate-limit backoff behavior", func() { _, ok := rateLimitBackoffs.Load(key) Expect(ok).To(BeFalse()) }) + + It("does not clear a newer backoff stored during an in-flight send", func() { + key := testconst.TestNotificationName + ":" + testconst.TestHostedClusterID + // Seed an old, expired backoff so the request is not skipped. + rateLimitBackoffs.Store(key, time.Now().Add(-rateLimitRetryInterval-time.Minute)) + + // Simulate a concurrent 429: while SendServiceLog is running, + // another goroutine stores a fresh backoff timestamp. + mockOCMClient.EXPECT().SendServiceLog(serviceLog).DoAndReturn( + func(sl *ocm.ServiceLog) error { + rateLimitBackoffs.Store(key, time.Now()) + return nil + }, + ) + + err := testHandler.processAlert(testAlertFiring, true) + Expect(err).ShouldNot(HaveOccurred()) + + // The fresh backoff must survive the success-path cleanup. + _, ok := rateLimitBackoffs.Load(key) + Expect(ok).To(BeTrue()) + }) }) From aaad6bcf832b07c6382b157460a3dffb3afaa82c Mon Sep 17 00:00:00 2001 From: Chai Bot Date: Thu, 6 Aug 2026 05:26:58 +0000 Subject: [PATCH 4/4] [ROSAENG-62134] fix: clear rate-limit backoff on alert resolution Co-Authored-By: Claude Opus 4.6 --- pkg/handlers/webhookrhobsreceiver.go | 7 +++++++ pkg/handlers/webhookrhobsreceiver_test.go | 12 ++++++++++++ 2 files changed, 19 insertions(+) diff --git a/pkg/handlers/webhookrhobsreceiver.go b/pkg/handlers/webhookrhobsreceiver.go index 9b9d283d..595004f2 100644 --- a/pkg/handlers/webhookrhobsreceiver.go +++ b/pkg/handlers/webhookrhobsreceiver.go @@ -384,6 +384,13 @@ func (h *WebhookRHOBSReceiverHandler) processAlert(alert template.Alert, isCurre return fmt.Errorf("unable to find ManagedFleetNotification %s", alert.Labels[AMLabelTemplateName]) } + // When an alert resolves, clear any rate-limit backoff entry so that + // if the alert fires again later, the send is not suppressed. + if !isCurrentlyFiring { + rateLimitKey := alert.Labels[AMLabelTemplateName] + ":" + alert.Labels[AMLabelAlertHCID] + rateLimitBackoffs.Delete(rateLimitKey) + } + if !fleetNotificationRetriever.fleetNotification.LimitedSupport && !isCurrentlyFiring { return nil } diff --git a/pkg/handlers/webhookrhobsreceiver_test.go b/pkg/handlers/webhookrhobsreceiver_test.go index a3b9103f..894a6e62 100644 --- a/pkg/handlers/webhookrhobsreceiver_test.go +++ b/pkg/handlers/webhookrhobsreceiver_test.go @@ -785,6 +785,18 @@ var _ = Describe("Rate-limit backoff behavior", func() { Expect(ok).To(BeFalse()) }) + It("clears backoff entry when a resolved alert is received", func() { + key := testconst.TestNotificationName + ":" + testconst.TestHostedClusterID + rateLimitBackoffs.Store(key, time.Now()) + + testAlertResolved := testconst.NewTestAlert(true, true) + err := testHandler.processAlert(testAlertResolved, false) + Expect(err).ShouldNot(HaveOccurred()) + + _, ok := rateLimitBackoffs.Load(key) + Expect(ok).To(BeFalse()) + }) + It("does not clear a newer backoff stored during an in-flight send", func() { key := testconst.TestNotificationName + ":" + testconst.TestHostedClusterID // Seed an old, expired backoff so the request is not skipped.