fix(hpa): pause HPAs via scaleUp selectPolicy instead of breaking scaleTargetRef

CHANGELOG.md

@@ -1,5 +1,18 @@
 # Changelog
 
+## v7.0.1 (2026-07-09)
+
+### Fix
+- Pause HPAs during migrations via `spec.behavior.scaleUp.selectPolicy: Disabled` instead of
+  renaming their `scaleTargetRef` to a non-existent Deployment. The rename made the HPA emit
+  `FailedGetScale` and turn ArgoCD applications **Degraded** for the whole migration window,
+  and was reverted by any GitOps sync anyway. The pause keeps the HPA Healthy while forbidding
+  scale-up; the pre-pause `spec.behavior` is saved in the `wiremind.io/pre-pause-scale-behavior`
+  annotation and restored exactly on resume. Because a leftover pause is not cleaned up by
+  GitOps (the field is invisible to ArgoCD), `start_pods()` and `restore_stopped_pods()` now
+  resume HPAs unconditionally for every tracked Deployment. The old rename-repair path is kept
+  for HPAs left broken by a pre-7.0.1 run (removal planned for 8.0).
+
 ## v7.0.0 (2026-07-08)
 
 ### BREAKING CHANGE

VERSION

@@ -1 +1 @@
-7.0.0
+7.0.1

src/chartreuse/kubernetes_helper.py

@@ -9,6 +9,7 @@
 """
 
 import functools
+import json
 import logging
 import os
 import pprint
@@ -20,10 +21,16 @@
 
 logger = logging.getLogger(__name__)
 
+# Target-rename prefix used by the pre-7.0.1 HPA disable mechanism. Only the repair path
+# (re_enable_hpa) remains, for HPAs left renamed by an older chartreuse.
+# TODO: remove in 8.0 together with re_enable_hpa.
 HPA_ID_PREFIX = "wm--disabled--kube"
 # Set by stop_pods() on every Deployment it scales down, cleared by start_pods() and
 # restore_stopped_pods(). Allows telling "stopped by us" apart from "deliberately scaled to 0".
 STOPPED_ANNOTATION = "wiremind.io/stopped-by"
+# Set by pause_hpa() on the HPA itself; stores the pre-pause spec.behavior as JSON ("" when
+# absent) so resume_hpa() can restore it exactly. Presence of the annotation == "paused by us".
+PAUSED_ANNOTATION = "wiremind.io/pre-pause-scale-behavior"
 
 
 def load_kubernetes_config(use_kubeconfig: bool | None = None) -> None:
@@ -254,14 +261,51 @@ def _get_expected_deployment_scale_dict(self) -> dict[int, dict[str, int]]:
         return eds_dict
 
     @retry_kubernetes_request
-    def disable_hpa(self, *, deployment_name: str) -> None:
+    def pause_hpa(self, *, deployment_name: str) -> None:
+        """
+        Forbid the Deployment's HPAs from scaling up (spec.behavior.scaleUp.selectPolicy:
+        Disabled) so they can't start pods mid-migration. Unlike breaking the scaleTargetRef,
+        the HPA stays Healthy: no FailedGetScale events, no Degraded ArgoCD application.
+
+        The pre-pause spec.behavior is saved in PAUSED_ANNOTATION so resume_hpa() can restore
+        it exactly. Idempotent: an already-paused HPA is left alone (never overwrite the
+        saved original with a paused state).
+        """
         for hpa in self.get_deployment_hpa(deployment_name=deployment_name):
-            # Tell the hpa to manage a non-existing Deployment
-            hpa.spec.scale_target_ref.name = f"{HPA_ID_PREFIX}-{deployment_name}"
-            self.patch_deployment_hpa(hpa_name=hpa.metadata.name, body=hpa)
+            if PAUSED_ANNOTATION in (hpa.metadata.annotations or {}):
+                continue
+            original_behavior = self.client_autoscalingv2_api.api_client.sanitize_for_serialization(hpa.spec.behavior)
+            saved = json.dumps(original_behavior) if original_behavior else ""
+            body = {
+                "metadata": {"annotations": {PAUSED_ANNOTATION: saved}},
+                "spec": {"behavior": {"scaleUp": {"selectPolicy": "Disabled"}}},
+            }
+            self.patch_deployment_hpa(hpa_name=hpa.metadata.name, body=body)
+
+    @retry_kubernetes_request
+    def resume_hpa(self, *, deployment_name: str) -> None:
+        """
+        Undo pause_hpa(): restore the exact pre-pause spec.behavior (or clear it when there
+        was none) and drop PAUSED_ANNOTATION. No-op on HPAs that don't carry the annotation,
+        so it is always safe to call.
+        """
+        for hpa in self.get_deployment_hpa(deployment_name=deployment_name):
+            saved = (hpa.metadata.annotations or {}).get(PAUSED_ANNOTATION)
+            if saved is None:
+                continue
+            body = {
+                "metadata": {"annotations": {PAUSED_ANNOTATION: None}},
+                "spec": {"behavior": json.loads(saved) if saved else None},
+            }
+            self.patch_deployment_hpa(hpa_name=hpa.metadata.name, body=body)
 
     @retry_kubernetes_request
     def re_enable_hpa(self, *, deployment_name: str) -> None:
+        """
+        Repair an HPA left pointing at a wm--disabled--kube- target by a pre-7.0.1 chartreuse
+        whose run crashed before re-enabling. 7.0.1+ never creates such renames.
+        TODO: remove in 8.0.
+        """
         for hpa in self.get_deployment_hpa(deployment_name=f"{HPA_ID_PREFIX}-{deployment_name}"):
             hpa.spec.scale_target_ref.name = deployment_name
             self.patch_deployment_hpa(hpa_name=hpa.metadata.name, body=hpa)
@@ -278,9 +322,9 @@ def _stop_deployments(self, deployment_dict: dict[str, int]) -> None:
         """
         for deployment_name in deployment_dict:
             self.annotate_deployment(deployment_name, {STOPPED_ANNOTATION: self.release_name})
+            self.pause_hpa(deployment_name=deployment_name)
         for _ in range(self.SCALE_DOWN_MAX_WAIT_TIME):
             for deployment_name in deployment_dict:
-                self.disable_hpa(deployment_name=deployment_name)
                 self.scale_down_deployment(deployment_name)
             if self._are_deployments_stopped(deployment_dict):
                 break
@@ -324,6 +368,7 @@ def start_pods(self) -> None:
             if len(priority_dict):
                 scaled = True
                 for name, expected_scale in priority_dict.items():
+                    self.resume_hpa(deployment_name=name)
                     self.re_enable_hpa(deployment_name=name)
                     self.scale_up_deployment(name, expected_scale)
                     self.annotate_deployment(name, {STOPPED_ANNOTATION: None})
@@ -336,8 +381,11 @@ def restore_stopped_pods(self) -> None:
         """
         Restore Deployments that stop_pods() scaled down and that nothing scaled back up since.
 
-        Only Deployments still carrying STOPPED_ANNOTATION are touched, so a Deployment
-        deliberately scaled to zero outside of stop_pods() is left alone.
+        Only the replica restore is gated on STOPPED_ANNOTATION, so a Deployment deliberately
+        scaled to zero outside of stop_pods() is left alone. HPA resume/repair on the other
+        hand runs for EVERY tracked Deployment: a pause left behind by a crashed run is not
+        self-healing (ArgoCD ignores the live-only spec.behavior field), and resuming an
+        unpaused HPA is a no-op.
 
         Meant to run after the deployment that follows a stop_pods() without start_pods()
         (CHARTREUSE_UPGRADE_BEFORE_DEPLOYMENT): that deployment restores spec.replicas of
@@ -351,6 +399,8 @@ def restore_stopped_pods(self) -> None:
         restored: list[str] = []
         for priority in sorted(expected_deployment_scale_dict):
             for name, expected_scale in expected_deployment_scale_dict[priority].items():
+                self.resume_hpa(deployment_name=name)
+                self.re_enable_hpa(deployment_name=name)
                 try:
                     deployment = self.client_appsv1_api.read_namespaced_deployment(name, self.namespace)
                 except kubernetes.client.rest.ApiException as e:
@@ -359,7 +409,6 @@ def restore_stopped_pods(self) -> None:
                     raise
                 if STOPPED_ANNOTATION not in (deployment.metadata.annotations or {}):
                     continue
-                self.re_enable_hpa(deployment_name=name)
                 if not deployment.spec.replicas:
                     target_scale = expected_scale
                     if target_scale < 1 and any(self.get_deployment_hpa(deployment_name=name)):

src/chartreuse/tests/unit_tests/test_kubernetes_helper.py

@@ -1,9 +1,10 @@
+import json
 import unittest
 
 import kubernetes
 from pytest_mock import MockerFixture
 
-from chartreuse.kubernetes_helper import STOPPED_ANNOTATION, KubernetesDeploymentManager
+from chartreuse.kubernetes_helper import PAUSED_ANNOTATION, STOPPED_ANNOTATION, KubernetesDeploymentManager
 
 
 def _make_kdm(mocker: MockerFixture) -> KubernetesDeploymentManager:
@@ -59,7 +60,7 @@ def test_stop_deployments_correctly_wait(mocker: MockerFixture) -> None:
     """
     kdm = _make_kdm(mocker)
     deployment_dict = {"my-pod": 42, "my-other-pod": 113}
-    mocker.patch.object(kdm, "disable_hpa")
+    mocker.patch.object(kdm, "pause_hpa")
     mocker.patch.object(kdm, "scale_down_deployment")
     mocker.patch.object(kdm, "annotate_deployment")
     mocked_are_deployments_stopped = mocker.patch.object(
@@ -82,7 +83,7 @@ def test_stop_deployments_annotates_stopped_deployments(mocker: MockerFixture) -
     """
     kdm = _make_kdm(mocker)
     mocker.patch.object(kdm, "_are_deployments_stopped", return_value=True)
-    mocker.patch.object(kdm, "disable_hpa")
+    mocker.patch.object(kdm, "pause_hpa")
     mocker.patch.object(kdm, "scale_down_deployment")
     mocked_annotate = mocker.patch.object(kdm, "annotate_deployment")
 
@@ -97,13 +98,17 @@ def test_stop_deployments_annotates_stopped_deployments(mocker: MockerFixture) -
 def test_start_pods_clears_stopped_annotation(mocker: MockerFixture) -> None:
     kdm = _make_kdm(mocker)
     mocker.patch.object(kdm, "_get_expected_deployment_scale_dict", return_value={0: {"worker": 2}})
-    mocker.patch.object(kdm, "re_enable_hpa")
+    mocked_resume = mocker.patch.object(kdm, "resume_hpa")
+    mocked_re_enable = mocker.patch.object(kdm, "re_enable_hpa")
     mocker.patch.object(kdm, "scale_up_deployment")
     mocked_annotate = mocker.patch.object(kdm, "annotate_deployment")
 
     kdm.start_pods()
 
     assert mocked_annotate.mock_calls == [unittest.mock.call("worker", {STOPPED_ANNOTATION: None})]
+    assert mocked_resume.mock_calls == [unittest.mock.call(deployment_name="worker")]
+    # Rename-repair transition shim for HPAs left broken by a pre-7.0.1 chartreuse
+    assert mocked_re_enable.mock_calls == [unittest.mock.call(deployment_name="worker")]
 
 
 def test_restore_stopped_pods(mocker: MockerFixture) -> None:
@@ -136,16 +141,23 @@ def read_deployment(name: str, namespace: str) -> unittest.mock.MagicMock:
         return deployments[name]
 
     kdm.client_appsv1_api.read_namespaced_deployment.side_effect = read_deployment
+    mocked_resume_hpa = mocker.patch.object(kdm, "resume_hpa")
     mocked_re_enable_hpa = mocker.patch.object(kdm, "re_enable_hpa")
     mocked_scale_up = mocker.patch.object(kdm, "scale_up_deployment")
     mocked_annotate = mocker.patch.object(kdm, "annotate_deployment")
 
     kdm.restore_stopped_pods()
 
-    assert mocked_re_enable_hpa.mock_calls == [
+    # HPA resume (and rename-repair) is unconditional: a pause left behind by a crashed run
+    # must always be cleaned up, even on Deployments that won't be scaled (or are gone).
+    expected_hpa_calls = [
         unittest.mock.call(deployment_name="stopped-worker"),
+        unittest.mock.call(deployment_name="untouched-worker"),
         unittest.mock.call(deployment_name="already-scaled"),
+        unittest.mock.call(deployment_name="gone"),
     ]
+    assert mocked_resume_hpa.mock_calls == expected_hpa_calls
+    assert mocked_re_enable_hpa.mock_calls == expected_hpa_calls
     assert mocked_scale_up.mock_calls == [unittest.mock.call("stopped-worker", 2)]
     assert mocked_annotate.mock_calls == [
         unittest.mock.call("stopped-worker", {STOPPED_ANNOTATION: None}),
@@ -163,6 +175,7 @@ def test_restore_stopped_pods_rearms_hpa_when_expected_scale_is_zero(mocker: Moc
     kdm.client_appsv1_api.read_namespaced_deployment.return_value = _make_deployment(
         mocker, {STOPPED_ANNOTATION: "concerned"}, 0
     )
+    mocker.patch.object(kdm, "resume_hpa")
     mocker.patch.object(kdm, "re_enable_hpa")
     mocker.patch.object(kdm, "get_deployment_hpa", return_value=iter([mocker.MagicMock()]))
     mocked_scale_up = mocker.patch.object(kdm, "scale_up_deployment")
@@ -171,3 +184,94 @@ def test_restore_stopped_pods_rearms_hpa_when_expected_scale_is_zero(mocker: Moc
     kdm.restore_stopped_pods()
 
     assert mocked_scale_up.mock_calls == [unittest.mock.call("worker", 1)]
+
+
+def _make_hpa(
+    mocker: MockerFixture, name: str, annotations: dict | None, behavior: dict | None
+) -> unittest.mock.MagicMock:
+    hpa = mocker.MagicMock()
+    hpa.metadata.name = name
+    hpa.metadata.annotations = annotations
+    hpa.spec.behavior = behavior
+    return hpa
+
+
+def test_pause_hpa_saves_behavior_and_disables_scale_up(mocker: MockerFixture) -> None:
+    """
+    Test that pause_hpa stores the pre-pause spec.behavior in PAUSED_ANNOTATION ("" when
+    absent) and forbids scaling up, leaving the HPA otherwise Healthy.
+    """
+    kdm = _make_kdm(mocker)
+    behavior = {"scaleDown": {"stabilizationWindowSeconds": 300}}
+    hpas = [
+        _make_hpa(mocker, "with-behavior", None, behavior),
+        _make_hpa(mocker, "without-behavior", None, None),
+    ]
+    mocker.patch.object(kdm, "get_deployment_hpa", return_value=iter(hpas))
+    kdm.client_autoscalingv2_api.api_client.sanitize_for_serialization.side_effect = lambda b: b
+    mocked_patch = mocker.patch.object(kdm, "patch_deployment_hpa")
+
+    kdm.pause_hpa(deployment_name="worker")
+
+    assert mocked_patch.mock_calls == [
+        unittest.mock.call(
+            hpa_name="with-behavior",
+            body={
+                "metadata": {"annotations": {PAUSED_ANNOTATION: json.dumps(behavior)}},
+                "spec": {"behavior": {"scaleUp": {"selectPolicy": "Disabled"}}},
+            },
+        ),
+        unittest.mock.call(
+            hpa_name="without-behavior",
+            body={
+                "metadata": {"annotations": {PAUSED_ANNOTATION: ""}},
+                "spec": {"behavior": {"scaleUp": {"selectPolicy": "Disabled"}}},
+            },
+        ),
+    ]
+
+
+def test_pause_hpa_is_idempotent(mocker: MockerFixture) -> None:
+    """
+    An already-paused HPA is left alone: never overwrite the saved original behavior with
+    the paused state.
+    """
+    kdm = _make_kdm(mocker)
+    paused = _make_hpa(mocker, "paused", {PAUSED_ANNOTATION: ""}, {"scaleUp": {"selectPolicy": "Disabled"}})
+    mocker.patch.object(kdm, "get_deployment_hpa", return_value=iter([paused]))
+    mocked_patch = mocker.patch.object(kdm, "patch_deployment_hpa")
+
+    kdm.pause_hpa(deployment_name="worker")
+
+    assert mocked_patch.mock_calls == []
+
+
+def test_resume_hpa_restores_original_behavior(mocker: MockerFixture) -> None:
+    """
+    Test that resume_hpa restores the exact pre-pause behavior (or clears the field when
+    there was none) and drops PAUSED_ANNOTATION; unannotated HPAs are untouched.
+    """
+    kdm = _make_kdm(mocker)
+    behavior = {"scaleDown": {"stabilizationWindowSeconds": 300}}
+    hpas = [
+        _make_hpa(
+            mocker, "had-behavior", {PAUSED_ANNOTATION: json.dumps(behavior)}, {"scaleUp": {"selectPolicy": "Disabled"}}
+        ),
+        _make_hpa(mocker, "had-none", {PAUSED_ANNOTATION: ""}, {"scaleUp": {"selectPolicy": "Disabled"}}),
+        _make_hpa(mocker, "never-paused", None, None),
+    ]
+    mocker.patch.object(kdm, "get_deployment_hpa", return_value=iter(hpas))
+    mocked_patch = mocker.patch.object(kdm, "patch_deployment_hpa")
+
+    kdm.resume_hpa(deployment_name="worker")
+
+    assert mocked_patch.mock_calls == [
+        unittest.mock.call(
+            hpa_name="had-behavior",
+            body={"metadata": {"annotations": {PAUSED_ANNOTATION: None}}, "spec": {"behavior": behavior}},
+        ),
+        unittest.mock.call(
+            hpa_name="had-none",
+            body={"metadata": {"annotations": {PAUSED_ANNOTATION: None}}, "spec": {"behavior": None}},
+        ),
+    ]