diff --git a/CHANGELOG.md b/CHANGELOG.md index 03d869e..355bca6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,18 @@ # Changelog +## v7.0.1 (2026-07-09) + +### Fix +- Pause HPAs during migrations via `spec.behavior.scaleUp.selectPolicy: Disabled` instead of + renaming their `scaleTargetRef` to a non-existent Deployment. The rename made the HPA emit + `FailedGetScale` and turn ArgoCD applications **Degraded** for the whole migration window, + and was reverted by any GitOps sync anyway. The pause keeps the HPA Healthy while forbidding + scale-up; the pre-pause `spec.behavior` is saved in the `wiremind.io/pre-pause-scale-behavior` + annotation and restored exactly on resume. Because a leftover pause is not cleaned up by + GitOps (the field is invisible to ArgoCD), `start_pods()` and `restore_stopped_pods()` now + resume HPAs unconditionally for every tracked Deployment. The old rename-repair path is kept + for HPAs left broken by a pre-7.0.1 run (removal planned for 8.0). + ## v7.0.0 (2026-07-08) ### BREAKING CHANGE diff --git a/VERSION b/VERSION index 66ce77b..9fe9ff9 100644 --- a/VERSION +++ b/VERSION @@ -1 +1 @@ -7.0.0 +7.0.1 diff --git a/src/chartreuse/kubernetes_helper.py b/src/chartreuse/kubernetes_helper.py index 558229e..1f636ad 100644 --- a/src/chartreuse/kubernetes_helper.py +++ b/src/chartreuse/kubernetes_helper.py @@ -9,6 +9,7 @@ """ import functools +import json import logging import os import pprint @@ -20,10 +21,16 @@ logger = logging.getLogger(__name__) +# Target-rename prefix used by the pre-7.0.1 HPA disable mechanism. Only the repair path +# (re_enable_hpa) remains, for HPAs left renamed by an older chartreuse. +# TODO: remove in 8.0 together with re_enable_hpa. HPA_ID_PREFIX = "wm--disabled--kube" # Set by stop_pods() on every Deployment it scales down, cleared by start_pods() and # restore_stopped_pods(). Allows telling "stopped by us" apart from "deliberately scaled to 0". STOPPED_ANNOTATION = "wiremind.io/stopped-by" +# Set by pause_hpa() on the HPA itself; stores the pre-pause spec.behavior as JSON ("" when +# absent) so resume_hpa() can restore it exactly. Presence of the annotation == "paused by us". +PAUSED_ANNOTATION = "wiremind.io/pre-pause-scale-behavior" def load_kubernetes_config(use_kubeconfig: bool | None = None) -> None: @@ -254,14 +261,51 @@ def _get_expected_deployment_scale_dict(self) -> dict[int, dict[str, int]]: return eds_dict @retry_kubernetes_request - def disable_hpa(self, *, deployment_name: str) -> None: + def pause_hpa(self, *, deployment_name: str) -> None: + """ + Forbid the Deployment's HPAs from scaling up (spec.behavior.scaleUp.selectPolicy: + Disabled) so they can't start pods mid-migration. Unlike breaking the scaleTargetRef, + the HPA stays Healthy: no FailedGetScale events, no Degraded ArgoCD application. + + The pre-pause spec.behavior is saved in PAUSED_ANNOTATION so resume_hpa() can restore + it exactly. Idempotent: an already-paused HPA is left alone (never overwrite the + saved original with a paused state). + """ for hpa in self.get_deployment_hpa(deployment_name=deployment_name): - # Tell the hpa to manage a non-existing Deployment - hpa.spec.scale_target_ref.name = f"{HPA_ID_PREFIX}-{deployment_name}" - self.patch_deployment_hpa(hpa_name=hpa.metadata.name, body=hpa) + if PAUSED_ANNOTATION in (hpa.metadata.annotations or {}): + continue + original_behavior = self.client_autoscalingv2_api.api_client.sanitize_for_serialization(hpa.spec.behavior) + saved = json.dumps(original_behavior) if original_behavior else "" + body = { + "metadata": {"annotations": {PAUSED_ANNOTATION: saved}}, + "spec": {"behavior": {"scaleUp": {"selectPolicy": "Disabled"}}}, + } + self.patch_deployment_hpa(hpa_name=hpa.metadata.name, body=body) + + @retry_kubernetes_request + def resume_hpa(self, *, deployment_name: str) -> None: + """ + Undo pause_hpa(): restore the exact pre-pause spec.behavior (or clear it when there + was none) and drop PAUSED_ANNOTATION. No-op on HPAs that don't carry the annotation, + so it is always safe to call. + """ + for hpa in self.get_deployment_hpa(deployment_name=deployment_name): + saved = (hpa.metadata.annotations or {}).get(PAUSED_ANNOTATION) + if saved is None: + continue + body = { + "metadata": {"annotations": {PAUSED_ANNOTATION: None}}, + "spec": {"behavior": json.loads(saved) if saved else None}, + } + self.patch_deployment_hpa(hpa_name=hpa.metadata.name, body=body) @retry_kubernetes_request def re_enable_hpa(self, *, deployment_name: str) -> None: + """ + Repair an HPA left pointing at a wm--disabled--kube- target by a pre-7.0.1 chartreuse + whose run crashed before re-enabling. 7.0.1+ never creates such renames. + TODO: remove in 8.0. + """ for hpa in self.get_deployment_hpa(deployment_name=f"{HPA_ID_PREFIX}-{deployment_name}"): hpa.spec.scale_target_ref.name = deployment_name self.patch_deployment_hpa(hpa_name=hpa.metadata.name, body=hpa) @@ -278,9 +322,9 @@ def _stop_deployments(self, deployment_dict: dict[str, int]) -> None: """ for deployment_name in deployment_dict: self.annotate_deployment(deployment_name, {STOPPED_ANNOTATION: self.release_name}) + self.pause_hpa(deployment_name=deployment_name) for _ in range(self.SCALE_DOWN_MAX_WAIT_TIME): for deployment_name in deployment_dict: - self.disable_hpa(deployment_name=deployment_name) self.scale_down_deployment(deployment_name) if self._are_deployments_stopped(deployment_dict): break @@ -324,6 +368,7 @@ def start_pods(self) -> None: if len(priority_dict): scaled = True for name, expected_scale in priority_dict.items(): + self.resume_hpa(deployment_name=name) self.re_enable_hpa(deployment_name=name) self.scale_up_deployment(name, expected_scale) self.annotate_deployment(name, {STOPPED_ANNOTATION: None}) @@ -336,8 +381,11 @@ def restore_stopped_pods(self) -> None: """ Restore Deployments that stop_pods() scaled down and that nothing scaled back up since. - Only Deployments still carrying STOPPED_ANNOTATION are touched, so a Deployment - deliberately scaled to zero outside of stop_pods() is left alone. + Only the replica restore is gated on STOPPED_ANNOTATION, so a Deployment deliberately + scaled to zero outside of stop_pods() is left alone. HPA resume/repair on the other + hand runs for EVERY tracked Deployment: a pause left behind by a crashed run is not + self-healing (ArgoCD ignores the live-only spec.behavior field), and resuming an + unpaused HPA is a no-op. Meant to run after the deployment that follows a stop_pods() without start_pods() (CHARTREUSE_UPGRADE_BEFORE_DEPLOYMENT): that deployment restores spec.replicas of @@ -351,6 +399,8 @@ def restore_stopped_pods(self) -> None: restored: list[str] = [] for priority in sorted(expected_deployment_scale_dict): for name, expected_scale in expected_deployment_scale_dict[priority].items(): + self.resume_hpa(deployment_name=name) + self.re_enable_hpa(deployment_name=name) try: deployment = self.client_appsv1_api.read_namespaced_deployment(name, self.namespace) except kubernetes.client.rest.ApiException as e: @@ -359,7 +409,6 @@ def restore_stopped_pods(self) -> None: raise if STOPPED_ANNOTATION not in (deployment.metadata.annotations or {}): continue - self.re_enable_hpa(deployment_name=name) if not deployment.spec.replicas: target_scale = expected_scale if target_scale < 1 and any(self.get_deployment_hpa(deployment_name=name)): diff --git a/src/chartreuse/tests/unit_tests/test_kubernetes_helper.py b/src/chartreuse/tests/unit_tests/test_kubernetes_helper.py index 053eca2..f931525 100644 --- a/src/chartreuse/tests/unit_tests/test_kubernetes_helper.py +++ b/src/chartreuse/tests/unit_tests/test_kubernetes_helper.py @@ -1,9 +1,10 @@ +import json import unittest import kubernetes from pytest_mock import MockerFixture -from chartreuse.kubernetes_helper import STOPPED_ANNOTATION, KubernetesDeploymentManager +from chartreuse.kubernetes_helper import PAUSED_ANNOTATION, STOPPED_ANNOTATION, KubernetesDeploymentManager def _make_kdm(mocker: MockerFixture) -> KubernetesDeploymentManager: @@ -59,7 +60,7 @@ def test_stop_deployments_correctly_wait(mocker: MockerFixture) -> None: """ kdm = _make_kdm(mocker) deployment_dict = {"my-pod": 42, "my-other-pod": 113} - mocker.patch.object(kdm, "disable_hpa") + mocker.patch.object(kdm, "pause_hpa") mocker.patch.object(kdm, "scale_down_deployment") mocker.patch.object(kdm, "annotate_deployment") mocked_are_deployments_stopped = mocker.patch.object( @@ -82,7 +83,7 @@ def test_stop_deployments_annotates_stopped_deployments(mocker: MockerFixture) - """ kdm = _make_kdm(mocker) mocker.patch.object(kdm, "_are_deployments_stopped", return_value=True) - mocker.patch.object(kdm, "disable_hpa") + mocker.patch.object(kdm, "pause_hpa") mocker.patch.object(kdm, "scale_down_deployment") mocked_annotate = mocker.patch.object(kdm, "annotate_deployment") @@ -97,13 +98,17 @@ def test_stop_deployments_annotates_stopped_deployments(mocker: MockerFixture) - def test_start_pods_clears_stopped_annotation(mocker: MockerFixture) -> None: kdm = _make_kdm(mocker) mocker.patch.object(kdm, "_get_expected_deployment_scale_dict", return_value={0: {"worker": 2}}) - mocker.patch.object(kdm, "re_enable_hpa") + mocked_resume = mocker.patch.object(kdm, "resume_hpa") + mocked_re_enable = mocker.patch.object(kdm, "re_enable_hpa") mocker.patch.object(kdm, "scale_up_deployment") mocked_annotate = mocker.patch.object(kdm, "annotate_deployment") kdm.start_pods() assert mocked_annotate.mock_calls == [unittest.mock.call("worker", {STOPPED_ANNOTATION: None})] + assert mocked_resume.mock_calls == [unittest.mock.call(deployment_name="worker")] + # Rename-repair transition shim for HPAs left broken by a pre-7.0.1 chartreuse + assert mocked_re_enable.mock_calls == [unittest.mock.call(deployment_name="worker")] def test_restore_stopped_pods(mocker: MockerFixture) -> None: @@ -136,16 +141,23 @@ def read_deployment(name: str, namespace: str) -> unittest.mock.MagicMock: return deployments[name] kdm.client_appsv1_api.read_namespaced_deployment.side_effect = read_deployment + mocked_resume_hpa = mocker.patch.object(kdm, "resume_hpa") mocked_re_enable_hpa = mocker.patch.object(kdm, "re_enable_hpa") mocked_scale_up = mocker.patch.object(kdm, "scale_up_deployment") mocked_annotate = mocker.patch.object(kdm, "annotate_deployment") kdm.restore_stopped_pods() - assert mocked_re_enable_hpa.mock_calls == [ + # HPA resume (and rename-repair) is unconditional: a pause left behind by a crashed run + # must always be cleaned up, even on Deployments that won't be scaled (or are gone). + expected_hpa_calls = [ unittest.mock.call(deployment_name="stopped-worker"), + unittest.mock.call(deployment_name="untouched-worker"), unittest.mock.call(deployment_name="already-scaled"), + unittest.mock.call(deployment_name="gone"), ] + assert mocked_resume_hpa.mock_calls == expected_hpa_calls + assert mocked_re_enable_hpa.mock_calls == expected_hpa_calls assert mocked_scale_up.mock_calls == [unittest.mock.call("stopped-worker", 2)] assert mocked_annotate.mock_calls == [ unittest.mock.call("stopped-worker", {STOPPED_ANNOTATION: None}), @@ -163,6 +175,7 @@ def test_restore_stopped_pods_rearms_hpa_when_expected_scale_is_zero(mocker: Moc kdm.client_appsv1_api.read_namespaced_deployment.return_value = _make_deployment( mocker, {STOPPED_ANNOTATION: "concerned"}, 0 ) + mocker.patch.object(kdm, "resume_hpa") mocker.patch.object(kdm, "re_enable_hpa") mocker.patch.object(kdm, "get_deployment_hpa", return_value=iter([mocker.MagicMock()])) mocked_scale_up = mocker.patch.object(kdm, "scale_up_deployment") @@ -171,3 +184,94 @@ def test_restore_stopped_pods_rearms_hpa_when_expected_scale_is_zero(mocker: Moc kdm.restore_stopped_pods() assert mocked_scale_up.mock_calls == [unittest.mock.call("worker", 1)] + + +def _make_hpa( + mocker: MockerFixture, name: str, annotations: dict | None, behavior: dict | None +) -> unittest.mock.MagicMock: + hpa = mocker.MagicMock() + hpa.metadata.name = name + hpa.metadata.annotations = annotations + hpa.spec.behavior = behavior + return hpa + + +def test_pause_hpa_saves_behavior_and_disables_scale_up(mocker: MockerFixture) -> None: + """ + Test that pause_hpa stores the pre-pause spec.behavior in PAUSED_ANNOTATION ("" when + absent) and forbids scaling up, leaving the HPA otherwise Healthy. + """ + kdm = _make_kdm(mocker) + behavior = {"scaleDown": {"stabilizationWindowSeconds": 300}} + hpas = [ + _make_hpa(mocker, "with-behavior", None, behavior), + _make_hpa(mocker, "without-behavior", None, None), + ] + mocker.patch.object(kdm, "get_deployment_hpa", return_value=iter(hpas)) + kdm.client_autoscalingv2_api.api_client.sanitize_for_serialization.side_effect = lambda b: b + mocked_patch = mocker.patch.object(kdm, "patch_deployment_hpa") + + kdm.pause_hpa(deployment_name="worker") + + assert mocked_patch.mock_calls == [ + unittest.mock.call( + hpa_name="with-behavior", + body={ + "metadata": {"annotations": {PAUSED_ANNOTATION: json.dumps(behavior)}}, + "spec": {"behavior": {"scaleUp": {"selectPolicy": "Disabled"}}}, + }, + ), + unittest.mock.call( + hpa_name="without-behavior", + body={ + "metadata": {"annotations": {PAUSED_ANNOTATION: ""}}, + "spec": {"behavior": {"scaleUp": {"selectPolicy": "Disabled"}}}, + }, + ), + ] + + +def test_pause_hpa_is_idempotent(mocker: MockerFixture) -> None: + """ + An already-paused HPA is left alone: never overwrite the saved original behavior with + the paused state. + """ + kdm = _make_kdm(mocker) + paused = _make_hpa(mocker, "paused", {PAUSED_ANNOTATION: ""}, {"scaleUp": {"selectPolicy": "Disabled"}}) + mocker.patch.object(kdm, "get_deployment_hpa", return_value=iter([paused])) + mocked_patch = mocker.patch.object(kdm, "patch_deployment_hpa") + + kdm.pause_hpa(deployment_name="worker") + + assert mocked_patch.mock_calls == [] + + +def test_resume_hpa_restores_original_behavior(mocker: MockerFixture) -> None: + """ + Test that resume_hpa restores the exact pre-pause behavior (or clears the field when + there was none) and drops PAUSED_ANNOTATION; unannotated HPAs are untouched. + """ + kdm = _make_kdm(mocker) + behavior = {"scaleDown": {"stabilizationWindowSeconds": 300}} + hpas = [ + _make_hpa( + mocker, "had-behavior", {PAUSED_ANNOTATION: json.dumps(behavior)}, {"scaleUp": {"selectPolicy": "Disabled"}} + ), + _make_hpa(mocker, "had-none", {PAUSED_ANNOTATION: ""}, {"scaleUp": {"selectPolicy": "Disabled"}}), + _make_hpa(mocker, "never-paused", None, None), + ] + mocker.patch.object(kdm, "get_deployment_hpa", return_value=iter(hpas)) + mocked_patch = mocker.patch.object(kdm, "patch_deployment_hpa") + + kdm.resume_hpa(deployment_name="worker") + + assert mocked_patch.mock_calls == [ + unittest.mock.call( + hpa_name="had-behavior", + body={"metadata": {"annotations": {PAUSED_ANNOTATION: None}}, "spec": {"behavior": behavior}}, + ), + unittest.mock.call( + hpa_name="had-none", + body={"metadata": {"annotations": {PAUSED_ANNOTATION: None}}, "spec": {"behavior": None}}, + ), + ]