Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension


Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
7 changes: 7 additions & 0 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -250,6 +250,13 @@ This feature allows you to pause rollouts for a deployment for a specified durat
1. ✅ Your deployment references multiple ConfigMaps or Secrets that may be updated at the same time.
1. ✅ You want to minimize unnecessary rollouts and reduce downtime caused by back-to-back configuration changes.

### Optional restart windows

Defer Secret and ConfigMap rollouts to per-workload maintenance windows with
`--enable-restart-windows` and `reloader.stakater.com/restart-window`.
See [restart windows](docs/restart-windows.md) for policy examples, supported
workloads, durable recovery, and GitOps ownership requirements. Disabled by default.

### 8. 🔐 CSI Secret Provider Support

Reloader supports the [Secrets Store CSI Driver](https://secrets-store-csi-driver.sigs.k8s.io/), which allows mounting secrets from external secret stores (like AWS Secrets Manager, Azure Key Vault, HashiCorp Vault) directly into pods.
Expand Down
8 changes: 8 additions & 0 deletions deployments/kubernetes/chart/reloader/README.md
Original file line number Diff line number Diff line change
Expand Up @@ -203,3 +203,11 @@ Helm chart will be released to the chart registry whenever files in `deployments
1. Create a new branch and update the Helm chart `appVersion` and `version`, example pull-request: [PR-846](https://github.com/stakater/Reloader/pull/846)
1. Label the PR with `release/helm-chart`
1. After approval and just before squash, make sure the squash commit message represents all changes, because it will be used to autogenerate the changelog message

### Optional restart windows

Set `reloader.enableRestartWindows: true` to enable per-workload restart windows
(default: `false`). This passes `--enable-restart-windows` and adds workload
watch permissions to the generated RBAC. Workloads must also opt in using
`reloader.stakater.com/restart-window` on their metadata alongside the normal
Reloader source annotations. See [restart window documentation](../../../../docs/restart-windows.md).
3 changes: 3 additions & 0 deletions deployments/kubernetes/chart/reloader/templates/_helpers.tpl
Original file line number Diff line number Diff line change
Expand Up @@ -169,6 +169,9 @@ the rule set is defined once. Expects the root context ($) as its argument.
- daemonsets
- statefulsets
verbs:
{{- if .Values.reloader.enableRestartWindows }}
- watch
{{- end }}
- list
- get
- update
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,9 @@ rules:
- daemonsets
- statefulsets
verbs:
{{- if .Values.reloader.enableRestartWindows }}
- watch
{{- end }}
- list
- get
- update
Expand Down
Original file line number Diff line number Diff line change
Expand Up @@ -219,8 +219,11 @@ spec:
{{- . | toYaml | nindent 10 }}
{{- end }}
{{- end }}
{{- if or (.Values.reloader.logFormat) (.Values.reloader.logLevel) (.Values.reloader.ignoreSecrets) (and .Values.reloader.ignoreNamespaces .Values.reloader.watchGlobally) (.Values.reloader.namespaces) (include "reloader-namespaceSelector" .) (.Values.reloader.resourceLabelSelector) (.Values.reloader.ignoreConfigMaps) (.Values.reloader.custom_annotations) (eq .Values.reloader.isArgoRollouts true) (eq .Values.reloader.reloadOnCreate true) (eq .Values.reloader.reloadOnDelete true) (ne .Values.reloader.reloadStrategy "default") (.Values.reloader.enableHA) (.Values.reloader.autoReloadAll) (.Values.reloader.ignoreJobs) (.Values.reloader.ignoreCronJobs) (.Values.reloader.enableCSIIntegration)}}
{{- if or (.Values.reloader.enableRestartWindows) (.Values.reloader.logFormat) (.Values.reloader.logLevel) (.Values.reloader.ignoreSecrets) (and .Values.reloader.ignoreNamespaces .Values.reloader.watchGlobally) (.Values.reloader.namespaces) (include "reloader-namespaceSelector" .) (.Values.reloader.resourceLabelSelector) (.Values.reloader.ignoreConfigMaps) (.Values.reloader.custom_annotations) (eq .Values.reloader.isArgoRollouts true) (eq .Values.reloader.reloadOnCreate true) (eq .Values.reloader.reloadOnDelete true) (ne .Values.reloader.reloadStrategy "default") (.Values.reloader.enableHA) (.Values.reloader.autoReloadAll) (.Values.reloader.ignoreJobs) (.Values.reloader.ignoreCronJobs) (.Values.reloader.enableCSIIntegration)}}
args:
{{- if .Values.reloader.enableRestartWindows }}
- "--enable-restart-windows"
{{- end }}
{{- if .Values.reloader.logFormat }}
- "--log-format={{ .Values.reloader.logFormat }}"
{{- end }}
Expand Down
1 change: 1 addition & 0 deletions deployments/kubernetes/chart/reloader/values.schema.json
Original file line number Diff line number Diff line change
Expand Up @@ -5,6 +5,7 @@
"reloader": {
"type": "object",
"properties": {
"enableRestartWindows": {"type": "boolean", "default": false},
"reloadStrategy": {
"type": "string",
"enum": [
Expand Down
2 changes: 2 additions & 0 deletions deployments/kubernetes/chart/reloader/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ image:
pullPolicy: IfNotPresent

reloader:
# Optional restart windows; only annotated workloads are scheduled.
enableRestartWindows: false
autoReloadAll: false
isArgoRollouts: false
isOpenshift: false
Expand Down
132 changes: 132 additions & 0 deletions docs/restart-windows.md
Original file line number Diff line number Diff line change
@@ -0,0 +1,132 @@
# Optional restart windows

When a Secret or ConfigMap changes outside a maintenance window, defer its
workload rollout until the next allowed interval. Pending changes survive
controller restarts and are coalesced using current source data.

Enable `--enable-restart-windows` or Helm value
`reloader.enableRestartWindows: true` (default: false), then add a policy to
workload metadata alongside its normal Reloader opt-in:

```yaml
metadata:
annotations:
secret.reloader.stakater.com/reload: "application-tls"
reloader.stakater.com/restart-window: >-
{"timezone":"UTC","windows":[{"cron":"0 0 * * *","duration":"1h"}]}
```

This allows rollout starts daily from 00:00 inclusive to 01:00 exclusive UTC.
Place the policy on workload metadata, not the pod template. Policies require
an explicit IANA timezone and 1–16 five-field cron windows, with durations from
1m to 24h. Day-of-month/day-of-week matching follows cron OR semantics; elapsed
duration is preserved across DST changes. Timezone data is embedded in the binary.

First-time enrollment with `--reload-on-create` and `--sync-after-restart`
queues existing sources even if the workload was manually restarted earlier.
Applied hashes track only this scheduler's successful patches. Recovery of
already-persisted pending work does not require either startup flag.

## Behavior and failure handling

- Without a window annotation, upstream restart behavior is unchanged.
- The annotation without the enabling flag skips only that workload and emits a
warning; other workloads that use the same source continue to be processed.
Pending state by itself does not block the normal path when the feature is
disabled. Unsupported workload/source kinds are isolated the same way.
- Secret and ConfigMap events persist source names and observed hashes on the
workload. Secret contents, private keys and passwords are never persisted in
annotations or logs by this extension.
- Recovery covers requests already persisted before an outage. Secret events
lost before persistence retain upstream Reloader startup-sync semantics.
- A workload informer recovers pending work on startup/leader election. Closed
windows are evaluated from the informer cache and scheduled directly for the
next opening; a live workload read is made immediately before mutation. Policy
edits requeue through informer updates. No new Secret event is needed. No
CronJob or new CRD is required.
- At execution, source content and opt-in/ignore/filter settings are read again.
Multiple pending sources produce one pod-template annotation change, using
their current hashes. Replayed events do not restart an already applied hash.
- Applied hashes and pending removal are committed atomically with the template
patch. resourceVersion and UID protect concurrent updates and recreated targets.
API errors retain the pending request and are retried.
- Invalid policies, missing sources, and paused Deployments hold pending work.
Missing Secrets are never a reason to restart. No automatic expiry override
exists: operational alerting must cover certificates and blocked rotations.
- `WindowReloaded` and deduplicated `RestartWindowBlocked` Kubernetes Events
expose outcomes; errors are logged with rate-limited retries. Successful and
failed rollout attempts update the normal reload/action metrics, and successful
rollouts use the normal `ALERT_ON_RELOAD` webhook. The
`reloader_skipped_total{reason="restart_window_queued"}` metric records deferred
event handling.
- Removing the window annotation releases pending work immediately while the
scheduler is enabled. To cancel a request, remove its Reloader trigger or
explicitly remove its pending annotation. Disabling the scheduler retains
pending state for the next time it is enabled.
- The window governs **starting** a rollout, not finishing it. Kubernetes may
complete replacement after the window closes. This extension does not verify
application health, TLS expiry, served certificates, or query completion.
- Windowed workloads use a deterministic pod-template hash annotation and update
the selected upstream reload-strategy marker in the same patch. This prevents
a later startup reconciliation from treating an already-applied source hash as
new. Windowed workloads do not use the upstream pause-period cooldown.
Manually paused Deployments remain paused and are not modified.
- Supported: apps/v1 Deployments, unpartitioned RollingUpdate StatefulSets, and
RollingUpdate DaemonSets. Jobs, CronJobs, Argo Rollouts, DeploymentConfigs,
CSI sources, and directly operator-managed pods are not supported.
- At most 128 pending/applied source references per workload. Prune obsolete
applied entries when permanently removing sources. Each new Secret content
update occurring after a rollout may produce another rollout in the same
open window; this is coalescing of queued work, not a global debounce policy.

For strict windows, run Reloader as the sole Reloader for these targets. Other
controllers, GitOps changes, manual restarts, and native application certificate
reloads are outside its scheduling scope. Webhook-only mode does not use the
workload restart scheduler.

## GitOps ownership

Keep policy annotations in Git. The following paths are controller-owned and
must not be reverted by Argo CD self-heal:

```yaml
ignoreDifferences:
- group: apps
kind: Deployment
jsonPointers:
- /metadata/annotations/reloader.stakater.com~1pending
- /metadata/annotations/reloader.stakater.com~1applied
- /spec/template/metadata/annotations/reloader.stakater.com~1restart-hash
syncPolicy:
syncOptions:
- RespectIgnoreDifferences=true
```

Scope the rule to opted-in resource names/namespaces, and add analogous rules
for StatefulSet/DaemonSet only if used. Do not ignore entire annotation maps.
The bundled chart conditionally adds workload `watch` permission when enabling
windows, in both cluster-wide and namespace-scoped RBAC modes.

## Build and validation

```sh
CGO_ENABLED=0 go build -o /tmp/reloader ./
go test -race ./internal/pkg/restartwindow
helm lint deployments/kubernetes/chart/reloader --set reloader.enableRestartWindows=true
```

For upstream handler/controller regression tests, use a temporary **loopback-only**
kubeconfig (`server: http://127.0.0.1:9`) because upstream package initialization
probes for OpenShift/CSI APIs. Never point test runs at a production kubeconfig.
The regression suites use fake Kubernetes clients.

```sh
KUBECONFIG=/path/to/loopback-test-kubeconfig.yaml go test -race \
./internal/pkg/restartwindow ./internal/pkg/controller ./internal/pkg/handler ./pkg/common
```

Before production release, exercise a short real window in an isolated cluster:
update two Secrets outside the window, restart the controller, verify no rollout
until opening and exactly one combined template change, then check policy edits,
missing Secrets, leader failover and GitOps reconciliation. Live cluster and
container-image deployment validation are not implied by local tests.
1 change: 1 addition & 0 deletions go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -3,6 +3,7 @@ module github.com/stakater/Reloader
go 1.26.8

require (
github.com/robfig/cron/v3 v3.0.1
github.com/argoproj/argo-rollouts v1.9.0
github.com/onsi/ginkgo/v2 v2.27.4
github.com/onsi/gomega v1.39.0
Expand Down
2 changes: 2 additions & 0 deletions go.sum
Original file line number Diff line number Diff line change
Expand Up @@ -448,6 +448,8 @@ github.com/raeperd/recvcheck v0.2.0/go.mod h1:n04eYkwIR0JbgD73wT8wL4JjPC3wm0nFtz
github.com/rivo/uniseg v0.2.0/go.mod h1:J6wj4VEh+S6ZtnVlnTBMWIodfgj8LQOQFoIToxlJtxc=
github.com/rivo/uniseg v0.4.7 h1:WUdvkW8uEhrYfLC4ZzdpI2ztxP1I582+49Oc5Mq64VQ=
github.com/rivo/uniseg v0.4.7/go.mod h1:FN3SvrM+Zdj16jyLfmOkMNblXMcoc8DfTHruCPUcx88=
github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs=
github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro=
github.com/rogpeppe/go-internal v1.14.1 h1:UQB4HGPB6osV0SQTLymcB4TgvyWu6ZyliaW0tI/otEQ=
github.com/rogpeppe/go-internal v1.14.1/go.mod h1:MaRKkUm5W0goXpeCfT7UZI6fk/L7L7so1lCWt35ZSgc=
github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM=
Expand Down
14 changes: 13 additions & 1 deletion internal/pkg/cmd/reloader.go
Original file line number Diff line number Diff line change
Expand Up @@ -174,7 +174,7 @@ func namespaceWatchScopeMessage(ignoredNamespaces []string) string {
}

func startReloader(cmd *cobra.Command, args []string) {
common.GetCommandLineOptions()
common.RefreshCommandLineOptions()
err := configureLogging(options.LogFormat, options.LogLevel)
if err != nil {
logrus.Warn(err)
Expand Down Expand Up @@ -263,6 +263,18 @@ func startReloader(cmd *cobra.Command, args []string) {
}
}

if options.EnableRestartWindows {
for _, currentNamespace := range watchNamespaces {
c := controller.NewRestartWindowController(clientset, currentNamespace, ignoredNamespacesList, namespaceLabelSelector, resourceLabelSelector, nil, collectors)
controllers = append(controllers, c)
if !options.EnableHA {
stop := make(chan struct{})
defer close(stop)
go c.Run(1, stop)
}
}
}

// Run leadership election
if options.EnableHA {
podName, podNamespace := getHAEnvs()
Expand Down
5 changes: 5 additions & 0 deletions internal/pkg/controller/controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -30,6 +30,7 @@ import (

// Controller for checking events
type Controller struct {
windowRunner func(chan struct{})
client kubernetes.Interface
queue workqueue.TypedRateLimitingInterface[any]
informer cache.Controller
Expand Down Expand Up @@ -291,6 +292,10 @@ func (c *Controller) enqueue(item interface{}) {

// Run function for controller which handles the queue
func (c *Controller) Run(threadiness int, stopCh chan struct{}) {
if c.windowRunner != nil {
c.windowRunner(stopCh)
return
}
defer runtime.HandleCrash()

var wg sync.WaitGroup
Expand Down
Loading
Loading