diff --git a/resources/mixins/kubernetes/generated/alerts.yml b/resources/mixins/kubernetes/generated/alerts.yml index f9406d5..7a7a888 100644 --- a/resources/mixins/kubernetes/generated/alerts.yml +++ b/resources/mixins/kubernetes/generated/alerts.yml @@ -7,7 +7,7 @@ "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodcrashlooping" "summary": "Pod is crash looping." "expr": | - max_over_time(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff", namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"}[5m]) >= 1 + max_over_time(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff", namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"}[5m]) >= 1 "for": "15m" "labels": "severity": "warning" @@ -20,7 +20,7 @@ "expr": | sum by (namespace, pod, cluster) ( max by(namespace, pod, cluster) ( - kube_pod_status_phase{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics", phase=~"Pending|Unknown|Failed"} + kube_pod_status_phase{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics", phase=~"Pending|Unknown|Failed"} ) * on(namespace, pod, cluster) group_left(owner_kind) topk by(namespace, pod, cluster) ( 1, max by(namespace, pod, owner_kind, cluster) (kube_pod_owner{owner_kind!="Job"}) ) @@ -35,9 +35,9 @@ "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentgenerationmismatch" "summary": "Deployment generation mismatch due to possible roll-back" "expr": | - kube_deployment_status_observed_generation{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_deployment_status_observed_generation{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} != - kube_deployment_metadata_generation{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_deployment_metadata_generation{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} "for": "15m" "labels": "severity": "warning" @@ -49,11 +49,11 @@ "summary": "Deployment has not matched the expected number of replicas." "expr": | ( - kube_deployment_spec_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_deployment_spec_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} > - kube_deployment_status_replicas_available{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_deployment_status_replicas_available{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} ) and ( - changes(kube_deployment_status_replicas_updated{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"}[10m]) + changes(kube_deployment_status_replicas_updated{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"}[10m]) == 0 ) @@ -67,7 +67,7 @@ "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentrolloutstuck" "summary": "Deployment rollout is not progressing." "expr": | - kube_deployment_status_condition{condition="Progressing", status="false",namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_deployment_status_condition{condition="Progressing", status="false",namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} != 0 "for": "15m" "labels": @@ -80,11 +80,11 @@ "summary": "StatefulSet has not matched the expected number of replicas." "expr": | ( - kube_statefulset_status_replicas_ready{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_statefulset_status_replicas_ready{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} != - kube_statefulset_status_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_statefulset_status_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} ) and ( - changes(kube_statefulset_status_replicas_updated{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"}[10m]) + changes(kube_statefulset_status_replicas_updated{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"}[10m]) == 0 ) @@ -98,9 +98,9 @@ "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetgenerationmismatch" "summary": "StatefulSet generation mismatch due to possible roll-back" "expr": | - kube_statefulset_status_observed_generation{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_statefulset_status_observed_generation{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} != - kube_statefulset_metadata_generation{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_statefulset_metadata_generation{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} "for": "15m" "labels": "severity": "warning" @@ -113,18 +113,18 @@ "expr": | ( max without (revision) ( - kube_statefulset_status_current_revision{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_statefulset_status_current_revision{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} unless - kube_statefulset_status_update_revision{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_statefulset_status_update_revision{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} ) * ( - kube_statefulset_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_statefulset_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} != - kube_statefulset_status_replicas_updated{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_statefulset_status_replicas_updated{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} ) ) and ( - changes(kube_statefulset_status_replicas_updated{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"}[5m]) + changes(kube_statefulset_status_replicas_updated{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"}[5m]) == 0 ) @@ -140,24 +140,24 @@ "expr": | ( ( - kube_daemonset_status_current_number_scheduled{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_daemonset_status_current_number_scheduled{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} != - kube_daemonset_status_desired_number_scheduled{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_daemonset_status_desired_number_scheduled{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} ) or ( - kube_daemonset_status_number_misscheduled{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_daemonset_status_number_misscheduled{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} != 0 ) or ( - kube_daemonset_status_updated_number_scheduled{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_daemonset_status_updated_number_scheduled{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} != - kube_daemonset_status_desired_number_scheduled{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_daemonset_status_desired_number_scheduled{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} ) or ( - kube_daemonset_status_number_available{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_daemonset_status_number_available{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} != - kube_daemonset_status_desired_number_scheduled{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_daemonset_status_desired_number_scheduled{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} ) ) and ( - changes(kube_daemonset_status_updated_number_scheduled{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"}[5m]) + changes(kube_daemonset_status_updated_number_scheduled{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"}[5m]) == 0 ) @@ -171,7 +171,7 @@ "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecontainerwaiting" "summary": "Pod container waiting longer than 1 hour" "expr": | - sum by (namespace, pod, container, cluster) (kube_pod_container_status_waiting_reason{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"}) > 0 + sum by (namespace, pod, container, cluster) (kube_pod_container_status_waiting_reason{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"}) > 0 "for": "1h" "labels": "severity": "warning" @@ -182,9 +182,9 @@ "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetnotscheduled" "summary": "DaemonSet pods are not scheduled." "expr": | - kube_daemonset_status_desired_number_scheduled{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_daemonset_status_desired_number_scheduled{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} - - kube_daemonset_status_current_number_scheduled{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} > 0 + kube_daemonset_status_current_number_scheduled{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} > 0 "for": "10m" "labels": "severity": "warning" @@ -195,7 +195,7 @@ "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetmisscheduled" "summary": "DaemonSet pods are misscheduled." "expr": | - kube_daemonset_status_number_misscheduled{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} > 0 + kube_daemonset_status_number_misscheduled{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} > 0 "for": "15m" "labels": "severity": "warning" @@ -206,9 +206,9 @@ "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobnotcompleted" "summary": "Job did not complete in time" "expr": | - time() - max by(namespace, job_name, cluster) (kube_job_status_start_time{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + time() - max by(namespace, job_name, cluster) (kube_job_status_start_time{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} and - kube_job_status_active{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} > 0) > 43200 + kube_job_status_active{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} > 0) > 43200 "labels": "severity": "warning" "source": "mixin/kubernetes" @@ -218,7 +218,7 @@ "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobfailed" "summary": "Job failed to complete." "expr": | - kube_job_failed{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} > 0 + kube_job_failed{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} > 0 "for": "15m" "labels": "severity": "warning" @@ -229,19 +229,19 @@ "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubehpareplicasmismatch" "summary": "HPA has not matched desired number of replicas." "expr": | - (kube_horizontalpodautoscaler_status_desired_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + (kube_horizontalpodautoscaler_status_desired_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} != - kube_horizontalpodautoscaler_status_current_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"}) + kube_horizontalpodautoscaler_status_current_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"}) and - (kube_horizontalpodautoscaler_status_current_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + (kube_horizontalpodautoscaler_status_current_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} > - kube_horizontalpodautoscaler_spec_min_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"}) + kube_horizontalpodautoscaler_spec_min_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"}) and - (kube_horizontalpodautoscaler_status_current_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + (kube_horizontalpodautoscaler_status_current_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} < - kube_horizontalpodautoscaler_spec_max_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"}) + kube_horizontalpodautoscaler_spec_max_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"}) and - changes(kube_horizontalpodautoscaler_status_current_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"}[15m]) == 0 + changes(kube_horizontalpodautoscaler_status_current_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"}[15m]) == 0 "for": "15m" "labels": "severity": "warning" @@ -252,9 +252,9 @@ "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubehpamaxedout" "summary": "HPA is running at max replicas" "expr": | - kube_horizontalpodautoscaler_status_current_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_horizontalpodautoscaler_status_current_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} == - kube_horizontalpodautoscaler_spec_max_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_horizontalpodautoscaler_spec_max_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} "for": "15m" "labels": "severity": "info" @@ -293,7 +293,7 @@ "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuquotaovercommit" "summary": "Cluster has overcommitted CPU resource requests." "expr": | - sum(min without(resource) (kube_resourcequota{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics", type="hard", resource=~"(cpu|requests.cpu)"})) + sum(min without(resource) (kube_resourcequota{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics", type="hard", resource=~"(cpu|requests.cpu)"})) / sum(kube_node_status_allocatable{resource="cpu", job="kube-state-metrics"}) > 1.5 @@ -307,7 +307,7 @@ "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememoryquotaovercommit" "summary": "Cluster has overcommitted memory resource requests." "expr": | - sum(min without(resource) (kube_resourcequota{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics", type="hard", resource=~"(memory|requests.memory)"})) + sum(min without(resource) (kube_resourcequota{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics", type="hard", resource=~"(memory|requests.memory)"})) / sum(kube_node_status_allocatable{resource="memory", job="kube-state-metrics"}) > 1.5 @@ -321,9 +321,9 @@ "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubequotaalmostfull" "summary": "Namespace quota is going to be full." "expr": | - kube_resourcequota{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics", type="used"} + kube_resourcequota{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics", type="used"} / ignoring(instance, job, type) - (kube_resourcequota{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics", type="hard"} > 0) + (kube_resourcequota{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics", type="hard"} > 0) > 0.9 < 1 "for": "15m" "labels": @@ -335,9 +335,9 @@ "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubequotafullyused" "summary": "Namespace quota is fully used." "expr": | - kube_resourcequota{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics", type="used"} + kube_resourcequota{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics", type="used"} / ignoring(instance, job, type) - (kube_resourcequota{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics", type="hard"} > 0) + (kube_resourcequota{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics", type="hard"} > 0) == 1 "for": "15m" "labels": @@ -349,9 +349,9 @@ "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubequotaexceeded" "summary": "Namespace quota has exceeded the limits." "expr": | - kube_resourcequota{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics", type="used"} + kube_resourcequota{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics", type="used"} / ignoring(instance, job, type) - (kube_resourcequota{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics", type="hard"} > 0) + (kube_resourcequota{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics", type="hard"} > 0) > 1 "for": "15m" "labels": @@ -381,16 +381,16 @@ "summary": "PersistentVolume is filling up." "expr": | ( - kubelet_volume_stats_available_bytes{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} + kubelet_volume_stats_available_bytes{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} / - kubelet_volume_stats_capacity_bytes{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} + kubelet_volume_stats_capacity_bytes{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} ) < 0.03 and - kubelet_volume_stats_used_bytes{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} > 0 + kubelet_volume_stats_used_bytes{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} > 0 unless on(cluster, namespace, persistentvolumeclaim) - kube_persistentvolumeclaim_access_mode{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*", access_mode="ReadOnlyMany"} == 1 + kube_persistentvolumeclaim_access_mode{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*", access_mode="ReadOnlyMany"} == 1 unless on(cluster, namespace, persistentvolumeclaim) - kube_persistentvolumeclaim_labels{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",label_excluded_from_alerts="true"} == 1 + kube_persistentvolumeclaim_labels{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",label_excluded_from_alerts="true"} == 1 "for": "1m" "labels": "severity": "critical" @@ -402,18 +402,18 @@ "summary": "PersistentVolume is filling up." "expr": | ( - kubelet_volume_stats_available_bytes{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} + kubelet_volume_stats_available_bytes{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} / - kubelet_volume_stats_capacity_bytes{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} + kubelet_volume_stats_capacity_bytes{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} ) < 0.15 and - kubelet_volume_stats_used_bytes{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} > 0 + kubelet_volume_stats_used_bytes{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} > 0 and - predict_linear(kubelet_volume_stats_available_bytes{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"}[6h], 4 * 24 * 3600) < 0 + predict_linear(kubelet_volume_stats_available_bytes{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"}[6h], 4 * 24 * 3600) < 0 unless on(cluster, namespace, persistentvolumeclaim) - kube_persistentvolumeclaim_access_mode{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*", access_mode="ReadOnlyMany"} == 1 + kube_persistentvolumeclaim_access_mode{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*", access_mode="ReadOnlyMany"} == 1 unless on(cluster, namespace, persistentvolumeclaim) - kube_persistentvolumeclaim_labels{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",label_excluded_from_alerts="true"} == 1 + kube_persistentvolumeclaim_labels{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",label_excluded_from_alerts="true"} == 1 unless on(cluster, namespace, persistentvolumeclaim) kubelet_volume_stats_available_bytes{persistentvolumeclaim=~"scanner-db.*|prometheus-data-prometheus-k8s.*"} "for": "1h" "labels": @@ -426,16 +426,16 @@ "summary": "PersistentVolumeInodes are filling up." "expr": | ( - kubelet_volume_stats_inodes_free{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} + kubelet_volume_stats_inodes_free{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} / - kubelet_volume_stats_inodes{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} + kubelet_volume_stats_inodes{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} ) < 0.03 and - kubelet_volume_stats_inodes_used{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} > 0 + kubelet_volume_stats_inodes_used{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} > 0 unless on(cluster, namespace, persistentvolumeclaim) - kube_persistentvolumeclaim_access_mode{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*", access_mode="ReadOnlyMany"} == 1 + kube_persistentvolumeclaim_access_mode{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*", access_mode="ReadOnlyMany"} == 1 unless on(cluster, namespace, persistentvolumeclaim) - kube_persistentvolumeclaim_labels{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",label_excluded_from_alerts="true"} == 1 + kube_persistentvolumeclaim_labels{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",label_excluded_from_alerts="true"} == 1 "for": "1m" "labels": "severity": "critical" @@ -447,18 +447,18 @@ "summary": "PersistentVolumeInodes are filling up." "expr": | ( - kubelet_volume_stats_inodes_free{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} + kubelet_volume_stats_inodes_free{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} / - kubelet_volume_stats_inodes{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} + kubelet_volume_stats_inodes{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} ) < 0.15 and - kubelet_volume_stats_inodes_used{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} > 0 + kubelet_volume_stats_inodes_used{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} > 0 and - predict_linear(kubelet_volume_stats_inodes_free{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"}[6h], 4 * 24 * 3600) < 0 + predict_linear(kubelet_volume_stats_inodes_free{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"}[6h], 4 * 24 * 3600) < 0 unless on(cluster, namespace, persistentvolumeclaim) - kube_persistentvolumeclaim_access_mode{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*", access_mode="ReadOnlyMany"} == 1 + kube_persistentvolumeclaim_access_mode{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*", access_mode="ReadOnlyMany"} == 1 unless on(cluster, namespace, persistentvolumeclaim) - kube_persistentvolumeclaim_labels{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",label_excluded_from_alerts="true"} == 1 + kube_persistentvolumeclaim_labels{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",label_excluded_from_alerts="true"} == 1 "for": "1h" "labels": "severity": "warning" @@ -469,7 +469,7 @@ "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeerrors" "summary": "PersistentVolume is having issues with provisioning." "expr": | - kube_persistentvolume_status_phase{phase=~"Failed|Pending",namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} > 0 + kube_persistentvolume_status_phase{phase=~"Failed|Pending",namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} > 0 "for": "5m" "labels": "severity": "critical" diff --git a/resources/mixins/kubernetes/mixin.libsonnet b/resources/mixins/kubernetes/mixin.libsonnet index 2129872..ae8fbef 100644 --- a/resources/mixins/kubernetes/mixin.libsonnet +++ b/resources/mixins/kubernetes/mixin.libsonnet @@ -10,7 +10,7 @@ kubernetes { kubeApiserverSelector: 'job="api"', kubeProxySelector: 'job="machine-config-daemon"', kubeSchedulerSelector: 'job="scheduler"', - namespaceSelector: 'namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*"', + namespaceSelector: 'namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*"', }, } + { // Customize alert labels. diff --git a/resources/prometheus/generated/kubernetes-mixin-alerts.yaml b/resources/prometheus/generated/kubernetes-mixin-alerts.yaml index dcf7538..2e45b94 100644 --- a/resources/prometheus/generated/kubernetes-mixin-alerts.yaml +++ b/resources/prometheus/generated/kubernetes-mixin-alerts.yaml @@ -15,7 +15,7 @@ spec: "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodcrashlooping" "summary": "Pod is crash looping." "expr": | - max_over_time(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff", namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"}[5m]) >= 1 + max_over_time(kube_pod_container_status_waiting_reason{reason="CrashLoopBackOff", namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"}[5m]) >= 1 "for": "15m" "labels": "severity": "warning" @@ -28,7 +28,7 @@ spec: "expr": | sum by (namespace, pod, cluster) ( max by(namespace, pod, cluster) ( - kube_pod_status_phase{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics", phase=~"Pending|Unknown|Failed"} + kube_pod_status_phase{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics", phase=~"Pending|Unknown|Failed"} ) * on(namespace, pod, cluster) group_left(owner_kind) topk by(namespace, pod, cluster) ( 1, max by(namespace, pod, owner_kind, cluster) (kube_pod_owner{owner_kind!="Job"}) ) @@ -43,9 +43,9 @@ spec: "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentgenerationmismatch" "summary": "Deployment generation mismatch due to possible roll-back" "expr": | - kube_deployment_status_observed_generation{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_deployment_status_observed_generation{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} != - kube_deployment_metadata_generation{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_deployment_metadata_generation{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} "for": "15m" "labels": "severity": "warning" @@ -57,11 +57,11 @@ spec: "summary": "Deployment has not matched the expected number of replicas." "expr": | ( - kube_deployment_spec_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_deployment_spec_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} > - kube_deployment_status_replicas_available{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_deployment_status_replicas_available{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} ) and ( - changes(kube_deployment_status_replicas_updated{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"}[10m]) + changes(kube_deployment_status_replicas_updated{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"}[10m]) == 0 ) @@ -75,7 +75,7 @@ spec: "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedeploymentrolloutstuck" "summary": "Deployment rollout is not progressing." "expr": | - kube_deployment_status_condition{condition="Progressing", status="false",namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_deployment_status_condition{condition="Progressing", status="false",namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} != 0 "for": "15m" "labels": @@ -88,11 +88,11 @@ spec: "summary": "StatefulSet has not matched the expected number of replicas." "expr": | ( - kube_statefulset_status_replicas_ready{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_statefulset_status_replicas_ready{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} != - kube_statefulset_status_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_statefulset_status_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} ) and ( - changes(kube_statefulset_status_replicas_updated{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"}[10m]) + changes(kube_statefulset_status_replicas_updated{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"}[10m]) == 0 ) @@ -106,9 +106,9 @@ spec: "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubestatefulsetgenerationmismatch" "summary": "StatefulSet generation mismatch due to possible roll-back" "expr": | - kube_statefulset_status_observed_generation{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_statefulset_status_observed_generation{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} != - kube_statefulset_metadata_generation{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_statefulset_metadata_generation{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} "for": "15m" "labels": "severity": "warning" @@ -121,18 +121,18 @@ spec: "expr": | ( max without (revision) ( - kube_statefulset_status_current_revision{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_statefulset_status_current_revision{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} unless - kube_statefulset_status_update_revision{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_statefulset_status_update_revision{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} ) * ( - kube_statefulset_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_statefulset_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} != - kube_statefulset_status_replicas_updated{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_statefulset_status_replicas_updated{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} ) ) and ( - changes(kube_statefulset_status_replicas_updated{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"}[5m]) + changes(kube_statefulset_status_replicas_updated{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"}[5m]) == 0 ) @@ -148,24 +148,24 @@ spec: "expr": | ( ( - kube_daemonset_status_current_number_scheduled{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_daemonset_status_current_number_scheduled{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} != - kube_daemonset_status_desired_number_scheduled{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_daemonset_status_desired_number_scheduled{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} ) or ( - kube_daemonset_status_number_misscheduled{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_daemonset_status_number_misscheduled{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} != 0 ) or ( - kube_daemonset_status_updated_number_scheduled{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_daemonset_status_updated_number_scheduled{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} != - kube_daemonset_status_desired_number_scheduled{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_daemonset_status_desired_number_scheduled{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} ) or ( - kube_daemonset_status_number_available{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_daemonset_status_number_available{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} != - kube_daemonset_status_desired_number_scheduled{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_daemonset_status_desired_number_scheduled{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} ) ) and ( - changes(kube_daemonset_status_updated_number_scheduled{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"}[5m]) + changes(kube_daemonset_status_updated_number_scheduled{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"}[5m]) == 0 ) @@ -179,7 +179,7 @@ spec: "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecontainerwaiting" "summary": "Pod container waiting longer than 1 hour" "expr": | - sum by (namespace, pod, container, cluster) (kube_pod_container_status_waiting_reason{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"}) > 0 + sum by (namespace, pod, container, cluster) (kube_pod_container_status_waiting_reason{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"}) > 0 "for": "1h" "labels": "severity": "warning" @@ -190,9 +190,9 @@ spec: "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetnotscheduled" "summary": "DaemonSet pods are not scheduled." "expr": | - kube_daemonset_status_desired_number_scheduled{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_daemonset_status_desired_number_scheduled{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} - - kube_daemonset_status_current_number_scheduled{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} > 0 + kube_daemonset_status_current_number_scheduled{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} > 0 "for": "10m" "labels": "severity": "warning" @@ -203,7 +203,7 @@ spec: "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubedaemonsetmisscheduled" "summary": "DaemonSet pods are misscheduled." "expr": | - kube_daemonset_status_number_misscheduled{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} > 0 + kube_daemonset_status_number_misscheduled{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} > 0 "for": "15m" "labels": "severity": "warning" @@ -214,9 +214,9 @@ spec: "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobnotcompleted" "summary": "Job did not complete in time" "expr": | - time() - max by(namespace, job_name, cluster) (kube_job_status_start_time{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + time() - max by(namespace, job_name, cluster) (kube_job_status_start_time{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} and - kube_job_status_active{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} > 0) > 43200 + kube_job_status_active{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} > 0) > 43200 "labels": "severity": "warning" "source": "mixin/kubernetes" @@ -226,7 +226,7 @@ spec: "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubejobfailed" "summary": "Job failed to complete." "expr": | - kube_job_failed{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} > 0 + kube_job_failed{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} > 0 "for": "15m" "labels": "severity": "warning" @@ -237,19 +237,19 @@ spec: "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubehpareplicasmismatch" "summary": "HPA has not matched desired number of replicas." "expr": | - (kube_horizontalpodautoscaler_status_desired_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + (kube_horizontalpodautoscaler_status_desired_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} != - kube_horizontalpodautoscaler_status_current_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"}) + kube_horizontalpodautoscaler_status_current_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"}) and - (kube_horizontalpodautoscaler_status_current_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + (kube_horizontalpodautoscaler_status_current_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} > - kube_horizontalpodautoscaler_spec_min_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"}) + kube_horizontalpodautoscaler_spec_min_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"}) and - (kube_horizontalpodautoscaler_status_current_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + (kube_horizontalpodautoscaler_status_current_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} < - kube_horizontalpodautoscaler_spec_max_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"}) + kube_horizontalpodautoscaler_spec_max_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"}) and - changes(kube_horizontalpodautoscaler_status_current_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"}[15m]) == 0 + changes(kube_horizontalpodautoscaler_status_current_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"}[15m]) == 0 "for": "15m" "labels": "severity": "warning" @@ -260,9 +260,9 @@ spec: "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubehpamaxedout" "summary": "HPA is running at max replicas" "expr": | - kube_horizontalpodautoscaler_status_current_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_horizontalpodautoscaler_status_current_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} == - kube_horizontalpodautoscaler_spec_max_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} + kube_horizontalpodautoscaler_spec_max_replicas{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} "for": "15m" "labels": "severity": "info" @@ -301,7 +301,7 @@ spec: "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubecpuquotaovercommit" "summary": "Cluster has overcommitted CPU resource requests." "expr": | - sum(min without(resource) (kube_resourcequota{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics", type="hard", resource=~"(cpu|requests.cpu)"})) + sum(min without(resource) (kube_resourcequota{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics", type="hard", resource=~"(cpu|requests.cpu)"})) / sum(kube_node_status_allocatable{resource="cpu", job="kube-state-metrics"}) > 1.5 @@ -315,7 +315,7 @@ spec: "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubememoryquotaovercommit" "summary": "Cluster has overcommitted memory resource requests." "expr": | - sum(min without(resource) (kube_resourcequota{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics", type="hard", resource=~"(memory|requests.memory)"})) + sum(min without(resource) (kube_resourcequota{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics", type="hard", resource=~"(memory|requests.memory)"})) / sum(kube_node_status_allocatable{resource="memory", job="kube-state-metrics"}) > 1.5 @@ -329,9 +329,9 @@ spec: "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubequotaalmostfull" "summary": "Namespace quota is going to be full." "expr": | - kube_resourcequota{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics", type="used"} + kube_resourcequota{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics", type="used"} / ignoring(instance, job, type) - (kube_resourcequota{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics", type="hard"} > 0) + (kube_resourcequota{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics", type="hard"} > 0) > 0.9 < 1 "for": "15m" "labels": @@ -343,9 +343,9 @@ spec: "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubequotafullyused" "summary": "Namespace quota is fully used." "expr": | - kube_resourcequota{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics", type="used"} + kube_resourcequota{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics", type="used"} / ignoring(instance, job, type) - (kube_resourcequota{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics", type="hard"} > 0) + (kube_resourcequota{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics", type="hard"} > 0) == 1 "for": "15m" "labels": @@ -357,9 +357,9 @@ spec: "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubequotaexceeded" "summary": "Namespace quota has exceeded the limits." "expr": | - kube_resourcequota{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics", type="used"} + kube_resourcequota{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics", type="used"} / ignoring(instance, job, type) - (kube_resourcequota{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics", type="hard"} > 0) + (kube_resourcequota{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics", type="hard"} > 0) > 1 "for": "15m" "labels": @@ -389,16 +389,16 @@ spec: "summary": "PersistentVolume is filling up." "expr": | ( - kubelet_volume_stats_available_bytes{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} + kubelet_volume_stats_available_bytes{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} / - kubelet_volume_stats_capacity_bytes{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} + kubelet_volume_stats_capacity_bytes{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} ) < 0.03 and - kubelet_volume_stats_used_bytes{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} > 0 + kubelet_volume_stats_used_bytes{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} > 0 unless on(cluster, namespace, persistentvolumeclaim) - kube_persistentvolumeclaim_access_mode{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*", access_mode="ReadOnlyMany"} == 1 + kube_persistentvolumeclaim_access_mode{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*", access_mode="ReadOnlyMany"} == 1 unless on(cluster, namespace, persistentvolumeclaim) - kube_persistentvolumeclaim_labels{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",label_excluded_from_alerts="true"} == 1 + kube_persistentvolumeclaim_labels{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",label_excluded_from_alerts="true"} == 1 "for": "1m" "labels": "severity": "critical" @@ -410,18 +410,18 @@ spec: "summary": "PersistentVolume is filling up." "expr": | ( - kubelet_volume_stats_available_bytes{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} + kubelet_volume_stats_available_bytes{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} / - kubelet_volume_stats_capacity_bytes{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} + kubelet_volume_stats_capacity_bytes{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} ) < 0.15 and - kubelet_volume_stats_used_bytes{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} > 0 + kubelet_volume_stats_used_bytes{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} > 0 and - predict_linear(kubelet_volume_stats_available_bytes{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"}[6h], 4 * 24 * 3600) < 0 + predict_linear(kubelet_volume_stats_available_bytes{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"}[6h], 4 * 24 * 3600) < 0 unless on(cluster, namespace, persistentvolumeclaim) - kube_persistentvolumeclaim_access_mode{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*", access_mode="ReadOnlyMany"} == 1 + kube_persistentvolumeclaim_access_mode{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*", access_mode="ReadOnlyMany"} == 1 unless on(cluster, namespace, persistentvolumeclaim) - kube_persistentvolumeclaim_labels{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",label_excluded_from_alerts="true"} == 1 + kube_persistentvolumeclaim_labels{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",label_excluded_from_alerts="true"} == 1 unless on(cluster, namespace, persistentvolumeclaim) kubelet_volume_stats_available_bytes{persistentvolumeclaim=~"scanner-db.*|prometheus-data-prometheus-k8s.*"} "for": "1h" "labels": @@ -434,16 +434,16 @@ spec: "summary": "PersistentVolumeInodes are filling up." "expr": | ( - kubelet_volume_stats_inodes_free{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} + kubelet_volume_stats_inodes_free{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} / - kubelet_volume_stats_inodes{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} + kubelet_volume_stats_inodes{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} ) < 0.03 and - kubelet_volume_stats_inodes_used{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} > 0 + kubelet_volume_stats_inodes_used{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} > 0 unless on(cluster, namespace, persistentvolumeclaim) - kube_persistentvolumeclaim_access_mode{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*", access_mode="ReadOnlyMany"} == 1 + kube_persistentvolumeclaim_access_mode{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*", access_mode="ReadOnlyMany"} == 1 unless on(cluster, namespace, persistentvolumeclaim) - kube_persistentvolumeclaim_labels{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",label_excluded_from_alerts="true"} == 1 + kube_persistentvolumeclaim_labels{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",label_excluded_from_alerts="true"} == 1 "for": "1m" "labels": "severity": "critical" @@ -455,18 +455,18 @@ spec: "summary": "PersistentVolumeInodes are filling up." "expr": | ( - kubelet_volume_stats_inodes_free{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} + kubelet_volume_stats_inodes_free{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} / - kubelet_volume_stats_inodes{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} + kubelet_volume_stats_inodes{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} ) < 0.15 and - kubelet_volume_stats_inodes_used{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} > 0 + kubelet_volume_stats_inodes_used{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"} > 0 and - predict_linear(kubelet_volume_stats_inodes_free{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"}[6h], 4 * 24 * 3600) < 0 + predict_linear(kubelet_volume_stats_inodes_free{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kubelet"}[6h], 4 * 24 * 3600) < 0 unless on(cluster, namespace, persistentvolumeclaim) - kube_persistentvolumeclaim_access_mode{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*", access_mode="ReadOnlyMany"} == 1 + kube_persistentvolumeclaim_access_mode{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*", access_mode="ReadOnlyMany"} == 1 unless on(cluster, namespace, persistentvolumeclaim) - kube_persistentvolumeclaim_labels{namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",label_excluded_from_alerts="true"} == 1 + kube_persistentvolumeclaim_labels{namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",label_excluded_from_alerts="true"} == 1 "for": "1h" "labels": "severity": "warning" @@ -477,7 +477,7 @@ spec: "runbook_url": "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeerrors" "summary": "PersistentVolume is having issues with provisioning." "expr": | - kube_persistentvolume_status_phase{phase=~"Failed|Pending",namespace!~"openshift-kube.*|openshift-logging|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} > 0 + kube_persistentvolume_status_phase{phase=~"Failed|Pending",namespace!~"openshift-kube.*|openshift-logging|openshift-custom-domains.*|openshift-marketplace|openshift-aws-vpce-operator|openshift-deployment.*|kube.*",job="kube-state-metrics"} > 0 "for": "5m" "labels": "severity": "critical"