Use workingHoursOnly on more alerts (#1009)

* Use workingHoursOnly on more alerts Signed-off-by: Marcus Noble <[email protected]> * Removed old openstack unit tests Signed-off-by: Marcus Noble <[email protected]> * No longer silence all CAPA and CAPZ alerts out of hours by default Signed-off-by: Marcus Noble <[email protected]> * Updated unit tests Signed-off-by: Marcus Noble <[email protected]> --------- Signed-off-by: Marcus Noble <[email protected]>
giantswarm · Jan 22, 2024 · c4b0db3 · c4b0db3
1 parent ee2ad40
commit c4b0db3
Show file tree

Hide file tree

Showing 22 changed files with 29 additions and 408 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -14,6 +14,8 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Changed
 
 - Changed teleport alerts to take into accont only `Provisioned` clusters
+- Made use of `workingHoursOnly` template on more alerts to ensure `stable-testing` MCs don't page out of hours
+- No longer silence all CAPA and CAPZ alerts out of hours by default
 
 ## [2.148.0] - 2024-01-17
 
@@ -150,7 +152,7 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ### Changed
 
-- fixed `aggregation:kyverno_policy_job_status_team` expression. 
+- fixed `aggregation:kyverno_policy_job_status_team` expression.
 
 ### Added
 

diff --git a/helm/prometheus-rules/templates/_helpers.tpl b/helm/prometheus-rules/templates/_helpers.tpl
@@ -40,7 +40,7 @@ phoenix
 {{- end -}}
 
 {{- define "workingHoursOnly" -}}
-{{- if has .Values.managementCluster.provider.kind (list "openstack" "capz" "capa") -}}
+{{- if has .Values.managementCluster.provider.kind (list "openstack") -}}
 "true"
 {{- else if eq .Values.managementCluster.pipeline "stable-testing" -}}
 "true"

diff --git a/helm/prometheus-rules/templates/alerting-rules/aws-load-balancer-controller.rules.yml b/helm/prometheus-rules/templates/alerting-rules/aws-load-balancer-controller.rules.yml
@@ -23,7 +23,7 @@ spec:
         cancel_if_cluster_status_creating: "true"
         cancel_if_cluster_status_deleting: "true"
         cancel_if_cluster_status_updating: "true"
-        cancel_if_outside_working_hours: "false"
+        cancel_if_outside_working_hours: {{ include "workingHoursOnly" . }}
         severity: page
         team: phoenix
         topic: alb
@@ -38,7 +38,7 @@ spec:
         cancel_if_cluster_status_creating: "true"
         cancel_if_cluster_status_deleting: "true"
         cancel_if_cluster_status_updating: "true"
-        cancel_if_outside_working_hours: "false"
+        cancel_if_outside_working_hours: {{ include "workingHoursOnly" . }}
         severity: page
         team: phoenix
         topic: alb

diff --git a/helm/prometheus-rules/templates/alerting-rules/kong.rules.yml b/helm/prometheus-rules/templates/alerting-rules/kong.rules.yml
@@ -22,7 +22,7 @@ spec:
         area: managedservices
         cancel_if_cluster_status_creating: "true"
         cancel_if_cluster_status_deleting: "true"
-        cancel_if_outside_working_hours: "false"
+        cancel_if_outside_working_hours: {{ include "workingHoursOnly" . }}
         severity: page
         team: cabbage
         topic: kong
@@ -36,7 +36,7 @@ spec:
         area: managedservices
         cancel_if_cluster_status_creating: "true"
         cancel_if_cluster_status_deleting: "true"
-        cancel_if_outside_working_hours: "false"
+        cancel_if_outside_working_hours: {{ include "workingHoursOnly" . }}
         severity: page
         team: cabbage
         topic: kong
diff --git a/helm/prometheus-rules/templates/alerting-rules/kube-state-metrics.rules.yml b/helm/prometheus-rules/templates/alerting-rules/kube-state-metrics.rules.yml
@@ -34,7 +34,7 @@ spec:
         inhibit_kube_state_metrics_down: "true"
         cancel_if_prometheus_agent_down: "true"
         cancel_if_kubelet_down: "true"
-        cancel_if_outside_working_hours: "false"
+        cancel_if_outside_working_hours: {{ include "workingHoursOnly" . }}
         severity: page
         team: atlas
         topic: observability

diff --git a/helm/prometheus-rules/templates/alerting-rules/kyverno.all.rules.yml b/helm/prometheus-rules/templates/alerting-rules/kyverno.all.rules.yml
@@ -21,7 +21,7 @@ spec:
         cancel_if_cluster_status_creating: "true"
         cancel_if_cluster_status_deleting: "true"
         cancel_if_cluster_status_updating: "true"
-        cancel_if_outside_working_hours: "false"
+        cancel_if_outside_working_hours: {{ include "workingHoursOnly" . }}
         severity: page
         team: shield
         topic: kyverno
@@ -38,7 +38,7 @@ spec:
         cancel_if_cluster_status_creating: "true"
         cancel_if_cluster_status_deleting: "true"
         cancel_if_cluster_status_updating: "true"
-        cancel_if_outside_working_hours: "false"
+        cancel_if_outside_working_hours: {{ include "workingHoursOnly" . }}
         severity: page
         team: shield
         topic: kyverno
@@ -53,7 +53,7 @@ spec:
         cancel_if_cluster_status_creating: "true"
         cancel_if_cluster_status_deleting: "true"
         cancel_if_cluster_status_updating: "true"
-        cancel_if_outside_working_hours: "false"
+        cancel_if_outside_working_hours: {{ include "workingHoursOnly" . }}
         severity: notify
         team: shield
         topic: kyverno

diff --git a/helm/prometheus-rules/templates/alerting-rules/linkerd.deployment.rules.yml b/helm/prometheus-rules/templates/alerting-rules/linkerd.deployment.rules.yml
@@ -18,7 +18,7 @@ spec:
       for: 30m
       labels:
         area: managedservices
-        cancel_if_outside_working_hours: "false"
+        cancel_if_outside_working_hours: {{ include "workingHoursOnly" . }}
         severity: page
         team: cabbage
         topic: linkerd
diff --git a/helm/prometheus-rules/templates/alerting-rules/prometheus.rules.yml b/helm/prometheus-rules/templates/alerting-rules/prometheus.rules.yml
@@ -23,7 +23,7 @@ spec:
         cancel_if_cluster_status_updating: "true"
         cancel_if_cluster_status_deleting: "true"
         cancel_if_cluster_has_no_workers: "true"
-        cancel_if_outside_working_hours: "false"
+        cancel_if_outside_working_hours: {{ include "workingHoursOnly" . }}
         severity: page
         team: atlas
         topic: observability

diff --git a/helm/prometheus-rules/templates/alerting-rules/sloth.rules.yml b/helm/prometheus-rules/templates/alerting-rules/sloth.rules.yml
@@ -21,7 +21,7 @@ spec:
         cancel_if_cluster_status_deleting: "true"
         cancel_if_cluster_status_updating: "true"
         cancel_if_scrape_timeout: "true"
-        cancel_if_outside_working_hours: "false"
+        cancel_if_outside_working_hours: {{ include "workingHoursOnly" . }}
         severity: page
         team: atlas
         topic: observability
diff --git a/test/conf/providers b/test/conf/providers
@@ -1,3 +1,2 @@
 vintage/aws
-capi/openstack
 capi/capz
diff --git a/test/tests/providers/capi/capz/capi-cluster.rules.test.yml b/test/tests/providers/capi/capz/capi-cluster.rules.test.yml
@@ -20,7 +20,7 @@ tests:
         exp_alerts:
           - exp_labels:
               area: kaas
-              cancel_if_outside_working_hours: "true"
+              cancel_if_outside_working_hours: "false"
               severity: page
               team: phoenix
               topic: managementcluster
@@ -35,7 +35,7 @@ tests:
         exp_alerts:
           - exp_labels:
               area: kaas
-              cancel_if_outside_working_hours: "true"
+              cancel_if_outside_working_hours: "false"
               severity: notify
               team: phoenix
               topic: managementcluster
@@ -51,7 +51,7 @@ tests:
         exp_alerts:
           - exp_labels:
               area: kaas
-              cancel_if_outside_working_hours: "true"
+              cancel_if_outside_working_hours: "false"
               severity: notify
               team: phoenix
               topic: managementcluster

diff --git a/test/tests/providers/capi/capz/capi-kubeadmcontrolplane.rules.test.yml b/test/tests/providers/capi/capz/capi-kubeadmcontrolplane.rules.test.yml
@@ -20,7 +20,7 @@ tests:
         exp_alerts:
           - exp_labels:
               area: kaas
-              cancel_if_outside_working_hours: "true"
+              cancel_if_outside_working_hours: "false"
               severity: notify
               team: phoenix
               topic: managementcluster
@@ -35,7 +35,7 @@ tests:
         exp_alerts:
           - exp_labels:
               area: kaas
-              cancel_if_outside_working_hours: "true"
+              cancel_if_outside_working_hours: "false"
               severity: notify
               team: phoenix
               topic: managementcluster

diff --git a/test/tests/providers/capi/capz/capi-machine.rules.test.yml b/test/tests/providers/capi/capz/capi-machine.rules.test.yml
@@ -16,7 +16,7 @@ tests:
         exp_alerts:
           - exp_labels:
               area: kaas
-              cancel_if_outside_working_hours: "true"
+              cancel_if_outside_working_hours: "false"
               severity: notify
               team: phoenix
               topic: managementcluster
@@ -32,7 +32,7 @@ tests:
         exp_alerts:
           - exp_labels:
               area: kaas
-              cancel_if_outside_working_hours: "true"
+              cancel_if_outside_working_hours: "false"
               severity: notify
               team: phoenix
               topic: managementcluster

diff --git a/test/tests/providers/capi/capz/capi-machinedeployment.rules.test.yml b/test/tests/providers/capi/capz/capi-machinedeployment.rules.test.yml
@@ -20,7 +20,7 @@ tests:
         exp_alerts:
           - exp_labels:
               area: kaas
-              cancel_if_outside_working_hours: "true"
+              cancel_if_outside_working_hours: "false"
               severity: notify
               team: phoenix
               topic: managementcluster
@@ -35,7 +35,7 @@ tests:
         exp_alerts:
           - exp_labels:
               area: kaas
-              cancel_if_outside_working_hours: "true"
+              cancel_if_outside_working_hours: "false"
               severity: notify
               team: phoenix
               topic: managementcluster

diff --git a/test/tests/providers/capi/capz/capi-machinepool.rules.test.yml b/test/tests/providers/capi/capz/capi-machinepool.rules.test.yml
@@ -20,7 +20,7 @@ tests:
         exp_alerts:
           - exp_labels:
               area: kaas
-              cancel_if_outside_working_hours: "true"
+              cancel_if_outside_working_hours: "false"
               severity: notify
               team: phoenix
               topic: managementcluster
@@ -35,7 +35,7 @@ tests:
         exp_alerts:
           - exp_labels:
               area: kaas
-              cancel_if_outside_working_hours: "true"
+              cancel_if_outside_working_hours: "false"
               severity: notify
               team: phoenix
               topic: managementcluster

diff --git a/test/tests/providers/capi/capz/capi-machineset.rules.test.yml b/test/tests/providers/capi/capz/capi-machineset.rules.test.yml
@@ -12,7 +12,7 @@ tests:
         exp_alerts:
           - exp_labels:
               area: kaas
-              cancel_if_outside_working_hours: "true"
+              cancel_if_outside_working_hours: "false"
               severity: notify
               team: phoenix
               topic: managementcluster

diff --git a/test/tests/providers/capi/capz/dns-operator-azure.rules.test.yml b/test/tests/providers/capi/capz/dns-operator-azure.rules.test.yml
@@ -20,7 +20,7 @@ tests:
         exp_alerts:
           - exp_labels:
               area: kaas
-              cancel_if_outside_working_hours: "true"
+              cancel_if_outside_working_hours: "false"
               severity: notify
               team: phoenix
               topic: managementcluster
@@ -36,7 +36,7 @@ tests:
         exp_alerts:
           - exp_labels:
               area: kaas
-              cancel_if_outside_working_hours: "true"
+              cancel_if_outside_working_hours: "false"
               severity: notify
               team: phoenix
               topic: managementcluster

diff --git a/test/tests/providers/capi/openstack/capi.rules.test.yml b/test/tests/providers/capi/openstack/capi.rules.test.yml