Fix Prometheus agent failing alerts

Signed-off-by: QuentinBisson <[email protected]>
giantswarm · Sep 19, 2023 · 413f305 · 413f305
1 parent 737a5e9
commit 413f305
Show file tree

Hide file tree

Showing 3 changed files with 5 additions and 7 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -10,7 +10,11 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 ### Changed
 
 - Add missing prometheus-agent inhibition to `KubeStateMetricsDown` alert
-- Change time duration before `ManagementClusterDeploymentMissingAWS` pages because it is dependant on the `PrometheusAgentFailing` alert. 
+- Change time duration before `ManagementClusterDeploymentMissingAWS` pages because it is dependant on the `PrometheusAgentFailing` alert.
+
+### Fixed
+
+- Remove `cancel_if_outside_working_hours` from PrometheusAgent alerts.
 
 ## [2.132.0] - 2023-09-15
 

diff --git a/helm/prometheus-rules/templates/alerting-rules/prometheus-agent.rules.yml b/helm/prometheus-rules/templates/alerting-rules/prometheus-agent.rules.yml
@@ -38,7 +38,6 @@ spec:
         cancel_if_cluster_is_not_running_prometheus_agent: "true"
         cancel_if_cluster_status_creating: "true"
         cancel_if_cluster_status_deleting: "true"
-        cancel_if_outside_working_hours: "true"
     ## Page Atlas if prometheus agent is missing shards to send samples to MC prometheus.
     - alert: PrometheusAgentShardsMissing
       annotations:
@@ -74,5 +73,4 @@ spec:
         cancel_if_cluster_is_not_running_prometheus_agent: "true"
         cancel_if_cluster_status_creating: "true"
         cancel_if_cluster_status_deleting: "true"
-        cancel_if_outside_working_hours: "true"
 {{- end }}
diff --git a/test/tests/providers/global/prometheus-agent.rules.test.yml b/test/tests/providers/global/prometheus-agent.rules.test.yml
@@ -22,7 +22,6 @@ tests:
               cancel_if_cluster_is_not_running_prometheus_agent: "true"
               cancel_if_cluster_status_creating: "true"
               cancel_if_cluster_status_deleting: "true"
-              cancel_if_outside_working_hours: "true"
             exp_annotations:
               dashboard: "promRW001/prometheus-remote-write"
               description: "Prometheus agent remote write is failing."
@@ -44,7 +43,6 @@ tests:
               cancel_if_cluster_is_not_running_prometheus_agent: "true"
               cancel_if_cluster_status_creating: "true"
               cancel_if_cluster_status_deleting: "true"
-              cancel_if_outside_working_hours: "true"
             exp_annotations:
               dashboard: "promRW001/prometheus-remote-write"
               description: "Prometheus agent remote write is failing."
@@ -80,7 +78,6 @@ tests:
               cancel_if_cluster_is_not_running_prometheus_agent: "true"
               cancel_if_cluster_status_creating: "true"
               cancel_if_cluster_status_deleting: "true"
-              cancel_if_outside_working_hours: "true"
             exp_annotations:
               description: "Prometheus agent is missing shards."
               opsrecipe: "prometheus-agent-missing-shards/"
@@ -97,7 +94,6 @@ tests:
               cancel_if_cluster_is_not_running_prometheus_agent: "true"
               cancel_if_cluster_status_creating: "true"
               cancel_if_cluster_status_deleting: "true"
-              cancel_if_outside_working_hours: "true"
             exp_annotations:
               description: "Prometheus agent is missing shards."
               opsrecipe: "prometheus-agent-missing-shards/"