Merge branch 'main' into fix-dex-absent-query-for-mimir

giantswarm · Jun 10, 2024 · 7b967ac · 7b967ac
2 parents e071c9c + 00856f0
commit 7b967ac
Show file tree

Hide file tree

Showing 20 changed files with 21 additions and 20 deletions.
diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -25,13 +25,14 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 - Use `ready` replicas for Kyverno webhooks alert.
 - Moves ownership of alerts for shared components to turtles.
 
+
 ### Fixed
 
 - Fixed usage of yq, and jq in check-opsrecipes.sh
 - Fetch jq with make install-tools
 - Fix and improve the check-opsrecipes.sh script so support <directory>/_index.md based ops-recipes.
 - Fix cabbage alerts for multi-provider wcs.
-- Fix shield alert area labels.
+- Fix a few area labels.
 - Fix `cert-exporter` alerting.
 
 ### Removed

diff --git a/helm/prometheus-rules/templates/_helpers.tpl b/helm/prometheus-rules/templates/_helpers.tpl
@@ -38,10 +38,6 @@ giantswarm.io/service-type: {{ .Values.serviceType }}
 {{- end -}}
 {{- end -}}
 
-{{- define "isBastionBeingMonitored" -}}
-{{ not (eq .Values.managementCluster.provider.flavor "capi") }}
-{{- end -}}
-
 {{- define "namespaceNotGiantswarm" -}}
 "(([^g]|g[^i]|gi[^a]|gia[^n]|gian[^t]|giant[^s]|giants[^w]|giantsw[^a]|giantswa[^r]|giantswar[^m])*)"
 {{- end -}}
diff --git a/helm/prometheus-rules/templates/kaas/bigmac/alerting-rules/dex.rules.yml b/helm/prometheus-rules/templates/kaas/bigmac/alerting-rules/dex.rules.yml
@@ -20,7 +20,7 @@ spec:
       expr: sum(increase(http_requests_total{app="dex", handler!="/token", code=~"^[4]..$|[5]..$", cluster_type="management_cluster"}[5m])) by (cluster_id, installation, pipeline, provider) > 10
       for: 30m
       labels:
-        area: managedapps
+        area: kaas
         cancel_if_outside_working_hours: "true"
         severity: page
         team: bigmac

diff --git a/...etheus-rules/templates/kaas/phoenix/alerting-rules/aws-load-balancer-controller.rules.yml b/...etheus-rules/templates/kaas/phoenix/alerting-rules/aws-load-balancer-controller.rules.yml
@@ -21,7 +21,7 @@ spec:
       expr: sum(increase(aws_api_calls_total{error_code != ""}[20m])) by (error_code,namespace,pod,cluster_id) > 0
       for: 40m
       labels:
-        area: managedservices
+        area: kaas
         cancel_if_cluster_status_creating: "true"
         cancel_if_cluster_status_deleting: "true"
         cancel_if_cluster_status_updating: "true"
@@ -36,7 +36,7 @@ spec:
       expr: sum(increase(controller_runtime_reconcile_total{service="aws-load-balancer-controller", result = "error"}[20m])) by (controller,namespace,pod,cluster_id) > 0
       for: 40m
       labels:
-        area: managedservices
+        area: kaas
         cancel_if_cluster_status_creating: "true"
         cancel_if_cluster_status_deleting: "true"
         cancel_if_cluster_status_updating: "true"

diff --git a/helm/prometheus-rules/templates/kaas/phoenix/alerting-rules/aws.management-cluster.rules.yml b/helm/prometheus-rules/templates/kaas/phoenix/alerting-rules/aws.management-cluster.rules.yml
@@ -116,7 +116,7 @@ spec:
       labels:
         area: kaas
         severity: notify
-        team: {{ include "providerTeam" . }}
+        team: phoenix
         topic: aws
     - alert: ServiceUsageApproachingLimit
       annotations:
@@ -127,7 +127,7 @@ spec:
       labels:
         area: kaas
         severity: notify
-        team: {{ include "providerTeam" . }}
+        team: phoenix
         topic: aws
     - alert: ManagementClusterContainerIsRestartingTooFrequentlyAWS
       annotations:

diff --git a/helm/prometheus-rules/templates/kaas/phoenix/alerting-rules/cluster-service.rules.yml b/helm/prometheus-rules/templates/kaas/phoenix/alerting-rules/cluster-service.rules.yml
@@ -22,7 +22,7 @@ spec:
       expr: cluster_service_key_pair_total > 2400
       for: 10m
       labels:
-        area: storage
+        area: kaas
         severity: page
         team: phoenix
         topic: managementcluster

diff --git a/helm/prometheus-rules/templates/kaas/phoenix/alerting-rules/kiam.rules.yml b/helm/prometheus-rules/templates/kaas/phoenix/alerting-rules/kiam.rules.yml
@@ -20,7 +20,7 @@ spec:
       expr: increase(kiam_metadata_find_role_errors_total[10m]) > 0
       for: 15m
       labels:
-        area: managedservices
+        area: kaas
         cancel_if_cluster_status_creating: "true"
         cancel_if_cluster_status_deleting: "true"
         cancel_if_cluster_status_updating: "true"

diff --git a/...ometheus-rules/templates/kaas/turtles/alerting-rules/apiserver.workload-cluster.rules.yml b/...ometheus-rules/templates/kaas/turtles/alerting-rules/apiserver.workload-cluster.rules.yml
@@ -44,6 +44,7 @@ spec:
         team: {{ include "providerTeam" . }}
         topic: kubernetes
 
+    ## TODO Split this alert into multiple alerts for each webhook.
     # Webhooks that are not explicitely owner by any team (customer owned ones).
     - alert: WorkloadClusterWebhookDurationExceedsTimeoutSolutionEngineers
       annotations:

diff --git a/.../shared/alerting-rules/bastions.rules.yml → ...turtles/alerting-rules/bastions.rules.yml b/.../shared/alerting-rules/bastions.rules.yml → ...turtles/alerting-rules/bastions.rules.yml
@@ -1,4 +1,5 @@
-{{- if eq (include "isBastionBeingMonitored" .) "true" }}
+{{- if eq .Values.managementCluster.provider.flavor "vintage" }}
+## TODO Remove when all vintage installations are gone
 apiVersion: monitoring.coreos.com/v1
 kind: PrometheusRule
 metadata:

diff --git a/helm/prometheus-rules/templates/platform/atlas/alerting-rules/storage.rules.yml b/helm/prometheus-rules/templates/platform/atlas/alerting-rules/storage.rules.yml
@@ -20,7 +20,7 @@ spec:
       expr: kubelet_volume_stats_available_bytes{cluster_type="management_cluster", persistentvolumeclaim=~".*(alertmanager|loki|mimir|prometheus|pyroscope|tempo).*"}/kubelet_volume_stats_capacity_bytes{persistentvolumeclaim=~".*(alertmanager|loki|mimir|prometheus|pyroscope|tempo).*"} < 0.10
       for: 1h
       labels:
-        area: empowerment
+        area: platform
         cancel_if_outside_working_hours: "true"
         severity: page
         team: atlas

diff --git a/...red/recording-rules/loki-mixins.rules.yml → ...las/recording-rules/loki-mixins.rules.yml b/...red/recording-rules/loki-mixins.rules.yml → ...las/recording-rules/loki-mixins.rules.yml
diff --git a/...ed/recording-rules/mimir-mixins.rules.yml → ...as/recording-rules/mimir-mixins.rules.yml b/...ed/recording-rules/mimir-mixins.rules.yml → ...as/recording-rules/mimir-mixins.rules.yml
diff --git a/...ring.resource-usage-estimation.rules.yaml → ...ring.resource-usage-estimation.rules.yaml b/...ring.resource-usage-estimation.rules.yaml → ...ring.resource-usage-estimation.rules.yaml
diff --git a/...bage/alerting-rules/network.all.rules.yml → .../cabbage/alerting-rules/network.rules.yml b/...bage/alerting-rules/network.all.rules.yml → .../cabbage/alerting-rules/network.rules.yml
@@ -4,11 +4,11 @@ metadata:
   creationTimestamp: null
   labels:
     {{- include "labels.common" . | nindent 4 }}
-  name: network.all.rules
+  name: network.rules
   namespace: {{ .Values.namespace  }}
 spec:
   groups:
-  - name: network.all
+  - name: network
     rules:
     - alert: DNSErrorRateTooHigh
       annotations:
@@ -44,6 +44,7 @@ spec:
         severity: page
         team: cabbage
         topic: network
+    ## TODO Sort those alerts ownership
     - alert: NetworkErrorRateTooHigh
       annotations:
         description: '{{`Network error rate is too high for {{ or $labels.pod_name $labels.instance }} to {{ $labels.host }}.`}}'

diff --git a/...s-managed-app-deployment-status.rules.yml → ...s-managed-app-deployment-status.rules.yml b/...s-managed-app-deployment-status.rules.yml → ...s-managed-app-deployment-status.rules.yml
@@ -1,3 +1,4 @@
+## Cabbage is the only user of those recording rules
 apiVersion: monitoring.coreos.com/v1
 kind: PrometheusRule
 metadata:

diff --git a/...recording-rules/helm-operations.rules.yml → ...recording-rules/helm-operations.rules.yml b/...recording-rules/helm-operations.rules.yml → ...recording-rules/helm-operations.rules.yml
diff --git a/...ield/alerting-rules/kyverno.all.rules.yml → ...m/shield/alerting-rules/kyverno.rules.yml b/...ield/alerting-rules/kyverno.all.rules.yml → ...m/shield/alerting-rules/kyverno.rules.yml
@@ -4,7 +4,7 @@ metadata:
   creationTimestamp: null
   labels:
     {{- include "labels.common" . | nindent 4 }}
-  name: kyverno.all.rules
+  name: kyverno.rules
   namespace: {{ .Values.namespace  }}
 spec:
   groups:

diff --git a/helm/prometheus-rules/templates/shared/alerting-rules/up.rules.yml b/helm/prometheus-rules/templates/shared/alerting-rules/up.rules.yml
@@ -17,7 +17,7 @@ spec:
       expr: label_replace(up{app=~"chart-operator.*"}, "ip", "$1.$2.$3.$4", "node", "ip-(\\d+)-(\\d+)-(\\d+)-(\\d+).*") == 0
       for: 15m
       labels:
-        area: managedservices
+        area: platform
         cancel_if_cluster_control_plane_unhealthy: "true"
         cancel_if_cluster_status_creating: "true"
         cancel_if_cluster_status_deleting: "true"

diff --git a/loki/update.sh b/loki/update.sh
@@ -11,7 +11,7 @@ set -e
 
 BRANCH="main"
 MIXIN_URL=https://github.com/grafana/loki/production/loki-mixin@$BRANCH
-OUTPUT_FILE="$(pwd)"/helm/prometheus-rules/templates/shared/recording-rules/loki-mixins.rules.yml
+OUTPUT_FILE="$(pwd)"/helm/prometheus-rules/templates/platform/atlas/recording-rules/loki-mixins.rules.yml
 
 cd loki
 rm -rf vendor jsonnetfile.* "$OUTPUT_FILE"

diff --git a/mimir/update.sh b/mimir/update.sh
@@ -11,7 +11,7 @@ set -e
 
 BRANCH="main"
 MIXIN_URL=https://github.com/grafana/mimir/operations/mimir-mixin@$BRANCH
-OUTPUT_FILE="$(pwd)"/helm/prometheus-rules/templates/shared/recording-rules/mimir-mixins.rules.yml
+OUTPUT_FILE="$(pwd)"/helm/prometheus-rules/templates/platform/atlas/recording-rules/mimir-mixins.rules.yml
 
 cd mimir
 rm -rf vendor jsonnetfile.* "$OUTPUT_FILE"