Add Alertmanager controller

CHANGELOG.md

@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- Add Alertmanager controller
+
 ## [0.10.1] - 2024-12-12
 
 ### Fixed

go.mod

@@ -16,6 +16,7 @@ require (
 	github.com/opsgenie/opsgenie-go-sdk-v2 v1.2.23
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.79.0
+	github.com/prometheus/alertmanager v0.27.0
 	github.com/prometheus/client_golang v1.20.5
 	github.com/prometheus/common v0.61.0
 	github.com/sirupsen/logrus v1.9.3
@@ -87,7 +88,10 @@ require (
 	github.com/Masterminds/goutils v1.1.1 // indirect
 	github.com/Masterminds/semver/v3 v3.3.0 // indirect
 	github.com/asaskevich/govalidator v0.0.0-20230301143203-a9d515a09cc2 // indirect
+	github.com/aws/aws-sdk-go v1.50.8 // indirect
 	github.com/fxamacker/cbor/v2 v2.7.0 // indirect
+	github.com/go-kit/log v0.2.1 // indirect
+	github.com/go-logfmt/logfmt v0.5.1 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/go-openapi/analysis v0.23.0 // indirect
 	github.com/go-openapi/errors v0.22.0 // indirect
@@ -97,12 +101,16 @@ require (
 	github.com/go-openapi/strfmt v0.23.0 // indirect
 	github.com/go-openapi/validate v0.24.0 // indirect
 	github.com/huandu/xstrings v1.5.0 // indirect
+	github.com/jmespath/go-jmespath v0.4.0 // indirect
+	github.com/jpillora/backoff v1.0.0 // indirect
 	github.com/klauspost/compress v1.17.9 // indirect
 	github.com/mitchellh/copystructure v1.2.0 // indirect
 	github.com/mitchellh/mapstructure v1.5.0 // indirect
 	github.com/mitchellh/reflectwalk v1.0.2 // indirect
+	github.com/mwitkow/go-conntrack v0.0.0-20190716064945-2f068394615f // indirect
 	github.com/oklog/ulid v1.3.1 // indirect
 	github.com/opentracing/opentracing-go v1.2.0 // indirect
+	github.com/prometheus/common/sigv4 v0.1.0 // indirect
 	github.com/shopspring/decimal v1.4.0 // indirect
 	github.com/spf13/cast v1.7.0 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
@@ -112,6 +120,7 @@ require (
 	go.opentelemetry.io/otel/trace v1.28.0 // indirect
 	golang.org/x/sync v0.10.0 // indirect
 	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 )
 
 replace (

helm/observability-operator/templates/alertmanager/_helpers.tpl

@@ -0,0 +1,5 @@
+{{/* vim: set filetype=mustache: */}}
+
+{{- define "alertmanager-secret.name" -}}
+{{- include "resource.default.name" . -}}-alertmanager
+{{- end }}

helm/observability-operator/templates/deployment.yaml

@@ -31,11 +31,14 @@ spec:
         - --management-cluster-pipeline={{ $.Values.managementCluster.pipeline }}
         - --management-cluster-region={{ $.Values.managementCluster.region }}
         # Monitoring configuration
+        - --alertmanager-secret-name={{ include "alertmanager-secret.name" . }}
+        - --alertmanager-url={{ $.Values.alerting.alertmanagerURL }}
         - --monitoring-enabled={{ $.Values.monitoring.enabled }}
         - --monitoring-agent={{ $.Values.monitoring.agent }}
         - --monitoring-sharding-scale-up-series-count={{ $.Values.monitoring.sharding.scaleUpSeriesCount }}
         - --monitoring-sharding-scale-down-percentage={{ $.Values.monitoring.sharding.scaleDownPercentage }}
         - --monitoring-wal-truncate-frequency={{ $.Values.monitoring.wal.truncateFrequency }}
+        - --namespace={{ include "resource.default.namespace" . }}
         {{- if .Values.monitoring.prometheusVersion }}
         - --prometheus-version={{ $.Values.monitoring.prometheusVersion }}
         {{- end }}

helm/observability-operator/values.yaml

@@ -15,6 +15,13 @@ managementCluster:
   pipeline: pipeline
   region: region
 
+alerting:
+  alertmanagerURL: ""
+  grafanaAddress: ""
+  proxyURL: ""
+  slackAPIToken: ""
+  slackAPIURL: ""
+
 monitoring:
   agent: alloy
   enabled: false

internal/controller/alertmanager_controller.go

@@ -0,0 +1,102 @@
+package controller
+
+import (
+	"context"
+
+	v1 "k8s.io/api/core/v1"
+	"k8s.io/apimachinery/pkg/types"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/log"
+	"sigs.k8s.io/controller-runtime/pkg/reconcile"
+
+	"github.com/pkg/errors"
+
+	"github.com/giantswarm/observability-operator/internal/controller/predicates"
+	"github.com/giantswarm/observability-operator/pkg/alertmanager"
+	"github.com/giantswarm/observability-operator/pkg/config"
+)
+
+// AlertmanagerReconciler reconciles the Alertmanager secret created by the observability-operator Helm chart
+// and configures the Alertmanager instance with the configuration stored in the secret.
+// This controller do not make use of finalizers as the configuration is not removed from Alertmanager when the secret is deleted.
+type AlertmanagerReconciler struct {
+	client client.Client
+
+	alertmanagerJob alertmanager.Job
+}
+
+// SetupAlertmanagerReconciler adds a controller into mgr that reconciles the Alertmanager secret.
+func SetupAlertmanagerReconciler(mgr ctrl.Manager, conf config.Config) error {
+	r := &AlertmanagerReconciler{
+		client:          mgr.GetClient(),
+		alertmanagerJob: alertmanager.New(conf),
+	}
+
+	// Filter only the Alertmanager secret created by the observability-operator Helm chart
+	secretPredicate := predicates.NewAlertmanagerSecretPredicate(conf.Monitoring.AlertmanagerSecretName, conf.Namespace)
+
+	// Filter only the Mimir Alertmanager pod
+	podPredicate := predicates.NewAlertmanagerPodPredicate()
+
+	// Requeue the Alertmanager secret when the Mimir Alertmanager pod changes
+	p := podEventHandler(conf.Monitoring.AlertmanagerSecretName, conf.Namespace)
+
+	// Setup the controller
+	return ctrl.NewControllerManagedBy(mgr).
+		For(&v1.Secret{}, builder.WithPredicates(secretPredicate)).
+		Watches(&v1.Pod{}, p, builder.WithPredicates(podPredicate)).
+		Complete(r)
+}
+
+// podEventHandler returns an event handler that enqueues requests for the Alertmanager secret only.
+// For now there is only one Alertmanager secret to be reconciled.
+func podEventHandler(secretName, namespace string) handler.EventHandler {
+	return handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, obj client.Object) []ctrl.Request {
+		return []reconcile.Request{
+			{
+				NamespacedName: types.NamespacedName{
+					Name:      secretName,
+					Namespace: namespace,
+				},
+			},
+		}
+	})
+}
+
+// Reconcile main logic
+func (r AlertmanagerReconciler) Reconcile(ctx context.Context, req reconcile.Request) (ctrl.Result, error) {
+	logger := log.FromContext(ctx)
+	logger = logger.WithValues("controller", "alertmanager")
+	log.IntoContext(ctx, logger)
+
+	logger.Info("Started reconciling")
+	defer logger.Info("Finished reconciling")
+
+	// Retrieve the secret being reconciled
+	secret := &v1.Secret{}
+	if err := r.client.Get(ctx, req.NamespacedName, secret); err != nil {
+		return ctrl.Result{}, errors.WithStack(client.IgnoreNotFound(err))
+	}
+
+	if !secret.DeletionTimestamp.IsZero() {
+		// Nothing to do if the secret is being deleted
+		// Configuration is not removed from Alertmanager when the secret is deleted.
+		return ctrl.Result{}, nil
+	}
+
+	return r.reconcileCreate(ctx, secret)
+}
+
+// Handle create and update events
+func (r AlertmanagerReconciler) reconcileCreate(ctx context.Context, secret *v1.Secret) (ctrl.Result, error) { // nolint: unparam
+	// Ensure the configuration is set and up to date in Alertmanager
+	err := r.alertmanagerJob.Configure(ctx, secret)
+	if err != nil {
+		return ctrl.Result{}, errors.WithStack(err)
+	}
+
+	return ctrl.Result{}, nil
+}

internal/controller/predicates/alertmanager_secret_predicate.go

@@ -0,0 +1,76 @@
+package predicates
+
+import (
+	v1 "k8s.io/api/core/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+)
+
+// NewAlertmanagerSecretPredicate returns a predicate that filters only the Alertmanager secret created by the observability-operator Helm chart.
+func NewAlertmanagerSecretPredicate(secretName, namespace string) predicate.Predicate {
+	filter := func(object client.Object) bool {
+		if object == nil {
+			return false
+		}
+
+		secret, ok := object.(*v1.Secret)
+		if !ok {
+			return false
+		}
+
+		if !secret.DeletionTimestamp.IsZero() {
+			return false
+		}
+
+		labels := secret.GetLabels()
+
+		ok = secret.GetName() == secretName &&
+			secret.GetNamespace() == namespace &&
+			labels != nil &&
+			labels["app.kubernetes.io/name"] == "observability-operator"
+
+		return ok
+	}
+
+	p := predicate.NewPredicateFuncs(filter)
+
+	return p
+}
+
+const (
+	mimirNamespace             = "mimir"
+	mimirInstance              = "mimir"
+	mimirAlertmanagerComponent = "alertmanager"
+)
+
+// NewAlertmanagerPodPredicate returns a predicate that filters only the Mimir Alertmanager pod.
+func NewAlertmanagerPodPredicate() predicate.Predicate {
+	filter := func(object client.Object) bool {
+		if object == nil {
+			return false
+		}
+
+		pod, ok := object.(*v1.Pod)
+		if !ok {
+			return false
+		}
+
+		if !pod.DeletionTimestamp.IsZero() {
+			return false
+		}
+
+		labels := pod.GetLabels()
+
+		ok = pod.GetNamespace() == mimirNamespace &&
+			labels != nil &&
+			labels["app.kubernetes.io/component"] == mimirAlertmanagerComponent &&
+			labels["app.kubernetes.io/instance"] == mimirInstance &&
+			isPodReady(pod)
+
+		return ok
+	}
+
+	p := predicate.NewPredicateFuncs(filter)
+
+	return p
+}

main.go

@@ -74,6 +74,8 @@ func main() {
 		"If set the metrics endpoint is served securely")
 	flag.BoolVar(&conf.EnableHTTP2, "enable-http2", false,
 		"If set, HTTP/2 will be enabled for the metrics and webhook servers")
+	flag.StringVar(&conf.Namespace, "namespace", "",
+		"The namespace where the observability-operator is running.")
 
 	// Management cluster configuration flags.
 	flag.StringVar(&conf.ManagementCluster.BaseDomain, "management-cluster-base-domain", "",
@@ -90,6 +92,10 @@ func main() {
 		"The region of the management cluster.")
 
 	// Monitoring configuration flags.
+	flag.StringVar(&conf.Monitoring.AlertmanagerSecretName, "alertmanager-secret-name", "",
+		"The name of the secret containing the Alertmanager configuration.")
+	flag.StringVar(&conf.Monitoring.AlertmanagerURL, "alertmanager-url", "",
+		"The URL of the Alertmanager API.")
 	flag.StringVar(&conf.Monitoring.MonitoringAgent, "monitoring-agent", commonmonitoring.MonitoringAgentAlloy,
 		fmt.Sprintf("select monitoring agent to use (%s or %s)", commonmonitoring.MonitoringAgentPrometheus, commonmonitoring.MonitoringAgentAlloy))
 	flag.BoolVar(&conf.Monitoring.Enabled, "monitoring-enabled", false,
@@ -109,15 +115,15 @@ func main() {
 	opts.BindFlags(flag.CommandLine)
 	flag.Parse()
 
+	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
+
 	// Load environment variables.
 	_, err := env.UnmarshalFromEnviron(&conf.Environment)
 	if err != nil {
 		setupLog.Error(err, "failed to unmarshal environment variables")
 		os.Exit(1)
 	}
 
-	ctrl.SetLogger(zap.New(zap.UseFlagOptions(&opts)))
-
 	// if the enable-http2 flag is false (the default), http/2 should be disabled
 	// due to its vulnerabilities. More specifically, disabling http/2 will
 	// prevent from being vulnerable to the HTTP/2 Stream Cancelation and
@@ -182,6 +188,13 @@ func main() {
 		setupLog.Error(err, "unable to setup controller", "controller", "GrafanaOrganizationReconciler")
 		os.Exit(1)
 	}
+
+	// Setup controller for Alertmanager
+	err = controller.SetupAlertmanagerReconciler(mgr, conf)
+	if err != nil {
+		setupLog.Error(err, "unable to setup controller", "controller", "AlertmanagerReconciler")
+		os.Exit(1)
+	}
 	//+kubebuilder:scaffold:builder
 
 	if err := mgr.AddHealthzCheck("healthz", healthz.Ping); err != nil {