Skip to content

Commit

Permalink
Revert "Parameterize absent pr name (#153)"
Browse files Browse the repository at this point in the history
This reverts commit 2cb900a.
  • Loading branch information
talal authored Sep 26, 2024
1 parent 2cb900a commit 4155a39
Show file tree
Hide file tree
Showing 3 changed files with 22 additions and 51 deletions.
56 changes: 16 additions & 40 deletions controllers/absence_prometheusrule.go
Original file line number Diff line number Diff line change
Expand Up @@ -21,43 +21,26 @@ import (
"reflect"
"sort"
"time"
"text/template"

monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
apierrors "k8s.io/apimachinery/pkg/api/errors"
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/types"
"sigs.k8s.io/controller-runtime/pkg/client"
"bytes"
"encoding/json"
)

const absencePromRuleNameSuffix = "-absent-metric-alert-rules"


// AbsencePrometheusRuleName returns the name of an AbsencePrometheusRule resource that
// holds the absence alert rules concerning a specific Prometheus server (e.g. openstack, kubernetes, etc.).
func AbsencePrometheusRuleName(prometheusRule monitoringv1.PrometheusRule, prometheusRuleString string) string {

t := template.Must(template.New("PrometheusRuleTemplate").Parse(prometheusRuleString))
b, err := json.Marshal(prometheusRule)

m := make(map[string]interface{})
err = json.Unmarshal(b, &m)

buf := &bytes.Buffer{}
err = t.Execute(buf, m)
if err != nil {
fmt.Println(err.Error())
return "default-absent-metrics"
}

return buf.String()
func AbsencePrometheusRuleName(promServer string) string {
return fmt.Sprintf("%s%s", promServer, absencePromRuleNameSuffix)
}

func (r *PrometheusRuleReconciler) newAbsencePrometheusRule(namespace, name string, promServer string) *monitoringv1.PrometheusRule {
func (r *PrometheusRuleReconciler) newAbsencePrometheusRule(namespace, promServer string) *monitoringv1.PrometheusRule {
return &monitoringv1.PrometheusRule{
ObjectMeta: metav1.ObjectMeta{
Name: name,
Name: AbsencePrometheusRuleName(promServer),
Namespace: namespace,
Labels: map[string]string{
// Add a label that identifies that this PrometheusRule resource is
Expand All @@ -72,12 +55,11 @@ func (r *PrometheusRuleReconciler) newAbsencePrometheusRule(namespace, name stri

func (r *PrometheusRuleReconciler) getExistingAbsencePrometheusRule(
ctx context.Context,
namespace, prometheusRuleString string,
rule monitoringv1.PrometheusRule,
namespace, promServer string,
) (*monitoringv1.PrometheusRule, error) {

var absencePromRule monitoringv1.PrometheusRule
nsName := types.NamespacedName{Namespace: namespace, Name: AbsencePrometheusRuleName(rule, prometheusRuleString)}
nsName := types.NamespacedName{Namespace: namespace, Name: AbsencePrometheusRuleName(promServer)}
if err := r.Get(ctx, nsName, &absencePromRule); err != nil {
return nil, err
}
Expand Down Expand Up @@ -152,20 +134,13 @@ func (r *PrometheusRuleReconciler) cleanUpOrphanedAbsenceAlertRules(
ctx context.Context,
promRule types.NamespacedName,
promServer string,
prometheusRuleString string,
) error {

var promRuleObj monitoringv1.PrometheusRule
if err := r.Get(ctx, promRule, &promRuleObj); err != nil {
return err
}


// Step 1: find the corresponding AbsencePrometheusRule that needs to be cleaned up.
var aPRToClean *monitoringv1.PrometheusRule
if promServer != "" {
var err error
if aPRToClean, err = r.getExistingAbsencePrometheusRule(ctx, promRule.Namespace, prometheusRuleString, promRuleObj); err != nil {
if aPRToClean, err = r.getExistingAbsencePrometheusRule(ctx, promRule.Namespace, promServer); err != nil {
return err
}
} else {
Expand Down Expand Up @@ -229,6 +204,9 @@ func (r *PrometheusRuleReconciler) cleanUpAbsencePrometheusRule(ctx context.Cont
// concerning Prometheus server.
var listOpts client.ListOptions
client.InNamespace(absencePromRule.GetNamespace()).ApplyToList(&listOpts)
client.MatchingLabels{
labelPrometheusServer: absencePromRule.Labels[labelPrometheusServer],
}.ApplyToList(&listOpts)
var promRules monitoringv1.PrometheusRuleList
if err := r.List(ctx, &promRules, &listOpts); err != nil {
return err
Expand Down Expand Up @@ -264,7 +242,7 @@ func (r *PrometheusRuleReconciler) cleanUpAbsencePrometheusRule(ctx context.Cont

// updateAbsenceAlertRules generates absence alert rules for the given PrometheusRule and
// adds them to the corresponding AbsencePrometheusRule.
func (r *PrometheusRuleReconciler) updateAbsenceAlertRules(ctx context.Context, promRule *monitoringv1.PrometheusRule, prometheusRuleString string) error {
func (r *PrometheusRuleReconciler) updateAbsenceAlertRules(ctx context.Context, promRule *monitoringv1.PrometheusRule) error {
promRuleName := promRule.GetName()
namespace := promRule.GetNamespace()
log := r.Log.WithValues("name", promRuleName, "namespace", namespace)
Expand All @@ -274,19 +252,17 @@ func (r *PrometheusRuleReconciler) updateAbsenceAlertRules(ctx context.Context,
promServer, ok := promRuleLabels["prometheus"]
if !ok {
// Normally this shouldn't happen but just in case that it does.
promServer = "default-prometheus"
// return errors.New("no 'prometheus' label found")
return errors.New("no 'prometheus' label found")
}

// Step 2: get the corresponding AbsencePrometheusRule if it exists.
existingAbsencePrometheusRule := false
absencePromRule, err := r.getExistingAbsencePrometheusRule(ctx, namespace, prometheusRuleString, *promRule)
absencePromRule, err := r.getExistingAbsencePrometheusRule(ctx, namespace, promServer)
switch {
case err == nil:
existingAbsencePrometheusRule = true
case apierrors.IsNotFound(err):
name := AbsencePrometheusRuleName(*promRule, prometheusRuleString)
absencePromRule = r.newAbsencePrometheusRule(namespace, name, promServer)
absencePromRule = r.newAbsencePrometheusRule(namespace, promServer)
default:
// This could have been caused by a temporary network failure, or any
// other transient reason.
Expand Down Expand Up @@ -319,7 +295,7 @@ func (r *PrometheusRuleReconciler) updateAbsenceAlertRules(ctx context.Context,
if len(absenceRuleGroups) == 0 {
if existingAbsencePrometheusRule {
key := types.NamespacedName{Namespace: namespace, Name: promRuleName}
return r.cleanUpOrphanedAbsenceAlertRules(ctx, key, promServer, prometheusRuleString)
return r.cleanUpOrphanedAbsenceAlertRules(ctx, key, promServer)
}
return nil
}
Expand Down
14 changes: 6 additions & 8 deletions controllers/prometheusrule_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -50,7 +50,6 @@ type PrometheusRuleReconciler struct {
// KeepLabel is a map of labels that will be retained from the original alert rule and
// passed on to its corresponding absent alert rule.
KeepLabel KeepLabel
PrometheusRuleString string
}

//+kubebuilder:rbac:groups=monitoring.coreos.com,resources=prometheusrules,verbs=get;list;watch;create;update;patch;delete
Expand All @@ -68,10 +67,10 @@ func (r *PrometheusRuleReconciler) Reconcile(ctx context.Context, req ctrl.Reque
err := r.Get(ctx, req.NamespacedName, &promRule)
switch {
case err == nil:
err = r.reconcileObject(ctx, req.NamespacedName, &promRule, r.PrometheusRuleString)
err = r.reconcileObject(ctx, req.NamespacedName, &promRule)
case apierrors.IsNotFound(err):
// Could not find object on the API server, maybe it has been deleted?
return r.handleObjectNotFound(ctx, req.NamespacedName, r.PrometheusRuleString)
return r.handleObjectNotFound(ctx, req.NamespacedName)
default:
// Handle err down below.
}
Expand Down Expand Up @@ -104,7 +103,7 @@ func (r *PrometheusRuleReconciler) SetupWithManager(mgr ctrl.Manager) error {

// handleObjectNotFound is a helper function for Reconcile(). It exists separately so that
// we can exit on error without making the `switch` in Reconcile() complex.
func (r *PrometheusRuleReconciler) handleObjectNotFound(ctx context.Context, key types.NamespacedName, prometheusRuleString string) (ctrl.Result, error) {
func (r *PrometheusRuleReconciler) handleObjectNotFound(ctx context.Context, key types.NamespacedName) (ctrl.Result, error) {
log := r.Log.WithValues("name", key.Name, "namespace", key.Namespace)

// Step 1: check if the object is a PrometheusRule or an AbsencePrometheusRule.
Expand All @@ -125,7 +124,7 @@ func (r *PrometheusRuleReconciler) handleObjectNotFound(ctx context.Context, key
// we wait until the next time when all AbsencePrometheusRules are requeued for
// processing (after the requeueInterval is elapsed).
log.V(logLevelDebug).Info("PrometheusRule no longer exists")
err := r.cleanUpOrphanedAbsenceAlertRules(ctx, key, "", prometheusRuleString)
err := r.cleanUpOrphanedAbsenceAlertRules(ctx, key, "")
if err != nil {
if !apierrors.IsNotFound(err) && !errors.Is(err, errCorrespondingAbsencePromRuleNotExists) {
log.Error(err, "could not clean up orphaned absence alert rules")
Expand All @@ -143,7 +142,6 @@ func (r *PrometheusRuleReconciler) reconcileObject(
ctx context.Context,
key types.NamespacedName,
obj *monitoringv1.PrometheusRule,
prometheusRuleString string,
) error {

log := r.Log.WithValues("name", key.Name, "namespace", key.Namespace)
Expand Down Expand Up @@ -178,7 +176,7 @@ func (r *PrometheusRuleReconciler) reconcileObject(
// elapsed).
if parseBool(l[labelOperatorDisable]) {
log.V(logLevelDebug).Info("operator disabled for this PrometheusRule")
err := r.cleanUpOrphanedAbsenceAlertRules(ctx, key, l[labelPrometheusServer], prometheusRuleString)
err := r.cleanUpOrphanedAbsenceAlertRules(ctx, key, l[labelPrometheusServer])
if err != nil {
if !apierrors.IsNotFound(err) && !errors.Is(err, errCorrespondingAbsencePromRuleNotExists) {
log.Error(err, "could not clean up orphaned absence alert rules")
Expand All @@ -191,7 +189,7 @@ func (r *PrometheusRuleReconciler) reconcileObject(
}

// Step 3: Generate the corresponding absence alert rules for this resource.
err := r.updateAbsenceAlertRules(ctx, obj, prometheusRuleString)
err := r.updateAbsenceAlertRules(ctx, obj)
if err == nil {
setReconcileGauge(key)
log.V(logLevelDebug).Info("successfully reconciled PrometheusRule")
Expand Down
3 changes: 0 additions & 3 deletions main.go
Original file line number Diff line number Diff line change
Expand Up @@ -59,15 +59,13 @@ func main() {
probeAddr string
enableLeaderElection bool
keepLabel labelsMap
prometheusRuleString string
)
bininfo.HandleVersionArgument()

flag.BoolVar(&debug, "debug", false, "Alias for '-zap-devel' flag.")
// Port `9659` has been allocated for absent metrics operator: https://github.com/prometheus/prometheus/wiki/Default-port-allocations
flag.StringVar(&metricsAddr, "metrics-bind-address", ":9659", "The address the metric endpoint binds to.")
flag.StringVar(&probeAddr, "health-probe-bind-address", ":8081", "The address the probe endpoint binds to.")
flag.StringVar(&prometheusRuleString, "rule", "{{ .metadata.labels.prometheus }}-absent-metrics", "Create new prometheusRules form this template string.")
flag.BoolVar(&enableLeaderElection, "leader-elect", false,
"Enable leader election for controller manager. "+
"Enabling this will ensure there is only one active controller manager.")
Expand Down Expand Up @@ -113,7 +111,6 @@ func main() {
Scheme: mgr.GetScheme(),
Log: ctrl.Log.WithName("controller").WithName("prometheusrule"),
KeepLabel: controllers.KeepLabel(keepLabel),
PrometheusRuleString: prometheusRuleString,
}).SetupWithManager(mgr); err != nil {
setupLog.Error(err, "unable to create controller", "controller", "PrometheusRule")
os.Exit(1)
Expand Down

0 comments on commit 4155a39

Please sign in to comment.