From 63cd7a179afd69464fa21f271a4785799a808ff2 Mon Sep 17 00:00:00 2001 From: zhuoyuan-liu <117184318+zhuoyuan-liu@users.noreply.github.com> Date: Wed, 13 Sep 2023 15:26:42 +0200 Subject: [PATCH] Ruler: Add alert source template (#6308) * Add alert source template in rule Signed-off-by: Zhuoyuan Liu * Validate template in start phase Signed-off-by: Zhuoyuan Liu * Move the start check to runrule Signed-off-by: Zhuoyuan Liu * move the flag to config.go Signed-off-by: Zhuoyuan Liu * Updates the docs Signed-off-by: Zhuoyuan Liu * Add test for validateTemplate Signed-off-by: Zhuoyuan Liu * Add new test case Signed-off-by: Zhuoyuan Liu * Remove unnecessary variable Signed-off-by: Zhuoyuan Liu * Add changelogs Signed-off-by: Zhuoyuan Liu * Update CHANGELOG.md Signed-off-by: Matej Gera <38492574+matej-g@users.noreply.github.com> --------- Signed-off-by: Zhuoyuan Liu Signed-off-by: Matej Gera <38492574+matej-g@users.noreply.github.com> Co-authored-by: Matej Gera <38492574+matej-g@users.noreply.github.com> --- CHANGELOG.md | 1 + cmd/thanos/config.go | 3 +++ cmd/thanos/rule.go | 48 +++++++++++++++++++++++++++++++++-- cmd/thanos/rule_test.go | 56 +++++++++++++++++++++++++++++++++++++++++ docs/components/rule.md | 3 +++ 5 files changed, 109 insertions(+), 2 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3f194961c3d..8148cf176ef 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -18,6 +18,7 @@ We use *breaking :warning:* to mark changes that are not backward compatible (re ### Added - [#6605](https://github.com/thanos-io/thanos/pull/6605) Query Frontend: Support vertical sharding binary expression with metric name when no matching labels specified. +- [#6308](https://github.com/thanos-io/thanos/pull/6308) Ruler: Support configuration flag that allows customizing template for alert message. ### Changed diff --git a/cmd/thanos/config.go b/cmd/thanos/config.go index 05e510d8b84..51d0eed06f3 100644 --- a/cmd/thanos/config.go +++ b/cmd/thanos/config.go @@ -220,6 +220,7 @@ type alertMgrConfig struct { alertExcludeLabels []string alertQueryURL *string alertRelabelConfigPath *extflag.PathOrContent + alertSourceTemplate *string } func (ac *alertMgrConfig) registerFlag(cmd extflag.FlagClause) *alertMgrConfig { @@ -234,5 +235,7 @@ func (ac *alertMgrConfig) registerFlag(cmd extflag.FlagClause) *alertMgrConfig { cmd.Flag("alert.label-drop", "Labels by name to drop before sending to alertmanager. This allows alert to be deduplicated on replica label (repeated). Similar Prometheus alert relabelling"). StringsVar(&ac.alertExcludeLabels) ac.alertRelabelConfigPath = extflag.RegisterPathOrContent(cmd, "alert.relabel-config", "YAML file that contains alert relabelling configuration.", extflag.WithEnvSubstitution()) + ac.alertSourceTemplate = cmd.Flag("alert.query-template", "Template to use in alerts source field. Need only include {{.Expr}} parameter").Default("/graph?g0.expr={{.Expr}}&g0.tab=1").String() + return ac } diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index ddbb468de64..2d519185a3f 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -4,7 +4,10 @@ package main import ( + "bytes" "context" + "fmt" + "html/template" "math/rand" "net/http" "net/url" @@ -38,7 +41,6 @@ import ( "github.com/prometheus/prometheus/tsdb" "github.com/prometheus/prometheus/tsdb/agent" "github.com/prometheus/prometheus/tsdb/wlog" - "github.com/prometheus/prometheus/util/strutil" "github.com/thanos-io/objstore" "github.com/thanos-io/objstore/client" @@ -101,6 +103,10 @@ type ruleConfig struct { storeRateLimits store.SeriesSelectLimits } +type Expression struct { + Expr string +} + func (rc *ruleConfig) registerFlag(cmd extkingpin.FlagClause) { rc.http.registerFlag(cmd) rc.grpc.registerFlag(cmd) @@ -329,6 +335,10 @@ func runRule( } } + if err := validateTemplate(*conf.alertmgr.alertSourceTemplate); err != nil { + return errors.Wrap(err, "invalid alert source template") + } + queryProvider := dns.NewProvider( logger, extprom.WrapRegistererWithPrefix("thanos_rule_query_apis_", reg), @@ -492,11 +502,15 @@ func runRule( if alrt.State == rules.StatePending { continue } + expressionURL, err := tableLinkForExpression(*conf.alertmgr.alertSourceTemplate, expr) + if err != nil { + level.Warn(logger).Log("msg", "failed to generate link for expression", "expr", expr, "err", err) + } a := ¬ifier.Alert{ StartsAt: alrt.FiredAt, Labels: alrt.Labels, Annotations: alrt.Annotations, - GeneratorURL: conf.alertQueryURL.String() + strutil.TableLinkForExpression(expr), + GeneratorURL: conf.alertQueryURL.String() + expressionURL, } if !alrt.ResolvedAt.IsZero() { a.EndsAt = alrt.ResolvedAt @@ -934,3 +948,33 @@ func reloadRules(logger log.Logger, } return errs.Err() } + +func tableLinkForExpression(tmpl string, expr string) (string, error) { + // template example: "/graph?g0.expr={{.Expr}}&g0.tab=1" + escapedExpression := url.QueryEscape(expr) + + escapedExpr := Expression{Expr: escapedExpression} + t, err := template.New("url").Parse(tmpl) + if err != nil { + return "", errors.Wrap(err, "failed to parse template") + } + + var buf bytes.Buffer + if err := t.Execute(&buf, escapedExpr); err != nil { + return "", errors.Wrap(err, "failed to execute template") + } + return buf.String(), nil +} + +func validateTemplate(tmplStr string) error { + tmpl, err := template.New("test").Parse(tmplStr) + if err != nil { + return fmt.Errorf("failed to parse the template: %w", err) + } + var buf bytes.Buffer + err = tmpl.Execute(&buf, Expression{Expr: "test_expr"}) + if err != nil { + return fmt.Errorf("failed to execute the template: %w", err) + } + return nil +} diff --git a/cmd/thanos/rule_test.go b/cmd/thanos/rule_test.go index 5703c7b3ccf..3ba4d65b250 100644 --- a/cmd/thanos/rule_test.go +++ b/cmd/thanos/rule_test.go @@ -48,3 +48,59 @@ func Test_parseFlagLabels(t *testing.T) { testutil.Equals(t, err != nil, td.expectErr) } } + +func Test_validateTemplate(t *testing.T) { + tData := []struct { + template string + expectErr bool + }{ + { + template: `/graph?g0.expr={{.Expr}}&g0.tab=1`, + expectErr: false, + }, + { + template: `/graph?g0.expr={{.Expression}}&g0.tab=1`, + expectErr: true, + }, + { + template: `another template includes {{.Expr}}`, + expectErr: false, + }, + } + for _, td := range tData { + err := validateTemplate(td.template) + testutil.Equals(t, err != nil, td.expectErr) + } +} + +func Test_tableLinkForExpression(t *testing.T) { + tData := []struct { + template string + expr string + expectStr string + expectErr bool + }{ + { + template: `/graph?g0.expr={{.Expr}}&g0.tab=1`, + expr: `up{app="foo"}`, + expectStr: `/graph?g0.expr=up%7Bapp%3D%22foo%22%7D&g0.tab=1`, + expectErr: false, + }, + { + template: `/graph?g0.expr={{.Expression}}&g0.tab=1`, + expr: "test_expr", + expectErr: true, + }, + { + template: `another template includes {{.Expr}}`, + expr: "test_expr", + expectStr: `another template includes test_expr`, + expectErr: false, + }, + } + for _, td := range tData { + resStr, err := tableLinkForExpression(td.template, td.expr) + testutil.Equals(t, err != nil, td.expectErr) + testutil.Equals(t, resStr, td.expectStr) + } +} diff --git a/docs/components/rule.md b/docs/components/rule.md index 25c98463aff..a2dbdc72ec6 100644 --- a/docs/components/rule.md +++ b/docs/components/rule.md @@ -267,6 +267,9 @@ Flags: to alertmanager. This allows alert to be deduplicated on replica label (repeated). Similar Prometheus alert relabelling + --alert.query-template="/graph?g0.expr={{.Expr}}&g0.tab=1" + Template to use in alerts source field. + Need only include {{.Expr}} parameter --alert.query-url=ALERT.QUERY-URL The external Thanos Query URL that would be set in all alerts 'Source' field