[pull] master from cortexproject:master

CHANGELOG.md

@@ -1,6 +1,7 @@
 # Changelog
 
 ## master / unreleased
+* [CHANGE] Ruler: Add `cortex_ruler_rule_group_load_duration_seconds` and `cortex_ruler_rule_group_sync_duration_seconds` metrics. #5609
 * [CHANGE] Ruler: Add contextual info and query statistics to log
 * [FEATURE] Ruler: Add support for disabling rule groups. #5521
 * [FEATURE] Added the flag `-alertmanager.alerts-gc-interval` to configure alert manager alerts Garbage collection interval. #5550

go.mod

@@ -52,8 +52,8 @@ require (
 	github.com/spf13/afero v1.9.5
 	github.com/stretchr/testify v1.8.4
 	github.com/thanos-io/objstore v0.0.0-20230921130928-63a603e651ed
-	github.com/thanos-io/promql-engine v0.0.0-20231003153358-8605b6afba51
-	github.com/thanos-io/thanos v0.32.5-0.20231012105405-728bda9c387a
+	github.com/thanos-io/promql-engine v0.0.0-20231013104847-4517c0d5f591
+	github.com/thanos-io/thanos v0.32.5-0.20231020043209-42792d9d0e8c
 	github.com/uber/jaeger-client-go v2.30.0+incompatible
 	github.com/weaveworks/common v0.0.0-20221201103051-7c2720a9024d
 	go.etcd.io/etcd/api/v3 v3.5.9

go.sum

@@ -1210,10 +1210,10 @@ github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e h1:f1
 github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e/go.mod h1:jXcofnrSln/cLI6/dhlBxPQZEEQHVPCcFaH75M+nSzM=
 github.com/thanos-io/objstore v0.0.0-20230921130928-63a603e651ed h1:iWQdY3S6DpWjelVvKKSKgS7LeLkhK4VaEnQfphB9ZXA=
 github.com/thanos-io/objstore v0.0.0-20230921130928-63a603e651ed/go.mod h1:oJ82xgcBDzGJrEgUsjlTj6n01+ZWUMMUR8BlZzX5xDE=
-github.com/thanos-io/promql-engine v0.0.0-20231003153358-8605b6afba51 h1:Av62ac0O9wRbLI6xvtm51BBZnxHyEgLXV/YmiJpdogc=
-github.com/thanos-io/promql-engine v0.0.0-20231003153358-8605b6afba51/go.mod h1:vfXJv1JXNdLfHnjsHsLLJl5tyI7KblF76Wo5lZ9YC4Q=
-github.com/thanos-io/thanos v0.32.5-0.20231012105405-728bda9c387a h1:5kCKbUycSvsSCahiMwGyE5zDkZjXttl502ojQ0mSihk=
-github.com/thanos-io/thanos v0.32.5-0.20231012105405-728bda9c387a/go.mod h1:tqT2FQHiOF16empgE3vvZrA++fN9Cx0lwmxlMmBaVzA=
+github.com/thanos-io/promql-engine v0.0.0-20231013104847-4517c0d5f591 h1:6bZbFM+Mvy2kL8BeL8TJ5+5pV3sUR2PSLaZyw911rtQ=
+github.com/thanos-io/promql-engine v0.0.0-20231013104847-4517c0d5f591/go.mod h1:vfXJv1JXNdLfHnjsHsLLJl5tyI7KblF76Wo5lZ9YC4Q=
+github.com/thanos-io/thanos v0.32.5-0.20231020043209-42792d9d0e8c h1:M0XaUc+MummgpYJJkYhe2+bEO8H0KNmm/mIIpMTgyYc=
+github.com/thanos-io/thanos v0.32.5-0.20231020043209-42792d9d0e8c/go.mod h1:qHdYVTXuIZMXn2VX5KprDH5utFxR+29uqq4GIQ1OM6g=
 github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab h1:7ZR3hmisBWw77ZpO1/o86g+JV3VKlk3d48jopJxzTjU=
 github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab/go.mod h1:eheTFp954zcWZXCU8d0AT76ftsQOTo4DTqkN/h3k1MY=
 github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk=

pkg/ruler/ruler.go

@@ -254,8 +254,10 @@ type Ruler struct {
 	// Pool of clients used to connect to other ruler replicas.
 	clientsPool ClientsPool
 
-	ringCheckErrors prometheus.Counter
-	rulerSync       *prometheus.CounterVec
+	ringCheckErrors            prometheus.Counter
+	rulerSync                  *prometheus.CounterVec
+	ruleGroupStoreLoadDuration prometheus.Gauge
+	ruleGroupSyncDuration      prometheus.Gauge
 
 	allowedTenants *util.AllowedTenants
 
@@ -288,6 +290,16 @@ func newRuler(cfg Config, manager MultiTenantManager, reg prometheus.Registerer,
 			Name: "cortex_ruler_sync_rules_total",
 			Help: "Total number of times the ruler sync operation triggered.",
 		}, []string{"reason"}),
+
+		ruleGroupStoreLoadDuration: promauto.With(reg).NewGauge(prometheus.GaugeOpts{
+			Name: "cortex_ruler_rule_group_load_duration_seconds",
+			Help: "Time taken to load rule groups from storage",
+		}),
+
+		ruleGroupSyncDuration: promauto.With(reg).NewGauge(prometheus.GaugeOpts{
+			Name: "cortex_ruler_rule_group_sync_duration_seconds",
+			Help: "The duration in seconds required to sync and load rule groups from storage.",
+		}),
 	}
 
 	if len(cfg.EnabledTenants) > 0 {
@@ -512,20 +524,41 @@ func (r *Ruler) run(ctx context.Context) error {
 func (r *Ruler) syncRules(ctx context.Context, reason string) {
 	level.Debug(r.logger).Log("msg", "syncing rules", "reason", reason)
 	r.rulerSync.WithLabelValues(reason).Inc()
+	timer := prometheus.NewTimer(nil)
+
+	defer func() {
+		ruleGroupSyncDuration := timer.ObserveDuration().Seconds()
+		r.ruleGroupSyncDuration.Set(ruleGroupSyncDuration)
+	}()
+
+	loadedConfigs, err := r.loadRuleGroups(ctx)
+	if err != nil {
+		return
+	}
+
+	// This will also delete local group files for users that are no longer in 'configs' map.
+	r.manager.SyncRuleGroups(ctx, loadedConfigs)
+}
+
+func (r *Ruler) loadRuleGroups(ctx context.Context) (map[string]rulespb.RuleGroupList, error) {
+	timer := prometheus.NewTimer(nil)
+
+	defer func() {
+		storeLoadSeconds := timer.ObserveDuration().Seconds()
+		r.ruleGroupStoreLoadDuration.Set(storeLoadSeconds)
+	}()
 
 	configs, err := r.listRules(ctx)
 	if err != nil {
 		level.Error(r.logger).Log("msg", "unable to list rules", "err", err)
-		return
+		return nil, err
 	}
 
 	loadedConfigs, err := r.store.LoadRuleGroups(ctx, configs)
 	if err != nil {
 		level.Warn(r.logger).Log("msg", "failed to load some rules owned by this ruler", "count", len(configs)-len(loadedConfigs), "err", err)
 	}
-
-	// This will also delete local group files for users that are no longer in 'configs' map.
-	r.manager.SyncRuleGroups(ctx, loadedConfigs)
+	return loadedConfigs, nil
 }
 
 func (r *Ruler) listRules(ctx context.Context) (result map[string]rulespb.RuleGroupList, err error) {