Add tracing span to Ruler.getLocalRules() (#6515)

* Add tracing span to Ruler.getLocalRules() Signed-off-by: Marco Pracucci <[email protected]> * Fixed tests Signed-off-by: Marco Pracucci <[email protected]> * Add nolint rule Signed-off-by: Marco Pracucci <[email protected]> --------- Signed-off-by: Marco Pracucci <[email protected]>
grafana · Nov 1, 2023 · 2143ff2 · 2143ff2
1 parent b885d1a
commit 2143ff2
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 5 deletions.
diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go
@@ -41,6 +41,7 @@ import (
 	"github.com/grafana/mimir/pkg/storage/tsdb/bucketcache"
 	"github.com/grafana/mimir/pkg/util"
 	util_log "github.com/grafana/mimir/pkg/util/log"
+	"github.com/grafana/mimir/pkg/util/spanlogger"
 	"github.com/grafana/mimir/pkg/util/validation"
 )
 
@@ -991,7 +992,7 @@ func (r *Ruler) Rules(ctx context.Context, in *RulesRequest) (*RulesResponse, er
 		return nil, fmt.Errorf("no user id found in context")
 	}
 
-	groupDescs, err := r.getLocalRules(userID, *in)
+	groupDescs, err := r.getLocalRules(ctx, userID, *in)
 	if err != nil {
 		return nil, err
 	}
@@ -1019,8 +1020,15 @@ func (fs StringFilterSet) IsFiltered(val string) bool {
 	return !ok
 }
 
-func (r *Ruler) getLocalRules(userID string, req RulesRequest) ([]*GroupStateDesc, error) {
+func (r *Ruler) getLocalRules(ctx context.Context, userID string, req RulesRequest) ([]*GroupStateDesc, error) {
+	spanLog, _ := spanlogger.NewWithLogger(ctx, r.logger, "Ruler.getLocalRules")
+	defer spanLog.Finish()
+
+	// Get the rule groups from the manager. We track the time it takes because the manager needs to
+	// take a lock to run GetRules() and we want to make sure we're not hanging here.
+	getRulesStart := time.Now()
 	groups := r.manager.GetRules(userID)
+	spanLog.DebugLog("msg", "fetched rules from manager", "duration", time.Since(getRulesStart))
 
 	groupDescs := make([]*GroupStateDesc, 0, len(groups))
 	prefix := filepath.Join(r.cfg.RulePath, userID) + "/"
@@ -1087,7 +1095,7 @@ func (r *Ruler) getLocalRules(userID string, req RulesRequest) ([]*GroupStateDes
 				if !getAlertingRules {
 					continue
 				}
-				rule.ActiveAlerts()
+
 				alerts := []*AlertStateDesc{}
 				for _, a := range rule.ActiveAlerts() {
 					alerts = append(alerts, &AlertStateDesc{

diff --git a/pkg/ruler/ruler_test.go b/pkg/ruler/ruler_test.go
@@ -1218,7 +1218,7 @@ func TestRuler_NotifySyncRulesAsync_ShouldTriggerRulesSyncingAndCorrectlyHandleT
 		var actualRulersWithRuleGroups int
 
 		for _, ruler := range rulers {
-			actualRuleGroups, err := ruler.getLocalRules(userID, RulesRequest{Filter: AnyRule})
+			actualRuleGroups, err := ruler.getLocalRules(ctx, userID, RulesRequest{Filter: AnyRule})
 			require.NoError(t, err)
 			actualRuleGroupsCount += len(actualRuleGroups)
 
@@ -1256,7 +1256,7 @@ func TestRuler_NotifySyncRulesAsync_ShouldTriggerRulesSyncingAndCorrectlyHandleT
 		var actualRuleGroupsCountPerRuler []int
 
 		for _, ruler := range rulers {
-			actualRuleGroups, err := ruler.getLocalRules(userID, RulesRequest{Filter: AnyRule})
+			actualRuleGroups, err := ruler.getLocalRules(ctx, userID, RulesRequest{Filter: AnyRule})
 			require.NoError(t, err)
 			actualRuleGroupsCountPerRuler = append(actualRuleGroupsCountPerRuler, len(actualRuleGroups))
 		}