diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go
index 29799acc45b..7476eaadbed 100644
--- a/pkg/ruler/ruler.go
+++ b/pkg/ruler/ruler.go
@@ -878,7 +878,7 @@ func (r *Ruler) getShardedRules(ctx context.Context, userID string, rulesRequest
 		ring = r.ring.ShuffleShard(userID, shardSize)
 	}
 
-	rulers, err := ring.GetReplicationSetForOperation(RingOp)
+	rulers, err := ring.GetReplicationSetForOperation(ListRuleRingOp)
 	if err != nil {
 		return nil, err
 	}
diff --git a/pkg/ruler/ruler_ring.go b/pkg/ruler/ruler_ring.go
index 1f5422f060a..5727fb6b249 100644
--- a/pkg/ruler/ruler_ring.go
+++ b/pkg/ruler/ruler_ring.go
@@ -26,6 +26,13 @@ var RingOp = ring.NewOp([]ring.InstanceState{ring.ACTIVE}, func(s ring.InstanceS
 	return s != ring.ACTIVE
 })
 
+// ListRuleRingOp is the operation used for getting rule groups from rulers.
+var ListRuleRingOp = ring.NewOp([]ring.InstanceState{ring.ACTIVE}, func(s ring.InstanceState) bool {
+	// Although LEAVING ruler does not get any rule groups. If it is excluded, list rule will fail because not enough healthy instance.
+	// So we still try to read from the LEAVING ruler, although it might return empty result
+	return s != ring.ACTIVE && s != ring.LEAVING
+})
+
 // RingConfig masks the ring lifecycler config which contains
 // many options not really required by the rulers ring. This config
 // is used to strip down the config to the minimum, and avoid confusion