add context timeout for waitInstanceState call for alertmanager and s… (

#5581)

CHANGELOG.md

@@ -87,6 +87,7 @@
 * [BUGFIX] DDBKV: When no change detected in ring, retry the CAS until there is change. #5502
 * [BUGFIX] Fix bug on objstore when configured to use S3 fips endpoints. #5540
 * [BUGFIX] Ruler: Fix bug on ruler where a failure to load a single RuleGroup would prevent rulers to sync all RuleGroup. #5563
+* [BUGFIX] Store-Gateway and AlertManager: Add a `wait_instance_time_out` to WaitInstanceState context to avoid waiting forever. #5581
 
 ## 1.15.1 2023-04-26
 

docs/blocks-storage/store-gateway.md

@@ -309,6 +309,10 @@ store_gateway:
     # CLI flag: -store-gateway.sharding-ring.wait-stability-max-duration
     [wait_stability_max_duration: <duration> | default = 5m]
 
+    # Timeout for waiting on store-gateway to become desired state in the ring.
+    # CLI flag: -store-gateway.sharding-ring.wait-instance-state-timeout
+    [wait_instance_state_timeout: <duration> | default = 10m]
+
     # The sleep seconds when store-gateway is shutting down. Need to be close to
     # or larger than KV Store information propagation delay
     # CLI flag: -store-gateway.sharding-ring.final-sleep

docs/configuration/config-file-reference.md

@@ -378,6 +378,10 @@ sharding_ring:
   # CLI flag: -alertmanager.sharding-ring.final-sleep
   [final_sleep: <duration> | default = 0s]
 
+  # Timeout for waiting on alertmanager to become desired state in the ring.
+  # CLI flag: -alertmanager.sharding-ring.wait-instance-state-timeout
+  [wait_instance_state_timeout: <duration> | default = 10m]
+
   # Name of network interface to read address from.
   # CLI flag: -alertmanager.sharding-ring.instance-interface-names
   [instance_interface_names: <list of string> | default = [eth0 en0]]
@@ -4867,6 +4871,10 @@ sharding_ring:
   # CLI flag: -store-gateway.sharding-ring.wait-stability-max-duration
   [wait_stability_max_duration: <duration> | default = 5m]
 
+  # Timeout for waiting on store-gateway to become desired state in the ring.
+  # CLI flag: -store-gateway.sharding-ring.wait-instance-state-timeout
+  [wait_instance_state_timeout: <duration> | default = 10m]
+
   # The sleep seconds when store-gateway is shutting down. Need to be close to
   # or larger than KV Store information propagation delay
   # CLI flag: -store-gateway.sharding-ring.final-sleep

pkg/alertmanager/alertmanager_ring.go

@@ -49,7 +49,8 @@ type RingConfig struct {
 	ReplicationFactor    int           `yaml:"replication_factor"`
 	ZoneAwarenessEnabled bool          `yaml:"zone_awareness_enabled"`
 
-	FinalSleep time.Duration `yaml:"final_sleep"`
+	FinalSleep               time.Duration `yaml:"final_sleep"`
+	WaitInstanceStateTimeout time.Duration `yaml:"wait_instance_state_timeout"`
 
 	// Instance details
 	InstanceID             string   `yaml:"instance_id" doc:"hidden"`
@@ -94,6 +95,9 @@ func (cfg *RingConfig) RegisterFlags(f *flag.FlagSet) {
 	f.StringVar(&cfg.InstanceZone, rfprefix+"instance-availability-zone", "", "The availability zone where this instance is running. Required if zone-awareness is enabled.")
 
 	cfg.RingCheckPeriod = 5 * time.Second
+
+	// Timeout durations
+	f.DurationVar(&cfg.WaitInstanceStateTimeout, rfprefix+"wait-instance-state-timeout", 10*time.Minute, "Timeout for waiting on alertmanager to become desired state in the ring.")
 }
 
 // ToLifecyclerConfig returns a LifecyclerConfig based on the alertmanager

pkg/alertmanager/multitenant.go

@@ -486,7 +486,10 @@ func (am *MultitenantAlertmanager) starting(ctx context.Context) (err error) {
 
 		// We wait until the instance is in the JOINING state, once it does we know that tokens are assigned to this instance and we'll be ready to perform an initial sync of configs.
 		level.Info(am.logger).Log("msg", "waiting until alertmanager is JOINING in the ring")
-		if err = ring.WaitInstanceState(ctx, am.ring, am.ringLifecycler.GetInstanceID(), ring.JOINING); err != nil {
+		ctxWithTimeout, cancel := context.WithTimeout(ctx, am.cfg.ShardingRing.WaitInstanceStateTimeout)
+		defer cancel()
+		if err = ring.WaitInstanceState(ctxWithTimeout, am.ring, am.ringLifecycler.GetInstanceID(), ring.JOINING); err != nil {
+			level.Error(am.logger).Log("msg", "alertmanager failed to become JOINING in the ring", "err", err)
 			return err
 		}
 		level.Info(am.logger).Log("msg", "alertmanager is JOINING in the ring")
@@ -519,7 +522,10 @@ func (am *MultitenantAlertmanager) starting(ctx context.Context) (err error) {
 
 		// Wait until the ring client detected this instance in the ACTIVE state.
 		level.Info(am.logger).Log("msg", "waiting until alertmanager is ACTIVE in the ring")
-		if err := ring.WaitInstanceState(ctx, am.ring, am.ringLifecycler.GetInstanceID(), ring.ACTIVE); err != nil {
+		ctxWithTimeout, cancel := context.WithTimeout(ctx, am.cfg.ShardingRing.WaitInstanceStateTimeout)
+		defer cancel()
+		if err := ring.WaitInstanceState(ctxWithTimeout, am.ring, am.ringLifecycler.GetInstanceID(), ring.ACTIVE); err != nil {
+			level.Error(am.logger).Log("msg", "alertmanager failed to become ACTIVE in the ring", "err", err)
 			return err
 		}
 		level.Info(am.logger).Log("msg", "alertmanager is ACTIVE in the ring")

pkg/storegateway/gateway.go

@@ -244,7 +244,10 @@ func (g *StoreGateway) starting(ctx context.Context) (err error) {
 		// make sure that when we'll run the initial sync we already know  the tokens
 		// assigned to this instance.
 		level.Info(g.logger).Log("msg", "waiting until store-gateway is JOINING in the ring")
-		if err := ring.WaitInstanceState(ctx, g.ring, g.ringLifecycler.GetInstanceID(), ring.JOINING); err != nil {
+		ctxWithTimeout, cancel := context.WithTimeout(ctx, g.gatewayCfg.ShardingRing.WaitInstanceStateTimeout)
+		defer cancel()
+		if err := ring.WaitInstanceState(ctxWithTimeout, g.ring, g.ringLifecycler.GetInstanceID(), ring.JOINING); err != nil {
+			level.Error(g.logger).Log("msg", "store-gateway failed to become JOINING in the ring", "err", err)
 			return err
 		}
 		level.Info(g.logger).Log("msg", "store-gateway is JOINING in the ring")
@@ -285,7 +288,10 @@ func (g *StoreGateway) starting(ctx context.Context) (err error) {
 		// make sure that when we'll run the loop it won't be detected as a ring
 		// topology change.
 		level.Info(g.logger).Log("msg", "waiting until store-gateway is ACTIVE in the ring")
-		if err := ring.WaitInstanceState(ctx, g.ring, g.ringLifecycler.GetInstanceID(), ring.ACTIVE); err != nil {
+		ctxWithTimeout, cancel := context.WithTimeout(ctx, g.gatewayCfg.ShardingRing.WaitInstanceStateTimeout)
+		defer cancel()
+		if err := ring.WaitInstanceState(ctxWithTimeout, g.ring, g.ringLifecycler.GetInstanceID(), ring.ACTIVE); err != nil {
+			level.Error(g.logger).Log("msg", "store-gateway failed to become ACTIVE in the ring", "err", err)
 			return err
 		}
 		level.Info(g.logger).Log("msg", "store-gateway is ACTIVE in the ring")

pkg/storegateway/gateway_ring.go

@@ -72,6 +72,7 @@ type RingConfig struct {
 	// Wait ring stability.
 	WaitStabilityMinDuration time.Duration `yaml:"wait_stability_min_duration"`
 	WaitStabilityMaxDuration time.Duration `yaml:"wait_stability_max_duration"`
+	WaitInstanceStateTimeout time.Duration `yaml:"wait_instance_state_timeout"`
 
 	FinalSleep time.Duration `yaml:"final_sleep"`
 
@@ -123,6 +124,9 @@ func (cfg *RingConfig) RegisterFlags(f *flag.FlagSet) {
 
 	// Defaults for internal settings.
 	cfg.RingCheckPeriod = 5 * time.Second
+
+	// Timeout durations
+	f.DurationVar(&cfg.WaitInstanceStateTimeout, ringFlagsPrefix+"wait-instance-state-timeout", 10*time.Minute, "Timeout for waiting on store-gateway to become desired state in the ring.")
 }
 
 func (cfg *RingConfig) ToRingConfig() ring.Config {