[pull] master from cortexproject:master

CHANGELOG.md

@@ -7,7 +7,8 @@
 * [FEATURE] AlertManager: Add support for Webex, Discord and Telegram Receiver. #5493
 * [FEATURE] Ingester: added `-admin-limit-message` to customize the message contained in limit errors.#5460
 * [FEATURE] AlertManager: Update version to v0.26.0 and bring in Microsoft Teams receiver. #5543
-* [FEATURE] Store Gateway: Support lazy expanded posting optimization. Added new flag `"blocks-storage.bucket-store.lazy-expanded-postings-enabled` and new metrics `cortex_bucket_store_lazy_expanded_postings_total`, `cortex_bucket_store_lazy_expanded_posting_size_bytes_total` and `cortex_bucket_store_lazy_expanded_posting_series_overfetched_size_bytes_total`. #5556.
+* [FEATURE] Store Gateway: Support lazy expanded posting optimization. Added new flag `blocks-storage.bucket-store.lazy-expanded-postings-enabled` and new metrics `cortex_bucket_store_lazy_expanded_postings_total`, `cortex_bucket_store_lazy_expanded_posting_size_bytes_total` and `cortex_bucket_store_lazy_expanded_posting_series_overfetched_size_bytes_total`. #5556.
+* [FEATURE] Store Gateway: Added new flag `blocks-storage.bucket-store.series-batch-size` to control how many series to fetch per batch in Store Gateway. #5582.
 * [CHANGE] AlertManager: include reason label in cortex_alertmanager_notifications_failed_total.#5409
 * [CHANGE] Query: Set CORS Origin headers for Query API #5388
 * [CHANGE] Updating prometheus/alertmanager from v0.25.0 to v0.25.1-0.20230505130626-263ca5c9438e. This includes the below changes. #5276
@@ -38,6 +39,7 @@
 * [FEATURE] Query Frontend: Add `-frontend.retry-on-too-many-outstanding-requests` to re-enqueue 429 requests if there are multiple query-schedulers available. #5496
 * [FEATURE] Store Gateway: Add `-blocks-storage.bucket-store.max-inflight-requests` for store gateways to reject further requests upon reaching the limit. #5553
 * [FEATURE] Store Gateway: Add `cortex_bucket_store_block_load_duration_seconds` histogram to track time to load blocks. #5580
+* [FEATURE] AlertManager: Add `cortex_alertmanager_dispatcher_aggregation_groups` and `cortex_alertmanager_dispatcher_alert_processing_duration_seconds` metrics for dispatcher. #5592
 * [ENHANCEMENT] Distributor/Ingester: Add span on push path #5319
 * [ENHANCEMENT] Support object storage backends for runtime configuration file. #5292
 * [ENHANCEMENT] Query Frontend: Reject subquery with too small step size. #5323
@@ -87,6 +89,7 @@
 * [BUGFIX] DDBKV: When no change detected in ring, retry the CAS until there is change. #5502
 * [BUGFIX] Fix bug on objstore when configured to use S3 fips endpoints. #5540
 * [BUGFIX] Ruler: Fix bug on ruler where a failure to load a single RuleGroup would prevent rulers to sync all RuleGroup. #5563
+* [BUGFIX] Store-Gateway and AlertManager: Add a `wait_instance_time_out` to WaitInstanceState context to avoid waiting forever. #5581
 
 ## 1.15.1 2023-04-26
 

docs/blocks-storage/querier.md

@@ -536,6 +536,11 @@ blocks_storage:
         # CLI flag: -blocks-storage.bucket-store.index-cache.inmemory.max-size-bytes
         [max_size_bytes: <int> | default = 1073741824]
 
+        # Selectively cache index item types. Supported values are Postings,
+        # ExpandedPostings and Series
+        # CLI flag: -blocks-storage.bucket-store.index-cache.inmemory.enabled-items
+        [enabled_items: <list of string> | default = []]
+
       memcached:
         # Comma separated list of memcached addresses. Supported prefixes are:
         # dns+ (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV
@@ -583,6 +588,11 @@ blocks_storage:
         # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.auto-discovery
         [auto_discovery: <boolean> | default = false]
 
+        # Selectively cache index item types. Supported values are Postings,
+        # ExpandedPostings and Series
+        # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.enabled-items
+        [enabled_items: <list of string> | default = []]
+
       redis:
         # Comma separated list of redis addresses. Supported prefixes are: dns+
         # (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query,
@@ -679,6 +689,11 @@ blocks_storage:
         # CLI flag: -blocks-storage.bucket-store.index-cache.redis.cache-size
         [cache_size: <int> | default = 0]
 
+        # Selectively cache index item types. Supported values are Postings,
+        # ExpandedPostings and Series
+        # CLI flag: -blocks-storage.bucket-store.index-cache.redis.enabled-items
+        [enabled_items: <list of string> | default = []]
+
     chunks_cache:
       # Backend for chunks cache, if not empty. Supported values: memcached.
       # CLI flag: -blocks-storage.bucket-store.chunks-cache.backend
@@ -1111,6 +1126,11 @@ blocks_storage:
     # CLI flag: -blocks-storage.bucket-store.lazy-expanded-postings-enabled
     [lazy_expanded_postings_enabled: <boolean> | default = false]
 
+    # Controls how many series to fetch per batch in Store Gateway. Default
+    # value is 10000.
+    # CLI flag: -blocks-storage.bucket-store.series-batch-size
+    [series_batch_size: <int> | default = 10000]
+
   tsdb:
     # Local directory to store TSDBs in the ingesters.
     # CLI flag: -blocks-storage.tsdb.dir

docs/blocks-storage/store-gateway.md

@@ -309,6 +309,10 @@ store_gateway:
     # CLI flag: -store-gateway.sharding-ring.wait-stability-max-duration
     [wait_stability_max_duration: <duration> | default = 5m]
 
+    # Timeout for waiting on store-gateway to become desired state in the ring.
+    # CLI flag: -store-gateway.sharding-ring.wait-instance-state-timeout
+    [wait_instance_state_timeout: <duration> | default = 10m]
+
     # The sleep seconds when store-gateway is shutting down. Need to be close to
     # or larger than KV Store information propagation delay
     # CLI flag: -store-gateway.sharding-ring.final-sleep
@@ -639,6 +643,11 @@ blocks_storage:
         # CLI flag: -blocks-storage.bucket-store.index-cache.inmemory.max-size-bytes
         [max_size_bytes: <int> | default = 1073741824]
 
+        # Selectively cache index item types. Supported values are Postings,
+        # ExpandedPostings and Series
+        # CLI flag: -blocks-storage.bucket-store.index-cache.inmemory.enabled-items
+        [enabled_items: <list of string> | default = []]
+
       memcached:
         # Comma separated list of memcached addresses. Supported prefixes are:
         # dns+ (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV
@@ -686,6 +695,11 @@ blocks_storage:
         # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.auto-discovery
         [auto_discovery: <boolean> | default = false]
 
+        # Selectively cache index item types. Supported values are Postings,
+        # ExpandedPostings and Series
+        # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.enabled-items
+        [enabled_items: <list of string> | default = []]
+
       redis:
         # Comma separated list of redis addresses. Supported prefixes are: dns+
         # (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query,
@@ -782,6 +796,11 @@ blocks_storage:
         # CLI flag: -blocks-storage.bucket-store.index-cache.redis.cache-size
         [cache_size: <int> | default = 0]
 
+        # Selectively cache index item types. Supported values are Postings,
+        # ExpandedPostings and Series
+        # CLI flag: -blocks-storage.bucket-store.index-cache.redis.enabled-items
+        [enabled_items: <list of string> | default = []]
+
     chunks_cache:
       # Backend for chunks cache, if not empty. Supported values: memcached.
       # CLI flag: -blocks-storage.bucket-store.chunks-cache.backend
@@ -1214,6 +1233,11 @@ blocks_storage:
     # CLI flag: -blocks-storage.bucket-store.lazy-expanded-postings-enabled
     [lazy_expanded_postings_enabled: <boolean> | default = false]
 
+    # Controls how many series to fetch per batch in Store Gateway. Default
+    # value is 10000.
+    # CLI flag: -blocks-storage.bucket-store.series-batch-size
+    [series_batch_size: <int> | default = 10000]
+
   tsdb:
     # Local directory to store TSDBs in the ingesters.
     # CLI flag: -blocks-storage.tsdb.dir

docs/configuration/config-file-reference.md

@@ -378,6 +378,10 @@ sharding_ring:
   # CLI flag: -alertmanager.sharding-ring.final-sleep
   [final_sleep: <duration> | default = 0s]
 
+  # Timeout for waiting on alertmanager to become desired state in the ring.
+  # CLI flag: -alertmanager.sharding-ring.wait-instance-state-timeout
+  [wait_instance_state_timeout: <duration> | default = 10m]
+
   # Name of network interface to read address from.
   # CLI flag: -alertmanager.sharding-ring.instance-interface-names
   [instance_interface_names: <list of string> | default = [eth0 en0]]
@@ -1079,6 +1083,11 @@ bucket_store:
       # CLI flag: -blocks-storage.bucket-store.index-cache.inmemory.max-size-bytes
       [max_size_bytes: <int> | default = 1073741824]
 
+      # Selectively cache index item types. Supported values are Postings,
+      # ExpandedPostings and Series
+      # CLI flag: -blocks-storage.bucket-store.index-cache.inmemory.enabled-items
+      [enabled_items: <list of string> | default = []]
+
     memcached:
       # Comma separated list of memcached addresses. Supported prefixes are:
       # dns+ (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query,
@@ -1126,6 +1135,11 @@ bucket_store:
       # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.auto-discovery
       [auto_discovery: <boolean> | default = false]
 
+      # Selectively cache index item types. Supported values are Postings,
+      # ExpandedPostings and Series
+      # CLI flag: -blocks-storage.bucket-store.index-cache.memcached.enabled-items
+      [enabled_items: <list of string> | default = []]
+
     redis:
       # Comma separated list of redis addresses. Supported prefixes are: dns+
       # (looked up as an A/AAAA query), dnssrv+ (looked up as a SRV query,
@@ -1222,6 +1236,11 @@ bucket_store:
       # CLI flag: -blocks-storage.bucket-store.index-cache.redis.cache-size
       [cache_size: <int> | default = 0]
 
+      # Selectively cache index item types. Supported values are Postings,
+      # ExpandedPostings and Series
+      # CLI flag: -blocks-storage.bucket-store.index-cache.redis.enabled-items
+      [enabled_items: <list of string> | default = []]
+
   chunks_cache:
     # Backend for chunks cache, if not empty. Supported values: memcached.
     # CLI flag: -blocks-storage.bucket-store.chunks-cache.backend
@@ -1653,6 +1672,11 @@ bucket_store:
   # CLI flag: -blocks-storage.bucket-store.lazy-expanded-postings-enabled
   [lazy_expanded_postings_enabled: <boolean> | default = false]
 
+  # Controls how many series to fetch per batch in Store Gateway. Default value
+  # is 10000.
+  # CLI flag: -blocks-storage.bucket-store.series-batch-size
+  [series_batch_size: <int> | default = 10000]
+
 tsdb:
   # Local directory to store TSDBs in the ingesters.
   # CLI flag: -blocks-storage.tsdb.dir
@@ -4867,6 +4891,10 @@ sharding_ring:
   # CLI flag: -store-gateway.sharding-ring.wait-stability-max-duration
   [wait_stability_max_duration: <duration> | default = 5m]
 
+  # Timeout for waiting on store-gateway to become desired state in the ring.
+  # CLI flag: -store-gateway.sharding-ring.wait-instance-state-timeout
+  [wait_instance_state_timeout: <duration> | default = 10m]
+
   # The sleep seconds when store-gateway is shutting down. Need to be close to
   # or larger than KV Store information propagation delay
   # CLI flag: -store-gateway.sharding-ring.final-sleep

go.mod

@@ -7,7 +7,7 @@ require (
 	github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137
 	github.com/alicebob/miniredis/v2 v2.30.4
 	github.com/armon/go-metrics v0.4.1
-	github.com/aws/aws-sdk-go v1.44.327
+	github.com/aws/aws-sdk-go v1.45.24
 	github.com/bradfitz/gomemcache v0.0.0-20190913173617-a41fca850d0b
 	github.com/cespare/xxhash v1.1.0
 	github.com/cortexproject/promqlsmith v0.0.0-20230502194647-ed3e43bb7a52
@@ -27,7 +27,7 @@ require (
 	github.com/gorilla/mux v1.8.0
 	github.com/grafana/regexp v0.0.0-20221122212121-6b5c0a4cb7fd
 	github.com/grpc-ecosystem/go-grpc-middleware v1.4.0
-	github.com/hashicorp/consul/api v1.22.0
+	github.com/hashicorp/consul/api v1.25.1
 	github.com/hashicorp/go-cleanhttp v0.5.2
 	github.com/hashicorp/go-sockaddr v1.0.2
 	github.com/hashicorp/memberlist v0.5.0
@@ -46,14 +46,14 @@ require (
 	github.com/prometheus/client_model v0.4.0
 	github.com/prometheus/common v0.44.0
 	// Prometheus maps version 2.x.y to tags v0.x.y.
-	github.com/prometheus/prometheus v0.46.1-0.20230818184859-4d8e380269da
+	github.com/prometheus/prometheus v0.47.2-0.20231009162353-f6d9c84fde6b
 	github.com/segmentio/fasthash v1.0.3
 	github.com/sony/gobreaker v0.5.0
 	github.com/spf13/afero v1.9.5
 	github.com/stretchr/testify v1.8.4
 	github.com/thanos-io/objstore v0.0.0-20230921130928-63a603e651ed
-	github.com/thanos-io/promql-engine v0.0.0-20230821193351-e1ae4275b96e
-	github.com/thanos-io/thanos v0.32.4-0.20230926060504-20d29008068f
+	github.com/thanos-io/promql-engine v0.0.0-20231003153358-8605b6afba51
+	github.com/thanos-io/thanos v0.32.5-0.20231010190130-dfe0bbff507b
 	github.com/uber/jaeger-client-go v2.30.0+incompatible
 	github.com/weaveworks/common v0.0.0-20221201103051-7c2720a9024d
 	go.etcd.io/etcd/api/v3 v3.5.9
@@ -198,6 +198,7 @@ require (
 	github.com/vimeo/galaxycache v0.0.0-20210323154928-b7e5d71c067a // indirect
 	github.com/weaveworks/promrus v1.2.0 // indirect
 	github.com/yuin/gopher-lua v1.1.0 // indirect
+	github.com/zhangyunhao116/umap v0.0.0-20221211160557-cb7705fafa39 // indirect
 	go.mongodb.org/mongo-driver v1.12.0 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/collector/pdata v1.0.0-rcv0014 // indirect
@@ -216,7 +217,7 @@ require (
 	go4.org/intern v0.0.0-20230525184215-6c62f75575cb // indirect
 	go4.org/unsafe/assume-no-moving-gc v0.0.0-20230525183740-e7c30c78aeb2 // indirect
 	golang.org/x/crypto v0.12.0 // indirect
-	golang.org/x/exp v0.0.0-20230713183714-613f0c0eb8a1 // indirect
+	golang.org/x/exp v0.0.0-20230801115018-d63ba01acd4b // indirect
 	golang.org/x/mod v0.12.0 // indirect
 	golang.org/x/oauth2 v0.11.0 // indirect
 	golang.org/x/sys v0.12.0 // indirect