From 7812330a0965385db3433aa64218b18be2a2b9af Mon Sep 17 00:00:00 2001 From: Ben Ye Date: Mon, 2 Oct 2023 08:30:57 -0700 Subject: [PATCH] Expose series batch size flag (#5582) * expose series batch size flag Signed-off-by: Ben Ye update docs Signed-off-by: Ben Ye * changelog Signed-off-by: Ben Ye --------- Signed-off-by: Ben Ye --- CHANGELOG.md | 3 ++- docs/blocks-storage/querier.md | 5 +++++ docs/blocks-storage/store-gateway.md | 5 +++++ docs/configuration/config-file-reference.md | 5 +++++ go.mod | 2 +- go.sum | 4 ++-- pkg/storage/tsdb/config.go | 4 ++++ pkg/storegateway/bucket_stores.go | 2 +- vendor/github.com/thanos-io/thanos/pkg/store/bucket.go | 2 +- .../github.com/thanos-io/thanos/pkg/store/cache/memcached.go | 4 ++-- vendor/modules.txt | 2 +- 11 files changed, 29 insertions(+), 9 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index ed4bc24eca..b91b4c0b6d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,7 +7,8 @@ * [FEATURE] AlertManager: Add support for Webex, Discord and Telegram Receiver. #5493 * [FEATURE] Ingester: added `-admin-limit-message` to customize the message contained in limit errors.#5460 * [FEATURE] AlertManager: Update version to v0.26.0 and bring in Microsoft Teams receiver. #5543 -* [FEATURE] Store Gateway: Support lazy expanded posting optimization. Added new flag `"blocks-storage.bucket-store.lazy-expanded-postings-enabled` and new metrics `cortex_bucket_store_lazy_expanded_postings_total`, `cortex_bucket_store_lazy_expanded_posting_size_bytes_total` and `cortex_bucket_store_lazy_expanded_posting_series_overfetched_size_bytes_total`. #5556. +* [FEATURE] Store Gateway: Support lazy expanded posting optimization. Added new flag `blocks-storage.bucket-store.lazy-expanded-postings-enabled` and new metrics `cortex_bucket_store_lazy_expanded_postings_total`, `cortex_bucket_store_lazy_expanded_posting_size_bytes_total` and `cortex_bucket_store_lazy_expanded_posting_series_overfetched_size_bytes_total`. #5556. +* [FEATURE] Store Gateway: Added new flag `blocks-storage.bucket-store.series-batch-size` to control how many series to fetch per batch in Store Gateway. #5582. * [CHANGE] AlertManager: include reason label in cortex_alertmanager_notifications_failed_total.#5409 * [CHANGE] Query: Set CORS Origin headers for Query API #5388 * [CHANGE] Updating prometheus/alertmanager from v0.25.0 to v0.25.1-0.20230505130626-263ca5c9438e. This includes the below changes. #5276 diff --git a/docs/blocks-storage/querier.md b/docs/blocks-storage/querier.md index 76ecf8a179..b7283df8aa 100644 --- a/docs/blocks-storage/querier.md +++ b/docs/blocks-storage/querier.md @@ -1111,6 +1111,11 @@ blocks_storage: # CLI flag: -blocks-storage.bucket-store.lazy-expanded-postings-enabled [lazy_expanded_postings_enabled: | default = false] + # Controls how many series to fetch per batch in Store Gateway. Default + # value is 10000. + # CLI flag: -blocks-storage.bucket-store.series-batch-size + [series_batch_size: | default = 10000] + tsdb: # Local directory to store TSDBs in the ingesters. # CLI flag: -blocks-storage.tsdb.dir diff --git a/docs/blocks-storage/store-gateway.md b/docs/blocks-storage/store-gateway.md index 77f7546761..82c30a121e 100644 --- a/docs/blocks-storage/store-gateway.md +++ b/docs/blocks-storage/store-gateway.md @@ -1218,6 +1218,11 @@ blocks_storage: # CLI flag: -blocks-storage.bucket-store.lazy-expanded-postings-enabled [lazy_expanded_postings_enabled: | default = false] + # Controls how many series to fetch per batch in Store Gateway. Default + # value is 10000. + # CLI flag: -blocks-storage.bucket-store.series-batch-size + [series_batch_size: | default = 10000] + tsdb: # Local directory to store TSDBs in the ingesters. # CLI flag: -blocks-storage.tsdb.dir diff --git a/docs/configuration/config-file-reference.md b/docs/configuration/config-file-reference.md index 6bb6d06ca7..71fce349af 100644 --- a/docs/configuration/config-file-reference.md +++ b/docs/configuration/config-file-reference.md @@ -1657,6 +1657,11 @@ bucket_store: # CLI flag: -blocks-storage.bucket-store.lazy-expanded-postings-enabled [lazy_expanded_postings_enabled: | default = false] + # Controls how many series to fetch per batch in Store Gateway. Default value + # is 10000. + # CLI flag: -blocks-storage.bucket-store.series-batch-size + [series_batch_size: | default = 10000] + tsdb: # Local directory to store TSDBs in the ingesters. # CLI flag: -blocks-storage.tsdb.dir diff --git a/go.mod b/go.mod index db3e9e7014..9716fb4769 100644 --- a/go.mod +++ b/go.mod @@ -53,7 +53,7 @@ require ( github.com/stretchr/testify v1.8.4 github.com/thanos-io/objstore v0.0.0-20230921130928-63a603e651ed github.com/thanos-io/promql-engine v0.0.0-20230821193351-e1ae4275b96e - github.com/thanos-io/thanos v0.32.4-0.20230926060504-20d29008068f + github.com/thanos-io/thanos v0.32.4-0.20231001083734-531cdb1e8ec3 github.com/uber/jaeger-client-go v2.30.0+incompatible github.com/weaveworks/common v0.0.0-20221201103051-7c2720a9024d go.etcd.io/etcd/api/v3 v3.5.9 diff --git a/go.sum b/go.sum index f77d8bb131..d901a4ad3a 100644 --- a/go.sum +++ b/go.sum @@ -1212,8 +1212,8 @@ github.com/thanos-io/objstore v0.0.0-20230921130928-63a603e651ed h1:iWQdY3S6DpWj github.com/thanos-io/objstore v0.0.0-20230921130928-63a603e651ed/go.mod h1:oJ82xgcBDzGJrEgUsjlTj6n01+ZWUMMUR8BlZzX5xDE= github.com/thanos-io/promql-engine v0.0.0-20230821193351-e1ae4275b96e h1:kwsFCU8eSkZehbrAN3nXPw5RdMHi/Bok/y8l2C4M+gk= github.com/thanos-io/promql-engine v0.0.0-20230821193351-e1ae4275b96e/go.mod h1:+T/ZYNCGybT6eTsGGvVtGb63nT1cvUmH6MjqRrcQoKw= -github.com/thanos-io/thanos v0.32.4-0.20230926060504-20d29008068f h1:OdZZLgF2eYIiad7h4WeUPkew7Uq6F9vFPg3aDZfMQLY= -github.com/thanos-io/thanos v0.32.4-0.20230926060504-20d29008068f/go.mod h1:Px5Boq60s+2WwR+V4v4oxgmxfw9WHrwMwjRou6pkUNw= +github.com/thanos-io/thanos v0.32.4-0.20231001083734-531cdb1e8ec3 h1:ekD3P1XF0Hlg/u7rSNqdyLhwYE4W4RGfkMDudtepRL8= +github.com/thanos-io/thanos v0.32.4-0.20231001083734-531cdb1e8ec3/go.mod h1:Px5Boq60s+2WwR+V4v4oxgmxfw9WHrwMwjRou6pkUNw= github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab h1:7ZR3hmisBWw77ZpO1/o86g+JV3VKlk3d48jopJxzTjU= github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab/go.mod h1:eheTFp954zcWZXCU8d0AT76ftsQOTo4DTqkN/h3k1MY= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= diff --git a/pkg/storage/tsdb/config.go b/pkg/storage/tsdb/config.go index e8af5e1c41..ebbd0b2886 100644 --- a/pkg/storage/tsdb/config.go +++ b/pkg/storage/tsdb/config.go @@ -280,6 +280,9 @@ type BucketStoreConfig struct { // On the contrary, smaller value will increase baseline memory usage, but improve latency slightly. // 1 will keep all in memory. Default value is the same as in Prometheus which gives a good balance. PostingOffsetsInMemSampling int `yaml:"postings_offsets_in_mem_sampling" doc:"hidden"` + + // Controls how many series to fetch per batch in Store Gateway. Default value is 10000. + SeriesBatchSize int `yaml:"series_batch_size"` } // RegisterFlags registers the BucketStore flags @@ -311,6 +314,7 @@ func (cfg *BucketStoreConfig) RegisterFlags(f *flag.FlagSet) { f.Uint64Var(&cfg.EstimatedMaxSeriesSizeBytes, "blocks-storage.bucket-store.estimated-max-series-size-bytes", store.EstimatedMaxSeriesSize, "Estimated max series size in bytes. Setting a large value might result in over fetching data while a small value might result in data refetch. Default value is 64KB.") f.Uint64Var(&cfg.EstimatedMaxChunkSizeBytes, "blocks-storage.bucket-store.estimated-max-chunk-size-bytes", store.EstimatedMaxChunkSize, "Estimated max chunk size in bytes. Setting a large value might result in over fetching data while a small value might result in data refetch. Default value is 16KiB.") f.BoolVar(&cfg.LazyExpandedPostingsEnabled, "blocks-storage.bucket-store.lazy-expanded-postings-enabled", false, "If true, Store Gateway will estimate postings size and try to lazily expand postings if it downloads less data than expanding all postings.") + f.IntVar(&cfg.SeriesBatchSize, "blocks-storage.bucket-store.series-batch-size", store.SeriesBatchSize, "Controls how many series to fetch per batch in Store Gateway. Default value is 10000.") } // Validate the config. diff --git a/pkg/storegateway/bucket_stores.go b/pkg/storegateway/bucket_stores.go index d7c709c4ec..e5630a2c1a 100644 --- a/pkg/storegateway/bucket_stores.go +++ b/pkg/storegateway/bucket_stores.go @@ -572,7 +572,7 @@ func (u *BucketStores) getOrCreateStore(userID string) (*store.BucketStore, erro store.WithIndexCache(u.indexCache), store.WithQueryGate(u.queryGate), store.WithChunkPool(u.chunksPool), - store.WithSeriesBatchSize(store.SeriesBatchSize), + store.WithSeriesBatchSize(u.cfg.BucketStore.SeriesBatchSize), store.WithBlockEstimatedMaxChunkFunc(func(m thanos_metadata.Meta) uint64 { if m.Thanos.IndexStats.ChunkMaxSize > 0 && uint64(m.Thanos.IndexStats.ChunkMaxSize) < u.cfg.BucketStore.EstimatedMaxChunkSizeBytes { diff --git a/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go b/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go index bc1507a367..2e40d010a3 100644 --- a/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go +++ b/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go @@ -1124,7 +1124,7 @@ func (b *blockSeriesClient) Recv() (*storepb.SeriesResponse, error) { func (b *blockSeriesClient) nextBatch() error { start := b.i - end := start + SeriesBatchSize + end := start + uint64(b.batchSize) if end > uint64(len(b.lazyPostings.postings)) { end = uint64(len(b.lazyPostings.postings)) } diff --git a/vendor/github.com/thanos-io/thanos/pkg/store/cache/memcached.go b/vendor/github.com/thanos-io/thanos/pkg/store/cache/memcached.go index a3dbce9940..104b936e8c 100644 --- a/vendor/github.com/thanos-io/thanos/pkg/store/cache/memcached.go +++ b/vendor/github.com/thanos-io/thanos/pkg/store/cache/memcached.go @@ -148,7 +148,7 @@ func (c *RemoteIndexCache) StoreExpandedPostings(blockID ulid.ULID, keys []*labe // and returns a map containing cache hits, along with a list of missing keys. // In case of error, it logs and return an empty cache hits map. func (c *RemoteIndexCache) FetchExpandedPostings(ctx context.Context, blockID ulid.ULID, lbls []*labels.Matcher) ([]byte, bool) { - timer := prometheus.NewTimer(c.postingsFetchDuration) + timer := prometheus.NewTimer(c.expandedPostingsFetchDuration) defer timer.ObserveDuration() key := cacheKey{blockID.String(), cacheKeyExpandedPostings(labelMatchersToString(lbls)), c.compressionScheme}.string() @@ -182,7 +182,7 @@ func (c *RemoteIndexCache) StoreSeries(blockID ulid.ULID, id storage.SeriesRef, // and returns a map containing cache hits, along with a list of missing IDs. // In case of error, it logs and return an empty cache hits map. func (c *RemoteIndexCache) FetchMultiSeries(ctx context.Context, blockID ulid.ULID, ids []storage.SeriesRef) (hits map[storage.SeriesRef][]byte, misses []storage.SeriesRef) { - timer := prometheus.NewTimer(c.postingsFetchDuration) + timer := prometheus.NewTimer(c.seriesFetchDuration) defer timer.ObserveDuration() keys := make([]string, 0, len(ids)) diff --git a/vendor/modules.txt b/vendor/modules.txt index d3cc541a84..b74cc59f1f 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -902,7 +902,7 @@ github.com/thanos-io/promql-engine/logicalplan github.com/thanos-io/promql-engine/parser github.com/thanos-io/promql-engine/query github.com/thanos-io/promql-engine/worker -# github.com/thanos-io/thanos v0.32.4-0.20230926060504-20d29008068f +# github.com/thanos-io/thanos v0.32.4-0.20231001083734-531cdb1e8ec3 ## explicit; go 1.18 github.com/thanos-io/thanos/pkg/block github.com/thanos-io/thanos/pkg/block/indexheader