diff --git a/CHANGELOG.md b/CHANGELOG.md index 2085470f50..4ccb5b981c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,7 @@ # Changelog ## master / unreleased +* [CHANGE] Ruler: Add `cortex_ruler_rule_group_load_duration_seconds` and `cortex_ruler_rule_group_sync_duration_seconds` metrics. #5609 * [CHANGE] Ruler: Add contextual info and query statistics to log * [FEATURE] Ruler: Add support for disabling rule groups. #5521 * [FEATURE] Added the flag `-alertmanager.alerts-gc-interval` to configure alert manager alerts Garbage collection interval. #5550 diff --git a/go.mod b/go.mod index d58988357c..11f225d71f 100644 --- a/go.mod +++ b/go.mod @@ -52,8 +52,8 @@ require ( github.com/spf13/afero v1.9.5 github.com/stretchr/testify v1.8.4 github.com/thanos-io/objstore v0.0.0-20230921130928-63a603e651ed - github.com/thanos-io/promql-engine v0.0.0-20231003153358-8605b6afba51 - github.com/thanos-io/thanos v0.32.5-0.20231012105405-728bda9c387a + github.com/thanos-io/promql-engine v0.0.0-20231013104847-4517c0d5f591 + github.com/thanos-io/thanos v0.32.5-0.20231020043209-42792d9d0e8c github.com/uber/jaeger-client-go v2.30.0+incompatible github.com/weaveworks/common v0.0.0-20221201103051-7c2720a9024d go.etcd.io/etcd/api/v3 v3.5.9 diff --git a/go.sum b/go.sum index ada7a63cf6..3297f27b1a 100644 --- a/go.sum +++ b/go.sum @@ -1210,10 +1210,10 @@ github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e h1:f1 github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e/go.mod h1:jXcofnrSln/cLI6/dhlBxPQZEEQHVPCcFaH75M+nSzM= github.com/thanos-io/objstore v0.0.0-20230921130928-63a603e651ed h1:iWQdY3S6DpWjelVvKKSKgS7LeLkhK4VaEnQfphB9ZXA= github.com/thanos-io/objstore v0.0.0-20230921130928-63a603e651ed/go.mod h1:oJ82xgcBDzGJrEgUsjlTj6n01+ZWUMMUR8BlZzX5xDE= -github.com/thanos-io/promql-engine v0.0.0-20231003153358-8605b6afba51 h1:Av62ac0O9wRbLI6xvtm51BBZnxHyEgLXV/YmiJpdogc= -github.com/thanos-io/promql-engine v0.0.0-20231003153358-8605b6afba51/go.mod h1:vfXJv1JXNdLfHnjsHsLLJl5tyI7KblF76Wo5lZ9YC4Q= -github.com/thanos-io/thanos v0.32.5-0.20231012105405-728bda9c387a h1:5kCKbUycSvsSCahiMwGyE5zDkZjXttl502ojQ0mSihk= -github.com/thanos-io/thanos v0.32.5-0.20231012105405-728bda9c387a/go.mod h1:tqT2FQHiOF16empgE3vvZrA++fN9Cx0lwmxlMmBaVzA= +github.com/thanos-io/promql-engine v0.0.0-20231013104847-4517c0d5f591 h1:6bZbFM+Mvy2kL8BeL8TJ5+5pV3sUR2PSLaZyw911rtQ= +github.com/thanos-io/promql-engine v0.0.0-20231013104847-4517c0d5f591/go.mod h1:vfXJv1JXNdLfHnjsHsLLJl5tyI7KblF76Wo5lZ9YC4Q= +github.com/thanos-io/thanos v0.32.5-0.20231020043209-42792d9d0e8c h1:M0XaUc+MummgpYJJkYhe2+bEO8H0KNmm/mIIpMTgyYc= +github.com/thanos-io/thanos v0.32.5-0.20231020043209-42792d9d0e8c/go.mod h1:qHdYVTXuIZMXn2VX5KprDH5utFxR+29uqq4GIQ1OM6g= github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab h1:7ZR3hmisBWw77ZpO1/o86g+JV3VKlk3d48jopJxzTjU= github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab/go.mod h1:eheTFp954zcWZXCU8d0AT76ftsQOTo4DTqkN/h3k1MY= github.com/tidwall/pretty v1.0.0/go.mod h1:XNkn88O1ChpSDQmQeStsy+sBenx6DDtFZJxhVysOjyk= diff --git a/pkg/ruler/ruler.go b/pkg/ruler/ruler.go index b676913b78..3cb56cb326 100644 --- a/pkg/ruler/ruler.go +++ b/pkg/ruler/ruler.go @@ -254,8 +254,10 @@ type Ruler struct { // Pool of clients used to connect to other ruler replicas. clientsPool ClientsPool - ringCheckErrors prometheus.Counter - rulerSync *prometheus.CounterVec + ringCheckErrors prometheus.Counter + rulerSync *prometheus.CounterVec + ruleGroupStoreLoadDuration prometheus.Gauge + ruleGroupSyncDuration prometheus.Gauge allowedTenants *util.AllowedTenants @@ -288,6 +290,16 @@ func newRuler(cfg Config, manager MultiTenantManager, reg prometheus.Registerer, Name: "cortex_ruler_sync_rules_total", Help: "Total number of times the ruler sync operation triggered.", }, []string{"reason"}), + + ruleGroupStoreLoadDuration: promauto.With(reg).NewGauge(prometheus.GaugeOpts{ + Name: "cortex_ruler_rule_group_load_duration_seconds", + Help: "Time taken to load rule groups from storage", + }), + + ruleGroupSyncDuration: promauto.With(reg).NewGauge(prometheus.GaugeOpts{ + Name: "cortex_ruler_rule_group_sync_duration_seconds", + Help: "The duration in seconds required to sync and load rule groups from storage.", + }), } if len(cfg.EnabledTenants) > 0 { @@ -512,20 +524,41 @@ func (r *Ruler) run(ctx context.Context) error { func (r *Ruler) syncRules(ctx context.Context, reason string) { level.Debug(r.logger).Log("msg", "syncing rules", "reason", reason) r.rulerSync.WithLabelValues(reason).Inc() + timer := prometheus.NewTimer(nil) + + defer func() { + ruleGroupSyncDuration := timer.ObserveDuration().Seconds() + r.ruleGroupSyncDuration.Set(ruleGroupSyncDuration) + }() + + loadedConfigs, err := r.loadRuleGroups(ctx) + if err != nil { + return + } + + // This will also delete local group files for users that are no longer in 'configs' map. + r.manager.SyncRuleGroups(ctx, loadedConfigs) +} + +func (r *Ruler) loadRuleGroups(ctx context.Context) (map[string]rulespb.RuleGroupList, error) { + timer := prometheus.NewTimer(nil) + + defer func() { + storeLoadSeconds := timer.ObserveDuration().Seconds() + r.ruleGroupStoreLoadDuration.Set(storeLoadSeconds) + }() configs, err := r.listRules(ctx) if err != nil { level.Error(r.logger).Log("msg", "unable to list rules", "err", err) - return + return nil, err } loadedConfigs, err := r.store.LoadRuleGroups(ctx, configs) if err != nil { level.Warn(r.logger).Log("msg", "failed to load some rules owned by this ruler", "count", len(configs)-len(loadedConfigs), "err", err) } - - // This will also delete local group files for users that are no longer in 'configs' map. - r.manager.SyncRuleGroups(ctx, loadedConfigs) + return loadedConfigs, nil } func (r *Ruler) listRules(ctx context.Context) (result map[string]rulespb.RuleGroupList, err error) { diff --git a/vendor/github.com/thanos-io/promql-engine/execution/storage/filter.go b/vendor/github.com/thanos-io/promql-engine/execution/storage/filter.go index 4af0f1a0a2..3a2f397be7 100644 --- a/vendor/github.com/thanos-io/promql-engine/execution/storage/filter.go +++ b/vendor/github.com/thanos-io/promql-engine/execution/storage/filter.go @@ -21,7 +21,7 @@ func (n nopFilter) Matches(storage.Series) bool { return true } type filter struct { matchers []*labels.Matcher - matcherSet map[string]*labels.Matcher + matcherSet map[string][]*labels.Matcher } func NewFilter(matchers []*labels.Matcher) Filter { @@ -29,9 +29,9 @@ func NewFilter(matchers []*labels.Matcher) Filter { return &nopFilter{} } - matcherSet := make(map[string]*labels.Matcher) + matcherSet := make(map[string][]*labels.Matcher) for _, m := range matchers { - matcherSet[m.Name] = m + matcherSet[m.Name] = append(matcherSet[m.Name], m) } return &filter{ matchers: matchers, @@ -46,10 +46,13 @@ func (f filter) Matches(series storage.Series) bool { return true } - for name, m := range f.matcherSet { + for name, matchers := range f.matcherSet { label := series.Labels().Get(name) - if !m.Matches(label) { - return false + + for _, m := range matchers { + if !m.Matches(label) { + return false + } } } diff --git a/vendor/github.com/thanos-io/thanos/pkg/block/fetcher.go b/vendor/github.com/thanos-io/thanos/pkg/block/fetcher.go index 69e27737e5..cd30084cfa 100644 --- a/vendor/github.com/thanos-io/thanos/pkg/block/fetcher.go +++ b/vendor/github.com/thanos-io/thanos/pkg/block/fetcher.go @@ -37,6 +37,13 @@ import ( const FetcherConcurrency = 32 +// BaseFetcherMetrics holds metrics tracked by the base fetcher. This struct and its fields are exported +// to allow depending projects (eg. Cortex) to implement their own custom metadata fetcher while tracking +// compatible metrics. +type BaseFetcherMetrics struct { + Syncs prometheus.Counter +} + // FetcherMetrics holds metrics tracked by the metadata fetcher. This struct and its fields are exported // to allow depending projects (eg. Cortex) to implement their own custom metadata fetcher while tracking // compatible metrics. @@ -88,6 +95,18 @@ const ( replicaRemovedMeta = "replica-label-removed" ) +func NewBaseFetcherMetrics(reg prometheus.Registerer) *BaseFetcherMetrics { + var m BaseFetcherMetrics + + m.Syncs = promauto.With(reg).NewCounter(prometheus.CounterOpts{ + Subsystem: fetcherSubSys, + Name: "base_syncs_total", + Help: "Total blocks metadata synchronization attempts by base Fetcher", + }) + + return &m +} + func NewFetcherMetrics(reg prometheus.Registerer, syncedExtraLabels, modifiedExtraLabels [][]string) *FetcherMetrics { var m FetcherMetrics @@ -176,6 +195,11 @@ type BaseFetcher struct { // NewBaseFetcher constructs BaseFetcher. func NewBaseFetcher(logger log.Logger, concurrency int, bkt objstore.InstrumentedBucketReader, dir string, reg prometheus.Registerer) (*BaseFetcher, error) { + return NewBaseFetcherWithMetrics(logger, concurrency, bkt, dir, NewBaseFetcherMetrics(reg)) +} + +// NewBaseFetcherWithMetrics constructs BaseFetcher. +func NewBaseFetcherWithMetrics(logger log.Logger, concurrency int, bkt objstore.InstrumentedBucketReader, dir string, metrics *BaseFetcherMetrics) (*BaseFetcher, error) { if logger == nil { logger = log.NewNopLogger() } @@ -194,11 +218,7 @@ func NewBaseFetcher(logger log.Logger, concurrency int, bkt objstore.Instrumente bkt: bkt, cacheDir: cacheDir, cached: map[ulid.ULID]*metadata.Meta{}, - syncs: promauto.With(reg).NewCounter(prometheus.CounterOpts{ - Subsystem: fetcherSubSys, - Name: "base_syncs_total", - Help: "Total blocks metadata synchronization attempts by base Fetcher", - }), + syncs: metrics.Syncs, }, nil } @@ -217,9 +237,23 @@ func NewMetaFetcher(logger log.Logger, concurrency int, bkt objstore.Instrumente return b.NewMetaFetcher(reg, filters), nil } +// NewMetaFetcherWithMetrics returns meta fetcher. +func NewMetaFetcherWithMetrics(logger log.Logger, concurrency int, bkt objstore.InstrumentedBucketReader, dir string, baseFetcherMetrics *BaseFetcherMetrics, fetcherMetrics *FetcherMetrics, filters []MetadataFilter) (*MetaFetcher, error) { + b, err := NewBaseFetcherWithMetrics(logger, concurrency, bkt, dir, baseFetcherMetrics) + if err != nil { + return nil, err + } + return b.NewMetaFetcherWithMetrics(fetcherMetrics, filters), nil +} + // NewMetaFetcher transforms BaseFetcher into actually usable *MetaFetcher. func (f *BaseFetcher) NewMetaFetcher(reg prometheus.Registerer, filters []MetadataFilter, logTags ...interface{}) *MetaFetcher { - return &MetaFetcher{metrics: NewFetcherMetrics(reg, nil, nil), wrapped: f, filters: filters, logger: log.With(f.logger, logTags...)} + return f.NewMetaFetcherWithMetrics(NewFetcherMetrics(reg, nil, nil), filters, logTags...) +} + +// NewMetaFetcherWithMetrics transforms BaseFetcher into actually usable *MetaFetcher. +func (f *BaseFetcher) NewMetaFetcherWithMetrics(fetcherMetrics *FetcherMetrics, filters []MetadataFilter, logTags ...interface{}) *MetaFetcher { + return &MetaFetcher{metrics: fetcherMetrics, wrapped: f, filters: filters, logger: log.With(f.logger, logTags...)} } var ( diff --git a/vendor/github.com/thanos-io/thanos/pkg/compact/compact.go b/vendor/github.com/thanos-io/thanos/pkg/compact/compact.go index af7425fa55..a6731beba6 100644 --- a/vendor/github.com/thanos-io/thanos/pkg/compact/compact.go +++ b/vendor/github.com/thanos-io/thanos/pkg/compact/compact.go @@ -51,44 +51,46 @@ const ( // It sorts them into compaction groups based on equal label sets. type Syncer struct { logger log.Logger - reg prometheus.Registerer bkt objstore.Bucket fetcher block.MetadataFetcher mtx sync.Mutex blocks map[ulid.ULID]*metadata.Meta partial map[ulid.ULID]error - metrics *syncerMetrics + metrics *SyncerMetrics duplicateBlocksFilter block.DeduplicateFilter ignoreDeletionMarkFilter *block.IgnoreDeletionMarkFilter } -type syncerMetrics struct { - garbageCollectedBlocks prometheus.Counter - garbageCollections prometheus.Counter - garbageCollectionFailures prometheus.Counter - garbageCollectionDuration prometheus.Histogram - blocksMarkedForDeletion prometheus.Counter +// SyncerMetrics holds metrics tracked by the syncer. This struct and its fields are exported +// to allow depending projects (eg. Cortex) to implement their own custom syncer while tracking +// compatible metrics. +type SyncerMetrics struct { + GarbageCollectedBlocks prometheus.Counter + GarbageCollections prometheus.Counter + GarbageCollectionFailures prometheus.Counter + GarbageCollectionDuration prometheus.Histogram + BlocksMarkedForDeletion prometheus.Counter } -func newSyncerMetrics(reg prometheus.Registerer, blocksMarkedForDeletion, garbageCollectedBlocks prometheus.Counter) *syncerMetrics { - var m syncerMetrics +func NewSyncerMetrics(reg prometheus.Registerer, blocksMarkedForDeletion, garbageCollectedBlocks prometheus.Counter) *SyncerMetrics { + var m SyncerMetrics - m.garbageCollectedBlocks = garbageCollectedBlocks - m.garbageCollections = promauto.With(reg).NewCounter(prometheus.CounterOpts{ + m.GarbageCollectedBlocks = garbageCollectedBlocks + m.GarbageCollections = promauto.With(reg).NewCounter(prometheus.CounterOpts{ Name: "thanos_compact_garbage_collection_total", Help: "Total number of garbage collection operations.", }) - m.garbageCollectionFailures = promauto.With(reg).NewCounter(prometheus.CounterOpts{ + m.GarbageCollectionFailures = promauto.With(reg).NewCounter(prometheus.CounterOpts{ Name: "thanos_compact_garbage_collection_failures_total", Help: "Total number of failed garbage collection operations.", }) - m.garbageCollectionDuration = promauto.With(reg).NewHistogram(prometheus.HistogramOpts{ + m.GarbageCollectionDuration = promauto.With(reg).NewHistogram(prometheus.HistogramOpts{ Name: "thanos_compact_garbage_collection_duration_seconds", Help: "Time it took to perform garbage collection iteration.", Buckets: []float64{0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120, 240, 360, 720}, }) - m.blocksMarkedForDeletion = blocksMarkedForDeletion + m.BlocksMarkedForDeletion = blocksMarkedForDeletion return &m } @@ -96,16 +98,19 @@ func newSyncerMetrics(reg prometheus.Registerer, blocksMarkedForDeletion, garbag // NewMetaSyncer returns a new Syncer for the given Bucket and directory. // Blocks must be at least as old as the sync delay for being considered. func NewMetaSyncer(logger log.Logger, reg prometheus.Registerer, bkt objstore.Bucket, fetcher block.MetadataFetcher, duplicateBlocksFilter block.DeduplicateFilter, ignoreDeletionMarkFilter *block.IgnoreDeletionMarkFilter, blocksMarkedForDeletion, garbageCollectedBlocks prometheus.Counter) (*Syncer, error) { + return NewMetaSyncerWithMetrics(logger, NewSyncerMetrics(reg, blocksMarkedForDeletion, garbageCollectedBlocks), bkt, fetcher, duplicateBlocksFilter, ignoreDeletionMarkFilter, blocksMarkedForDeletion, garbageCollectedBlocks) +} + +func NewMetaSyncerWithMetrics(logger log.Logger, metrics *SyncerMetrics, bkt objstore.Bucket, fetcher block.MetadataFetcher, duplicateBlocksFilter block.DeduplicateFilter, ignoreDeletionMarkFilter *block.IgnoreDeletionMarkFilter, blocksMarkedForDeletion, garbageCollectedBlocks prometheus.Counter) (*Syncer, error) { if logger == nil { logger = log.NewNopLogger() } return &Syncer{ logger: logger, - reg: reg, bkt: bkt, fetcher: fetcher, blocks: map[ulid.ULID]*metadata.Meta{}, - metrics: newSyncerMetrics(reg, blocksMarkedForDeletion, garbageCollectedBlocks), + metrics: metrics, duplicateBlocksFilter: duplicateBlocksFilter, ignoreDeletionMarkFilter: ignoreDeletionMarkFilter, }, nil @@ -154,7 +159,12 @@ func (s *Syncer) Metas() map[ulid.ULID]*metadata.Meta { s.mtx.Lock() defer s.mtx.Unlock() - return s.blocks + metas := make(map[ulid.ULID]*metadata.Meta, len(s.blocks)) + for k, v := range s.blocks { + metas[k] = v + } + + return metas } // GarbageCollect marks blocks for deletion from bucket if their data is available as part of a @@ -189,20 +199,20 @@ func (s *Syncer) GarbageCollect(ctx context.Context) error { delCtx, cancel := context.WithTimeout(context.Background(), 5*time.Minute) level.Info(s.logger).Log("msg", "marking outdated block for deletion", "block", id) - err := block.MarkForDeletion(delCtx, s.logger, s.bkt, id, "outdated block", s.metrics.blocksMarkedForDeletion) + err := block.MarkForDeletion(delCtx, s.logger, s.bkt, id, "outdated block", s.metrics.BlocksMarkedForDeletion) cancel() if err != nil { - s.metrics.garbageCollectionFailures.Inc() + s.metrics.GarbageCollectionFailures.Inc() return retry(errors.Wrapf(err, "mark block %s for deletion", id)) } // Immediately update our in-memory state so no further call to SyncMetas is needed // after running garbage collection. delete(s.blocks, id) - s.metrics.garbageCollectedBlocks.Inc() + s.metrics.GarbageCollectedBlocks.Inc() } - s.metrics.garbageCollections.Inc() - s.metrics.garbageCollectionDuration.Observe(time.Since(begin).Seconds()) + s.metrics.GarbageCollections.Inc() + s.metrics.GarbageCollectionDuration.Observe(time.Since(begin).Seconds()) return nil } @@ -1381,7 +1391,7 @@ func (c *BucketCompactor) Compact(ctx context.Context) (rerr error) { } if IsIssue347Error(err) { - if err := RepairIssue347(workCtx, c.logger, c.bkt, c.sy.metrics.blocksMarkedForDeletion, err); err == nil { + if err := RepairIssue347(workCtx, c.logger, c.bkt, c.sy.metrics.BlocksMarkedForDeletion, err); err == nil { mtx.Lock() finishedAllGroups = false mtx.Unlock() diff --git a/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go b/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go index 688d48e14a..7b9676e915 100644 --- a/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go +++ b/vendor/github.com/thanos-io/thanos/pkg/store/bucket.go @@ -1163,7 +1163,7 @@ func (b *blockSeriesClient) nextBatch(tenant string) error { return nil } - b.indexr.reset() + b.indexr.reset(len(postingsBatch)) if !b.skipChunks { b.chunkr.reset() } @@ -2413,8 +2413,8 @@ func (r *bucketIndexReader) IndexVersion() (int, error) { return v, nil } -func (r *bucketIndexReader) reset() { - r.loadedSeries = map[storage.SeriesRef][]byte{} +func (r *bucketIndexReader) reset(size int) { + r.loadedSeries = make(map[storage.SeriesRef][]byte, size) } // ExpandedPostings returns postings in expanded list instead of index.Postings. @@ -2664,8 +2664,9 @@ func matchersToPostingGroups(ctx context.Context, lvalsFn func(name string) ([]s } // Cache label values because label name is the same. if !valuesCached && vals != nil { + lvals := vals lvalsFunc = func(_ string) ([]string, error) { - return vals, nil + return lvals, nil } valuesCached = true } diff --git a/vendor/github.com/thanos-io/thanos/pkg/store/cache/inmemory.go b/vendor/github.com/thanos-io/thanos/pkg/store/cache/inmemory.go index 5404e85fd7..66f05d4177 100644 --- a/vendor/github.com/thanos-io/thanos/pkg/store/cache/inmemory.go +++ b/vendor/github.com/thanos-io/thanos/pkg/store/cache/inmemory.go @@ -198,9 +198,7 @@ func (c *InMemoryIndexCache) onEvict(key, val interface{}) { c.curSize -= entrySize } -func (c *InMemoryIndexCache) get(typ string, key cacheKey, tenant string) ([]byte, bool) { - c.commonMetrics.requestTotal.WithLabelValues(typ, tenant).Inc() - +func (c *InMemoryIndexCache) get(key cacheKey) ([]byte, bool) { c.mtx.Lock() defer c.mtx.Unlock() @@ -208,7 +206,6 @@ func (c *InMemoryIndexCache) get(typ string, key cacheKey, tenant string) ([]byt if !ok { return nil, false } - c.commonMetrics.hitsTotal.WithLabelValues(typ, tenant).Inc() return v.([]byte), true } @@ -309,17 +306,25 @@ func (c *InMemoryIndexCache) FetchMultiPostings(ctx context.Context, blockID uli hits = map[labels.Label][]byte{} blockIDKey := blockID.String() + requests := 0 + hit := 0 for _, key := range keys { if ctx.Err() != nil { + c.commonMetrics.requestTotal.WithLabelValues(cacheTypePostings, tenant).Add(float64(requests)) + c.commonMetrics.hitsTotal.WithLabelValues(cacheTypePostings, tenant).Add(float64(hit)) return hits, misses } - if b, ok := c.get(cacheTypePostings, cacheKey{blockIDKey, cacheKeyPostings(key), ""}, tenant); ok { + requests++ + if b, ok := c.get(cacheKey{blockIDKey, cacheKeyPostings(key), ""}); ok { + hit++ hits[key] = b continue } misses = append(misses, key) } + c.commonMetrics.requestTotal.WithLabelValues(cacheTypePostings, tenant).Add(float64(requests)) + c.commonMetrics.hitsTotal.WithLabelValues(cacheTypePostings, tenant).Add(float64(hit)) return hits, misses } @@ -338,7 +343,9 @@ func (c *InMemoryIndexCache) FetchExpandedPostings(ctx context.Context, blockID if ctx.Err() != nil { return nil, false } - if b, ok := c.get(cacheTypeExpandedPostings, cacheKey{blockID.String(), cacheKeyExpandedPostings(labelMatchersToString(matchers)), ""}, tenant); ok { + c.commonMetrics.requestTotal.WithLabelValues(cacheTypeExpandedPostings, tenant).Inc() + if b, ok := c.get(cacheKey{blockID.String(), cacheKeyExpandedPostings(labelMatchersToString(matchers)), ""}); ok { + c.commonMetrics.hitsTotal.WithLabelValues(cacheTypeExpandedPostings, tenant).Inc() return b, true } return nil, false @@ -360,17 +367,25 @@ func (c *InMemoryIndexCache) FetchMultiSeries(ctx context.Context, blockID ulid. hits = map[storage.SeriesRef][]byte{} blockIDKey := blockID.String() + requests := 0 + hit := 0 for _, id := range ids { if ctx.Err() != nil { + c.commonMetrics.requestTotal.WithLabelValues(cacheTypeSeries, tenant).Add(float64(requests)) + c.commonMetrics.hitsTotal.WithLabelValues(cacheTypeSeries, tenant).Add(float64(hit)) return hits, misses } - if b, ok := c.get(cacheTypeSeries, cacheKey{blockIDKey, cacheKeySeries(id), ""}, tenant); ok { + requests++ + if b, ok := c.get(cacheKey{blockIDKey, cacheKeySeries(id), ""}); ok { + hit++ hits[id] = b continue } misses = append(misses, id) } + c.commonMetrics.requestTotal.WithLabelValues(cacheTypeSeries, tenant).Add(float64(requests)) + c.commonMetrics.hitsTotal.WithLabelValues(cacheTypeSeries, tenant).Add(float64(hit)) return hits, misses } diff --git a/vendor/github.com/thanos-io/thanos/pkg/store/prometheus.go b/vendor/github.com/thanos-io/thanos/pkg/store/prometheus.go index fd6d4c0195..6f5bd1b7de 100644 --- a/vendor/github.com/thanos-io/thanos/pkg/store/prometheus.go +++ b/vendor/github.com/thanos-io/thanos/pkg/store/prometheus.go @@ -699,11 +699,6 @@ func (p *PrometheusStore) LabelValues(ctx context.Context, r *storepb.LabelValue extLset := p.externalLabelsFn() - // First check for matching external label which has priority. - if l := extLset.Get(r.Label); l != "" { - return &storepb.LabelValuesResponse{Values: []string{l}}, nil - } - match, matchers, err := matchesExternalLabels(r.Matchers, extLset) if err != nil { return nil, status.Error(codes.InvalidArgument, err.Error()) @@ -712,6 +707,11 @@ func (p *PrometheusStore) LabelValues(ctx context.Context, r *storepb.LabelValue return &storepb.LabelValuesResponse{Values: nil}, nil } + // First check for matching external label which has priority. + if l := extLset.Get(r.Label); l != "" { + return &storepb.LabelValuesResponse{Values: []string{l}}, nil + } + var ( sers []map[string]string vals []string diff --git a/vendor/github.com/thanos-io/thanos/pkg/store/proxy.go b/vendor/github.com/thanos-io/thanos/pkg/store/proxy.go index 63801b1dc8..ce4d391bf3 100644 --- a/vendor/github.com/thanos-io/thanos/pkg/store/proxy.go +++ b/vendor/github.com/thanos-io/thanos/pkg/store/proxy.go @@ -13,7 +13,6 @@ import ( "github.com/go-kit/log" "github.com/go-kit/log/level" - "github.com/opentracing/opentracing-go" "github.com/pkg/errors" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promauto" @@ -540,7 +539,6 @@ func (s *ProxyStore) LabelValues(ctx context.Context, r *storepb.LabelValuesRequ mtx sync.Mutex g, gctx = errgroup.WithContext(ctx) storeDebugMsgs []string - span opentracing.Span ) // We may arrive here either via the promql engine @@ -564,12 +562,6 @@ func (s *ProxyStore) LabelValues(ctx context.Context, r *storepb.LabelValuesRequ if storeID == "" { storeID = "Store Gateway" } - span, gctx = tracing.StartSpan(gctx, "proxy.label_values", tracing.Tags{ - "store.id": storeID, - "store.addr": storeAddr, - "store.is_local": isLocalStore, - }) - defer span.Finish() // We might be able to skip the store if its meta information indicates it cannot have series matching our query. if ok, reason := storeMatches(gctx, st, s.debugLogging, r.Start, r.End); !ok { @@ -583,7 +575,14 @@ func (s *ProxyStore) LabelValues(ctx context.Context, r *storepb.LabelValuesRequ } g.Go(func() error { - resp, err := st.LabelValues(gctx, &storepb.LabelValuesRequest{ + span, spanCtx := tracing.StartSpan(gctx, "proxy.label_values", tracing.Tags{ + "store.id": storeID, + "store.addr": storeAddr, + "store.is_local": isLocalStore, + }) + defer span.Finish() + + resp, err := st.LabelValues(spanCtx, &storepb.LabelValuesRequest{ Label: r.Label, PartialResponseDisabled: r.PartialResponseDisabled, Start: r.Start, diff --git a/vendor/modules.txt b/vendor/modules.txt index 4b02d13afa..3673cace66 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -881,7 +881,7 @@ github.com/thanos-io/objstore/providers/gcs github.com/thanos-io/objstore/providers/s3 github.com/thanos-io/objstore/providers/swift github.com/thanos-io/objstore/tracing/opentracing -# github.com/thanos-io/promql-engine v0.0.0-20231003153358-8605b6afba51 +# github.com/thanos-io/promql-engine v0.0.0-20231013104847-4517c0d5f591 ## explicit; go 1.19 github.com/thanos-io/promql-engine/api github.com/thanos-io/promql-engine/engine @@ -903,7 +903,7 @@ github.com/thanos-io/promql-engine/extlabels github.com/thanos-io/promql-engine/logicalplan github.com/thanos-io/promql-engine/query github.com/thanos-io/promql-engine/worker -# github.com/thanos-io/thanos v0.32.5-0.20231012105405-728bda9c387a +# github.com/thanos-io/thanos v0.32.5-0.20231020043209-42792d9d0e8c ## explicit; go 1.18 github.com/thanos-io/thanos/pkg/block github.com/thanos-io/thanos/pkg/block/indexheader