From 9c39e9336541ccfefa08cca8b589851fef9c9079 Mon Sep 17 00:00:00 2001 From: Charles Korn Date: Mon, 25 Mar 2024 14:34:25 +1100 Subject: [PATCH 1/8] Fix typo in log message (#7711) --- pkg/storegateway/indexheader/stream_binary_reader.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/storegateway/indexheader/stream_binary_reader.go b/pkg/storegateway/indexheader/stream_binary_reader.go index fabc023105c..de08bc4200c 100644 --- a/pkg/storegateway/indexheader/stream_binary_reader.go +++ b/pkg/storegateway/indexheader/stream_binary_reader.go @@ -190,7 +190,7 @@ func (r *StreamBinaryReader) loadFromSparseIndexHeader(logger *spanlogger.SpanLo level.Info(logger).Log("msg", "loaded sparse index-header from disk", "id", id, "path", sparseHeadersPath, "elapsed", time.Since(start)) }() - level.Info(logger).Log("msg", "loading from sparse index-header from disk", "id", id, "path", sparseHeadersPath) + level.Info(logger).Log("msg", "loading sparse index-header from disk", "id", id, "path", sparseHeadersPath) sparseHeaders := &indexheaderpb.Sparse{} gzipped := bytes.NewReader(sparseData) From ea17a4fd8c66b1f893f583fa34b4aceab1dce0ff Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 25 Mar 2024 08:35:00 +0100 Subject: [PATCH 2/8] fix(deps): update golang.org/x/exp digest to a85f2c6 (#7642) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- go.mod | 2 +- go.sum | 4 ++-- vendor/modules.txt | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/go.mod b/go.mod index 64cc3adcb8f..0e66577eed7 100644 --- a/go.mod +++ b/go.mod @@ -80,7 +80,7 @@ require ( go.opentelemetry.io/otel v1.24.0 go.opentelemetry.io/otel/trace v1.24.0 go.uber.org/multierr v1.11.0 - golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 + golang.org/x/exp v0.0.0-20240318143956-a85f2c67cd81 google.golang.org/api v0.168.0 google.golang.org/protobuf v1.33.0 sigs.k8s.io/kustomize/kyaml v0.16.0 diff --git a/go.sum b/go.sum index 0b551f4df8d..5c04dc6bce1 100644 --- a/go.sum +++ b/go.sum @@ -987,8 +987,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4= golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM= golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU= -golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 h1:LfspQV/FYTatPTr/3HzIcmiUFH7PGP+OQ6mgDYo3yuQ= -golang.org/x/exp v0.0.0-20240222234643-814bf88cf225/go.mod h1:CxmFvTBINI24O/j8iY7H1xHzx2i4OsyguNBmN/uPtqc= +golang.org/x/exp v0.0.0-20240318143956-a85f2c67cd81 h1:6R2FC06FonbXQ8pK11/PDFY6N6LWlf9KlzibaCapmqc= +golang.org/x/exp v0.0.0-20240318143956-a85f2c67cd81/go.mod h1:CQ1k9gNrJ50XIzaKCRR2hssIjF07kZFEiieALBM/ARQ= golang.org/x/image v0.0.0-20180708004352-c73c2afc3b81/go.mod h1:ux5Hcp/YLpHSI86hEcLt0YII63i6oz57MZXIpbrjZUs= golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js= golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0= diff --git a/vendor/modules.txt b/vendor/modules.txt index df2d5a270d6..cb9f28106fa 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -1213,7 +1213,7 @@ golang.org/x/crypto/internal/poly1305 golang.org/x/crypto/pbkdf2 golang.org/x/crypto/pkcs12 golang.org/x/crypto/pkcs12/internal/rc2 -# golang.org/x/exp v0.0.0-20240222234643-814bf88cf225 +# golang.org/x/exp v0.0.0-20240318143956-a85f2c67cd81 ## explicit; go 1.20 golang.org/x/exp/constraints golang.org/x/exp/slices From 1071e98028e485eefdc18d7fb6373a58c41d9170 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 25 Mar 2024 07:40:07 +0000 Subject: [PATCH 3/8] chore(deps): update alpine/helm docker tag to v3.14.3 (main) (#7710) * chore(deps): update alpine/helm docker tag to v3.14.3 * Update build image version to pr7710-7746ee9368 --------- Signed-off-by: Arve Knudsen Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> Co-authored-by: renovate[bot] --- Makefile | 2 +- mimir-build-image/Dockerfile | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/Makefile b/Makefile index 04e143fae13..20f5eb2f113 100644 --- a/Makefile +++ b/Makefile @@ -242,7 +242,7 @@ mimir-build-image/$(UPTODATE): mimir-build-image/* # All the boiler plate for building golang follows: SUDO := $(shell docker info >/dev/null 2>&1 || echo "sudo -E") BUILD_IN_CONTAINER ?= true -LATEST_BUILD_IMAGE_TAG ?= pr7557-ae15c572b6 +LATEST_BUILD_IMAGE_TAG ?= pr7710-7746ee9368 # TTY is parameterized to allow Google Cloud Builder to run builds, # as it currently disallows TTY devices. This value needs to be overridden diff --git a/mimir-build-image/Dockerfile b/mimir-build-image/Dockerfile index c0f28d7c6c7..f1c0c9ab544 100644 --- a/mimir-build-image/Dockerfile +++ b/mimir-build-image/Dockerfile @@ -4,7 +4,7 @@ # Provenance-includes-copyright: The Cortex Authors. FROM registry.k8s.io/kustomize/kustomize:v5.3.0 as kustomize -FROM alpine/helm:3.14.2 as helm +FROM alpine/helm:3.14.3 as helm FROM golang:1.21.8-bookworm ARG goproxyValue ENV GOPROXY=${goproxyValue} From e3f4649ec085d8e2b4f0f0d459b11f5c6de89f31 Mon Sep 17 00:00:00 2001 From: Marco Pracucci Date: Mon, 25 Mar 2024 08:44:04 +0100 Subject: [PATCH 4/8] Add cortex_ingest_storage_reader_last_consumed_offset metric (#7701) Signed-off-by: Marco Pracucci --- pkg/storage/ingest/reader.go | 10 +++++- pkg/storage/ingest/reader_test.go | 55 +++++++++++++++++++++++++++---- 2 files changed, 57 insertions(+), 8 deletions(-) diff --git a/pkg/storage/ingest/reader.go b/pkg/storage/ingest/reader.go index d27afd8ddc1..1718adec583 100644 --- a/pkg/storage/ingest/reader.go +++ b/pkg/storage/ingest/reader.go @@ -360,6 +360,8 @@ func (r *PartitionReader) notifyLastConsumedOffset(fetches kgo.Fetches) { // Records are expected to be sorted by offsets, so we can simply look at the last one. rec := partition.Records[len(partition.Records)-1] r.consumedOffsetWatcher.Notify(rec.Offset) + + r.metrics.lastConsumedOffset.Set(float64(rec.Offset)) }) } @@ -550,7 +552,7 @@ func newPartitionCommitter(kafkaCfg KafkaConfig, admClient *kadm.Client, partiti }), lastCommittedOffset: promauto.With(reg).NewGauge(prometheus.GaugeOpts{ Name: "cortex_ingest_storage_reader_last_committed_offset", - Help: "Total last consumed offset successfully committed by the partition reader. Set to -1 if not offset has been committed yet.", + Help: "The last consumed offset successfully committed by the partition reader. Set to -1 if not offset has been committed yet.", ConstLabels: prometheus.Labels{"partition": strconv.Itoa(int(partitionID))}, }), } @@ -642,6 +644,7 @@ type readerMetrics struct { strongConsistencyRequests prometheus.Counter strongConsistencyFailures prometheus.Counter strongConsistencyLatency prometheus.Histogram + lastConsumedOffset prometheus.Gauge kprom *kprom.Metrics } @@ -688,6 +691,11 @@ func newReaderMetrics(partitionID int32, reg prometheus.Registerer) readerMetric NativeHistogramMinResetDuration: 1 * time.Hour, Buckets: prometheus.DefBuckets, }), + lastConsumedOffset: promauto.With(reg).NewGauge(prometheus.GaugeOpts{ + Name: "cortex_ingest_storage_reader_last_consumed_offset", + Help: "The last offset successfully consumed by the partition reader. Set to -1 if not offset has been consumed yet.", + ConstLabels: prometheus.Labels{"partition": strconv.Itoa(int(partitionID))}, + }), kprom: kprom.NewMetrics("cortex_ingest_storage_reader", kprom.Registerer(prometheus.WrapRegistererWith(prometheus.Labels{"partition": strconv.Itoa(int(partitionID))}, reg)), // Do not export the client ID, because we use it to specify options to the backend. diff --git a/pkg/storage/ingest/reader_test.go b/pkg/storage/ingest/reader_test.go index 521094498f5..ee49c76c715 100644 --- a/pkg/storage/ingest/reader_test.go +++ b/pkg/storage/ingest/reader_test.go @@ -344,7 +344,8 @@ func TestPartitionReader_ConsumeAtStartup(t *testing.T) { t.Log("produced 2 records") // Create and start the reader. - reader := createReader(t, clusterAddr, topicName, partitionID, consumer, withMaxConsumerLagAtStartup(time.Second)) + reg := prometheus.NewPedanticRegistry() + reader := createReader(t, clusterAddr, topicName, partitionID, consumer, withMaxConsumerLagAtStartup(time.Second), withRegistry(reg)) require.NoError(t, reader.StartAsync(ctx)) t.Cleanup(func() { require.NoError(t, services.StopAndAwaitTerminated(ctx, reader)) @@ -369,6 +370,15 @@ func TestPartitionReader_ConsumeAtStartup(t *testing.T) { }) assert.Equal(t, int64(2), consumedRecordsCount.Load()) + + // We expect the last consumed offset to be tracked in a metric. + test.Poll(t, time.Second, nil, func() interface{} { + return promtest.GatherAndCompare(reg, strings.NewReader(` + # HELP cortex_ingest_storage_reader_last_consumed_offset The last offset successfully consumed by the partition reader. Set to -1 if not offset has been consumed yet. + # TYPE cortex_ingest_storage_reader_last_consumed_offset gauge + cortex_ingest_storage_reader_last_consumed_offset{partition="1"} 1 + `), "cortex_ingest_storage_reader_last_consumed_offset") + }) }) t.Run("should consume partition from start if last committed offset is missing and wait until max lag is honored and retry if a failure occurs when fetching last produced offset", func(t *testing.T) { @@ -404,7 +414,8 @@ func TestPartitionReader_ConsumeAtStartup(t *testing.T) { t.Log("produced 2 records") // Create and start the reader. - reader := createReader(t, clusterAddr, topicName, partitionID, consumer, withMaxConsumerLagAtStartup(time.Second)) + reg := prometheus.NewPedanticRegistry() + reader := createReader(t, clusterAddr, topicName, partitionID, consumer, withMaxConsumerLagAtStartup(time.Second), withRegistry(reg)) require.NoError(t, reader.StartAsync(ctx)) t.Cleanup(func() { require.NoError(t, services.StopAndAwaitTerminated(ctx, reader)) @@ -429,6 +440,15 @@ func TestPartitionReader_ConsumeAtStartup(t *testing.T) { }) assert.Equal(t, int64(2), consumedRecordsCount.Load()) + + // We expect the last consumed offset to be tracked in a metric. + test.Poll(t, time.Second, nil, func() interface{} { + return promtest.GatherAndCompare(reg, strings.NewReader(` + # HELP cortex_ingest_storage_reader_last_consumed_offset The last offset successfully consumed by the partition reader. Set to -1 if not offset has been consumed yet. + # TYPE cortex_ingest_storage_reader_last_consumed_offset gauge + cortex_ingest_storage_reader_last_consumed_offset{partition="1"} 1 + `), "cortex_ingest_storage_reader_last_consumed_offset") + }) }) t.Run("should consume partition from end if position=end, and skip honoring max lag", func(t *testing.T) { @@ -552,7 +572,8 @@ func TestPartitionReader_ConsumeAtStartup(t *testing.T) { consumedRecordsMx.Unlock() // Create and start the reader. - reader := createReader(t, clusterAddr, topicName, partitionID, consumer, withConsumeFromPositionAtStartup(consumeFromStart), withMaxConsumerLagAtStartup(time.Second)) + reg := prometheus.NewPedanticRegistry() + reader := createReader(t, clusterAddr, topicName, partitionID, consumer, withConsumeFromPositionAtStartup(consumeFromStart), withMaxConsumerLagAtStartup(time.Second), withRegistry(reg)) require.NoError(t, reader.StartAsync(ctx)) t.Cleanup(func() { require.NoError(t, services.StopAndAwaitTerminated(ctx, reader)) @@ -581,6 +602,15 @@ func TestPartitionReader_ConsumeAtStartup(t *testing.T) { defer consumedRecordsMx.Unlock() return slices.Clone(consumedRecords) }) + + // We expect the last consumed offset to be tracked in a metric. + test.Poll(t, time.Second, nil, func() interface{} { + return promtest.GatherAndCompare(reg, strings.NewReader(` + # HELP cortex_ingest_storage_reader_last_consumed_offset The last offset successfully consumed by the partition reader. Set to -1 if not offset has been consumed yet. + # TYPE cortex_ingest_storage_reader_last_consumed_offset gauge + cortex_ingest_storage_reader_last_consumed_offset{partition="1"} 1 + `), "cortex_ingest_storage_reader_last_consumed_offset") + }) }) } }) @@ -633,7 +663,8 @@ func TestPartitionReader_ConsumeAtStartup(t *testing.T) { t.Log("produced 1 record") // Create and start the reader. - reader := createReader(t, clusterAddr, topicName, partitionID, consumer, withConsumeFromPositionAtStartup(consumeFromLastOffset), withMaxConsumerLagAtStartup(time.Second)) + reg := prometheus.NewPedanticRegistry() + reader := createReader(t, clusterAddr, topicName, partitionID, consumer, withConsumeFromPositionAtStartup(consumeFromLastOffset), withMaxConsumerLagAtStartup(time.Second), withRegistry(reg)) require.NoError(t, reader.StartAsync(ctx)) t.Cleanup(func() { require.NoError(t, services.StopAndAwaitTerminated(ctx, reader)) @@ -662,6 +693,16 @@ func TestPartitionReader_ConsumeAtStartup(t *testing.T) { defer consumedRecordsMx.Unlock() return slices.Clone(consumedRecords) }) + + // We expect the last consumed offset to be tracked in a metric. + expectedConsumedOffset := run - 1 + test.Poll(t, time.Second, nil, func() interface{} { + return promtest.GatherAndCompare(reg, strings.NewReader(fmt.Sprintf(` + # HELP cortex_ingest_storage_reader_last_consumed_offset The last offset successfully consumed by the partition reader. Set to -1 if not offset has been consumed yet. + # TYPE cortex_ingest_storage_reader_last_consumed_offset gauge + cortex_ingest_storage_reader_last_consumed_offset{partition="1"} %d + `, expectedConsumedOffset)), "cortex_ingest_storage_reader_last_consumed_offset") + }) }) } }) @@ -939,7 +980,7 @@ func TestPartitionCommitter(t *testing.T) { // Now we expect the commit to succeed, once the committer will trigger the commit the next interval. test.Poll(t, 10*interval, nil, func() interface{} { return promtest.GatherAndCompare(reg, strings.NewReader(` - # HELP cortex_ingest_storage_reader_last_committed_offset Total last consumed offset successfully committed by the partition reader. Set to -1 if not offset has been committed yet. + # HELP cortex_ingest_storage_reader_last_committed_offset The last consumed offset successfully committed by the partition reader. Set to -1 if not offset has been committed yet. # TYPE cortex_ingest_storage_reader_last_committed_offset gauge cortex_ingest_storage_reader_last_committed_offset{partition="1"} 124 @@ -990,7 +1031,7 @@ func TestPartitionCommitter_commit(t *testing.T) { require.NoError(t, committer.commit(context.Background(), 123)) assert.NoError(t, promtest.GatherAndCompare(reg, strings.NewReader(` - # HELP cortex_ingest_storage_reader_last_committed_offset Total last consumed offset successfully committed by the partition reader. Set to -1 if not offset has been committed yet. + # HELP cortex_ingest_storage_reader_last_committed_offset The last consumed offset successfully committed by the partition reader. Set to -1 if not offset has been committed yet. # TYPE cortex_ingest_storage_reader_last_committed_offset gauge cortex_ingest_storage_reader_last_committed_offset{partition="1"} 124 @@ -1030,7 +1071,7 @@ func TestPartitionCommitter_commit(t *testing.T) { require.Error(t, committer.commit(context.Background(), 123)) assert.NoError(t, promtest.GatherAndCompare(reg, strings.NewReader(` - # HELP cortex_ingest_storage_reader_last_committed_offset Total last consumed offset successfully committed by the partition reader. Set to -1 if not offset has been committed yet. + # HELP cortex_ingest_storage_reader_last_committed_offset The last consumed offset successfully committed by the partition reader. Set to -1 if not offset has been committed yet. # TYPE cortex_ingest_storage_reader_last_committed_offset gauge cortex_ingest_storage_reader_last_committed_offset{partition="1"} -1 From f3f0e8cd9e37e8b458612e0979a87c373ffd25a7 Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 25 Mar 2024 07:59:58 +0000 Subject: [PATCH 5/8] chore(deps): update grafana/agent docker tag to v0.40.3 (#7713) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- .../docker-compose.yml | 2 +- development/mimir-monolithic-mode/docker-compose.yml | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/development/mimir-monolithic-mode-with-swift-storage/docker-compose.yml b/development/mimir-monolithic-mode-with-swift-storage/docker-compose.yml index c5dba048ca9..074dbabcd89 100644 --- a/development/mimir-monolithic-mode-with-swift-storage/docker-compose.yml +++ b/development/mimir-monolithic-mode-with-swift-storage/docker-compose.yml @@ -43,7 +43,7 @@ services: # Scrape the metrics also with the Grafana agent (useful to test metadata ingestion # until metadata remote write is not supported by Prometheus). grafana-agent: - image: grafana/agent:v0.40.2 + image: grafana/agent:v0.40.3 command: ["-config.file=/etc/agent-config/grafana-agent.yaml", "-metrics.wal-directory=/tmp", "-server.http.address=127.0.0.1:9091"] volumes: - ./config:/etc/agent-config diff --git a/development/mimir-monolithic-mode/docker-compose.yml b/development/mimir-monolithic-mode/docker-compose.yml index 7e6d4db1c37..b2eb1e23f6f 100644 --- a/development/mimir-monolithic-mode/docker-compose.yml +++ b/development/mimir-monolithic-mode/docker-compose.yml @@ -44,7 +44,7 @@ services: grafana-agent: profiles: - grafana-agent-static - image: grafana/agent:v0.40.2 + image: grafana/agent:v0.40.3 command: ["-config.file=/etc/agent-config/grafana-agent.yaml", "-metrics.wal-directory=/tmp", "-server.http.address=127.0.0.1:9091"] volumes: - ./config:/etc/agent-config @@ -54,7 +54,7 @@ services: grafana-agent-flow: profiles: - grafana-agent-flow - image: grafana/agent:v0.40.2 + image: grafana/agent:v0.40.3 environment: - AGENT_MODE=flow command: ["run", "--server.http.listen-addr=0.0.0.0:9092", "/etc/agent/config.river"] From e09d7798c437f3a46ccfc768e1557badad38287f Mon Sep 17 00:00:00 2001 From: "renovate[bot]" <29139614+renovate[bot]@users.noreply.github.com> Date: Mon, 25 Mar 2024 09:41:01 +0100 Subject: [PATCH 6/8] chore(deps): update memcached docker tag to v1.6.25 (#7716) Co-authored-by: renovate[bot] <29139614+renovate[bot]@users.noreply.github.com> --- operations/helm/charts/mimir-distributed/values.yaml | 2 +- .../templates/chunks-cache/chunks-cache-statefulset.yaml | 2 +- .../templates/index-cache/index-cache-statefulset.yaml | 2 +- .../templates/metadata-cache/metadata-cache-statefulset.yaml | 2 +- .../templates/results-cache/results-cache-statefulset.yaml | 2 +- .../graphite-aggregation-cache-statefulset.yaml | 2 +- .../graphite-metric-name-cache-statefulset.yaml | 2 +- .../templates/chunks-cache/chunks-cache-statefulset.yaml | 2 +- .../templates/index-cache/index-cache-statefulset.yaml | 2 +- .../templates/metadata-cache/metadata-cache-statefulset.yaml | 2 +- .../templates/results-cache/results-cache-statefulset.yaml | 2 +- .../templates/chunks-cache/chunks-cache-statefulset.yaml | 2 +- .../templates/index-cache/index-cache-statefulset.yaml | 2 +- .../templates/metadata-cache/metadata-cache-statefulset.yaml | 2 +- .../templates/results-cache/results-cache-statefulset.yaml | 2 +- .../templates/admin-cache/admin-cache-statefulset.yaml | 2 +- .../templates/chunks-cache/chunks-cache-statefulset.yaml | 2 +- .../templates/index-cache/index-cache-statefulset.yaml | 2 +- .../templates/metadata-cache/metadata-cache-statefulset.yaml | 2 +- .../templates/results-cache/results-cache-statefulset.yaml | 2 +- .../templates/chunks-cache/chunks-cache-statefulset.yaml | 2 +- .../templates/index-cache/index-cache-statefulset.yaml | 2 +- .../templates/metadata-cache/metadata-cache-statefulset.yaml | 2 +- .../templates/results-cache/results-cache-statefulset.yaml | 2 +- .../templates/chunks-cache/chunks-cache-statefulset.yaml | 2 +- .../templates/index-cache/index-cache-statefulset.yaml | 2 +- .../templates/metadata-cache/metadata-cache-statefulset.yaml | 2 +- .../templates/results-cache/results-cache-statefulset.yaml | 2 +- 28 files changed, 28 insertions(+), 28 deletions(-) diff --git a/operations/helm/charts/mimir-distributed/values.yaml b/operations/helm/charts/mimir-distributed/values.yaml index 238a2b9c5eb..e9720360308 100644 --- a/operations/helm/charts/mimir-distributed/values.yaml +++ b/operations/helm/charts/mimir-distributed/values.yaml @@ -1875,7 +1875,7 @@ memcached: # -- Memcached Docker image repository repository: memcached # -- Memcached Docker image tag - tag: 1.6.24-alpine + tag: 1.6.25-alpine # -- Memcached Docker image pull policy pullPolicy: IfNotPresent diff --git a/operations/helm/tests/enterprise-https-values-generated/mimir-distributed/templates/chunks-cache/chunks-cache-statefulset.yaml b/operations/helm/tests/enterprise-https-values-generated/mimir-distributed/templates/chunks-cache/chunks-cache-statefulset.yaml index b7d5f6e6f11..6233ea15fcd 100644 --- a/operations/helm/tests/enterprise-https-values-generated/mimir-distributed/templates/chunks-cache/chunks-cache-statefulset.yaml +++ b/operations/helm/tests/enterprise-https-values-generated/mimir-distributed/templates/chunks-cache/chunks-cache-statefulset.yaml @@ -50,7 +50,7 @@ spec: secretName: tls-certs containers: - name: memcached - image: memcached:1.6.24-alpine + image: memcached:1.6.25-alpine imagePullPolicy: IfNotPresent resources: limits: diff --git a/operations/helm/tests/enterprise-https-values-generated/mimir-distributed/templates/index-cache/index-cache-statefulset.yaml b/operations/helm/tests/enterprise-https-values-generated/mimir-distributed/templates/index-cache/index-cache-statefulset.yaml index 5fbbedc49d6..d2e4238ab8f 100644 --- a/operations/helm/tests/enterprise-https-values-generated/mimir-distributed/templates/index-cache/index-cache-statefulset.yaml +++ b/operations/helm/tests/enterprise-https-values-generated/mimir-distributed/templates/index-cache/index-cache-statefulset.yaml @@ -50,7 +50,7 @@ spec: secretName: tls-certs containers: - name: memcached - image: memcached:1.6.24-alpine + image: memcached:1.6.25-alpine imagePullPolicy: IfNotPresent resources: limits: diff --git a/operations/helm/tests/enterprise-https-values-generated/mimir-distributed/templates/metadata-cache/metadata-cache-statefulset.yaml b/operations/helm/tests/enterprise-https-values-generated/mimir-distributed/templates/metadata-cache/metadata-cache-statefulset.yaml index a14e6514cd9..0237b11a25a 100644 --- a/operations/helm/tests/enterprise-https-values-generated/mimir-distributed/templates/metadata-cache/metadata-cache-statefulset.yaml +++ b/operations/helm/tests/enterprise-https-values-generated/mimir-distributed/templates/metadata-cache/metadata-cache-statefulset.yaml @@ -50,7 +50,7 @@ spec: secretName: tls-certs containers: - name: memcached - image: memcached:1.6.24-alpine + image: memcached:1.6.25-alpine imagePullPolicy: IfNotPresent resources: limits: diff --git a/operations/helm/tests/enterprise-https-values-generated/mimir-distributed/templates/results-cache/results-cache-statefulset.yaml b/operations/helm/tests/enterprise-https-values-generated/mimir-distributed/templates/results-cache/results-cache-statefulset.yaml index 0b5579bd6a9..5c5d05102f6 100644 --- a/operations/helm/tests/enterprise-https-values-generated/mimir-distributed/templates/results-cache/results-cache-statefulset.yaml +++ b/operations/helm/tests/enterprise-https-values-generated/mimir-distributed/templates/results-cache/results-cache-statefulset.yaml @@ -50,7 +50,7 @@ spec: secretName: tls-certs containers: - name: memcached - image: memcached:1.6.24-alpine + image: memcached:1.6.25-alpine imagePullPolicy: IfNotPresent resources: limits: diff --git a/operations/helm/tests/graphite-enabled-values-generated/mimir-distributed/templates/graphite-proxy/graphite-aggregation-cache/graphite-aggregation-cache-statefulset.yaml b/operations/helm/tests/graphite-enabled-values-generated/mimir-distributed/templates/graphite-proxy/graphite-aggregation-cache/graphite-aggregation-cache-statefulset.yaml index 67f1cd30fc3..e82814bc258 100644 --- a/operations/helm/tests/graphite-enabled-values-generated/mimir-distributed/templates/graphite-proxy/graphite-aggregation-cache/graphite-aggregation-cache-statefulset.yaml +++ b/operations/helm/tests/graphite-enabled-values-generated/mimir-distributed/templates/graphite-proxy/graphite-aggregation-cache/graphite-aggregation-cache-statefulset.yaml @@ -46,7 +46,7 @@ spec: terminationGracePeriodSeconds: 60 containers: - name: memcached - image: memcached:1.6.24-alpine + image: memcached:1.6.25-alpine imagePullPolicy: IfNotPresent resources: limits: diff --git a/operations/helm/tests/graphite-enabled-values-generated/mimir-distributed/templates/graphite-proxy/graphite-metric-name-cache/graphite-metric-name-cache-statefulset.yaml b/operations/helm/tests/graphite-enabled-values-generated/mimir-distributed/templates/graphite-proxy/graphite-metric-name-cache/graphite-metric-name-cache-statefulset.yaml index 41e1b27769e..10ad6a5e796 100644 --- a/operations/helm/tests/graphite-enabled-values-generated/mimir-distributed/templates/graphite-proxy/graphite-metric-name-cache/graphite-metric-name-cache-statefulset.yaml +++ b/operations/helm/tests/graphite-enabled-values-generated/mimir-distributed/templates/graphite-proxy/graphite-metric-name-cache/graphite-metric-name-cache-statefulset.yaml @@ -46,7 +46,7 @@ spec: terminationGracePeriodSeconds: 60 containers: - name: memcached - image: memcached:1.6.24-alpine + image: memcached:1.6.25-alpine imagePullPolicy: IfNotPresent resources: limits: diff --git a/operations/helm/tests/large-values-generated/mimir-distributed/templates/chunks-cache/chunks-cache-statefulset.yaml b/operations/helm/tests/large-values-generated/mimir-distributed/templates/chunks-cache/chunks-cache-statefulset.yaml index 79fd316791f..d1622e93017 100644 --- a/operations/helm/tests/large-values-generated/mimir-distributed/templates/chunks-cache/chunks-cache-statefulset.yaml +++ b/operations/helm/tests/large-values-generated/mimir-distributed/templates/chunks-cache/chunks-cache-statefulset.yaml @@ -46,7 +46,7 @@ spec: terminationGracePeriodSeconds: 60 containers: - name: memcached - image: memcached:1.6.24-alpine + image: memcached:1.6.25-alpine imagePullPolicy: IfNotPresent resources: limits: diff --git a/operations/helm/tests/large-values-generated/mimir-distributed/templates/index-cache/index-cache-statefulset.yaml b/operations/helm/tests/large-values-generated/mimir-distributed/templates/index-cache/index-cache-statefulset.yaml index 16b1eaef860..3c7e0bdf752 100644 --- a/operations/helm/tests/large-values-generated/mimir-distributed/templates/index-cache/index-cache-statefulset.yaml +++ b/operations/helm/tests/large-values-generated/mimir-distributed/templates/index-cache/index-cache-statefulset.yaml @@ -46,7 +46,7 @@ spec: terminationGracePeriodSeconds: 60 containers: - name: memcached - image: memcached:1.6.24-alpine + image: memcached:1.6.25-alpine imagePullPolicy: IfNotPresent resources: limits: diff --git a/operations/helm/tests/large-values-generated/mimir-distributed/templates/metadata-cache/metadata-cache-statefulset.yaml b/operations/helm/tests/large-values-generated/mimir-distributed/templates/metadata-cache/metadata-cache-statefulset.yaml index 76c15f5c486..382015a7d86 100644 --- a/operations/helm/tests/large-values-generated/mimir-distributed/templates/metadata-cache/metadata-cache-statefulset.yaml +++ b/operations/helm/tests/large-values-generated/mimir-distributed/templates/metadata-cache/metadata-cache-statefulset.yaml @@ -46,7 +46,7 @@ spec: terminationGracePeriodSeconds: 60 containers: - name: memcached - image: memcached:1.6.24-alpine + image: memcached:1.6.25-alpine imagePullPolicy: IfNotPresent resources: limits: diff --git a/operations/helm/tests/large-values-generated/mimir-distributed/templates/results-cache/results-cache-statefulset.yaml b/operations/helm/tests/large-values-generated/mimir-distributed/templates/results-cache/results-cache-statefulset.yaml index 0d3b11feb06..068db1677c4 100644 --- a/operations/helm/tests/large-values-generated/mimir-distributed/templates/results-cache/results-cache-statefulset.yaml +++ b/operations/helm/tests/large-values-generated/mimir-distributed/templates/results-cache/results-cache-statefulset.yaml @@ -46,7 +46,7 @@ spec: terminationGracePeriodSeconds: 60 containers: - name: memcached - image: memcached:1.6.24-alpine + image: memcached:1.6.25-alpine imagePullPolicy: IfNotPresent resources: limits: diff --git a/operations/helm/tests/small-values-generated/mimir-distributed/templates/chunks-cache/chunks-cache-statefulset.yaml b/operations/helm/tests/small-values-generated/mimir-distributed/templates/chunks-cache/chunks-cache-statefulset.yaml index 25a98e31967..7af95611c46 100644 --- a/operations/helm/tests/small-values-generated/mimir-distributed/templates/chunks-cache/chunks-cache-statefulset.yaml +++ b/operations/helm/tests/small-values-generated/mimir-distributed/templates/chunks-cache/chunks-cache-statefulset.yaml @@ -46,7 +46,7 @@ spec: terminationGracePeriodSeconds: 60 containers: - name: memcached - image: memcached:1.6.24-alpine + image: memcached:1.6.25-alpine imagePullPolicy: IfNotPresent resources: limits: diff --git a/operations/helm/tests/small-values-generated/mimir-distributed/templates/index-cache/index-cache-statefulset.yaml b/operations/helm/tests/small-values-generated/mimir-distributed/templates/index-cache/index-cache-statefulset.yaml index b2a9287eb00..4f75b865123 100644 --- a/operations/helm/tests/small-values-generated/mimir-distributed/templates/index-cache/index-cache-statefulset.yaml +++ b/operations/helm/tests/small-values-generated/mimir-distributed/templates/index-cache/index-cache-statefulset.yaml @@ -46,7 +46,7 @@ spec: terminationGracePeriodSeconds: 60 containers: - name: memcached - image: memcached:1.6.24-alpine + image: memcached:1.6.25-alpine imagePullPolicy: IfNotPresent resources: limits: diff --git a/operations/helm/tests/small-values-generated/mimir-distributed/templates/metadata-cache/metadata-cache-statefulset.yaml b/operations/helm/tests/small-values-generated/mimir-distributed/templates/metadata-cache/metadata-cache-statefulset.yaml index 448ff6558dc..bd650ed775c 100644 --- a/operations/helm/tests/small-values-generated/mimir-distributed/templates/metadata-cache/metadata-cache-statefulset.yaml +++ b/operations/helm/tests/small-values-generated/mimir-distributed/templates/metadata-cache/metadata-cache-statefulset.yaml @@ -46,7 +46,7 @@ spec: terminationGracePeriodSeconds: 60 containers: - name: memcached - image: memcached:1.6.24-alpine + image: memcached:1.6.25-alpine imagePullPolicy: IfNotPresent resources: limits: diff --git a/operations/helm/tests/small-values-generated/mimir-distributed/templates/results-cache/results-cache-statefulset.yaml b/operations/helm/tests/small-values-generated/mimir-distributed/templates/results-cache/results-cache-statefulset.yaml index ffffa7281d5..3ac01dfb7c9 100644 --- a/operations/helm/tests/small-values-generated/mimir-distributed/templates/results-cache/results-cache-statefulset.yaml +++ b/operations/helm/tests/small-values-generated/mimir-distributed/templates/results-cache/results-cache-statefulset.yaml @@ -46,7 +46,7 @@ spec: terminationGracePeriodSeconds: 60 containers: - name: memcached - image: memcached:1.6.24-alpine + image: memcached:1.6.25-alpine imagePullPolicy: IfNotPresent resources: limits: diff --git a/operations/helm/tests/test-enterprise-configmap-values-generated/mimir-distributed/templates/admin-cache/admin-cache-statefulset.yaml b/operations/helm/tests/test-enterprise-configmap-values-generated/mimir-distributed/templates/admin-cache/admin-cache-statefulset.yaml index a6c8836b174..b2e93dc63f3 100644 --- a/operations/helm/tests/test-enterprise-configmap-values-generated/mimir-distributed/templates/admin-cache/admin-cache-statefulset.yaml +++ b/operations/helm/tests/test-enterprise-configmap-values-generated/mimir-distributed/templates/admin-cache/admin-cache-statefulset.yaml @@ -47,7 +47,7 @@ spec: terminationGracePeriodSeconds: 60 containers: - name: memcached - image: memcached:1.6.24-alpine + image: memcached:1.6.25-alpine imagePullPolicy: IfNotPresent resources: limits: null diff --git a/operations/helm/tests/test-enterprise-configmap-values-generated/mimir-distributed/templates/chunks-cache/chunks-cache-statefulset.yaml b/operations/helm/tests/test-enterprise-configmap-values-generated/mimir-distributed/templates/chunks-cache/chunks-cache-statefulset.yaml index 468a2b5a992..08f3519807b 100644 --- a/operations/helm/tests/test-enterprise-configmap-values-generated/mimir-distributed/templates/chunks-cache/chunks-cache-statefulset.yaml +++ b/operations/helm/tests/test-enterprise-configmap-values-generated/mimir-distributed/templates/chunks-cache/chunks-cache-statefulset.yaml @@ -47,7 +47,7 @@ spec: terminationGracePeriodSeconds: 60 containers: - name: memcached - image: memcached:1.6.24-alpine + image: memcached:1.6.25-alpine imagePullPolicy: IfNotPresent resources: limits: null diff --git a/operations/helm/tests/test-enterprise-configmap-values-generated/mimir-distributed/templates/index-cache/index-cache-statefulset.yaml b/operations/helm/tests/test-enterprise-configmap-values-generated/mimir-distributed/templates/index-cache/index-cache-statefulset.yaml index 48e0c046e0d..51b6c5438af 100644 --- a/operations/helm/tests/test-enterprise-configmap-values-generated/mimir-distributed/templates/index-cache/index-cache-statefulset.yaml +++ b/operations/helm/tests/test-enterprise-configmap-values-generated/mimir-distributed/templates/index-cache/index-cache-statefulset.yaml @@ -47,7 +47,7 @@ spec: terminationGracePeriodSeconds: 60 containers: - name: memcached - image: memcached:1.6.24-alpine + image: memcached:1.6.25-alpine imagePullPolicy: IfNotPresent resources: limits: null diff --git a/operations/helm/tests/test-enterprise-configmap-values-generated/mimir-distributed/templates/metadata-cache/metadata-cache-statefulset.yaml b/operations/helm/tests/test-enterprise-configmap-values-generated/mimir-distributed/templates/metadata-cache/metadata-cache-statefulset.yaml index 0c5fb05b64f..19f4a595e4c 100644 --- a/operations/helm/tests/test-enterprise-configmap-values-generated/mimir-distributed/templates/metadata-cache/metadata-cache-statefulset.yaml +++ b/operations/helm/tests/test-enterprise-configmap-values-generated/mimir-distributed/templates/metadata-cache/metadata-cache-statefulset.yaml @@ -47,7 +47,7 @@ spec: terminationGracePeriodSeconds: 60 containers: - name: memcached - image: memcached:1.6.24-alpine + image: memcached:1.6.25-alpine imagePullPolicy: IfNotPresent resources: limits: null diff --git a/operations/helm/tests/test-enterprise-configmap-values-generated/mimir-distributed/templates/results-cache/results-cache-statefulset.yaml b/operations/helm/tests/test-enterprise-configmap-values-generated/mimir-distributed/templates/results-cache/results-cache-statefulset.yaml index a74d5e00a08..030aadb8eda 100644 --- a/operations/helm/tests/test-enterprise-configmap-values-generated/mimir-distributed/templates/results-cache/results-cache-statefulset.yaml +++ b/operations/helm/tests/test-enterprise-configmap-values-generated/mimir-distributed/templates/results-cache/results-cache-statefulset.yaml @@ -47,7 +47,7 @@ spec: terminationGracePeriodSeconds: 60 containers: - name: memcached - image: memcached:1.6.24-alpine + image: memcached:1.6.25-alpine imagePullPolicy: IfNotPresent resources: limits: null diff --git a/operations/helm/tests/test-oss-k8s-1.25-values-generated/mimir-distributed/templates/chunks-cache/chunks-cache-statefulset.yaml b/operations/helm/tests/test-oss-k8s-1.25-values-generated/mimir-distributed/templates/chunks-cache/chunks-cache-statefulset.yaml index cdd57106844..89589ead7a2 100644 --- a/operations/helm/tests/test-oss-k8s-1.25-values-generated/mimir-distributed/templates/chunks-cache/chunks-cache-statefulset.yaml +++ b/operations/helm/tests/test-oss-k8s-1.25-values-generated/mimir-distributed/templates/chunks-cache/chunks-cache-statefulset.yaml @@ -46,7 +46,7 @@ spec: terminationGracePeriodSeconds: 60 containers: - name: memcached - image: memcached:1.6.24-alpine + image: memcached:1.6.25-alpine imagePullPolicy: IfNotPresent resources: limits: diff --git a/operations/helm/tests/test-oss-k8s-1.25-values-generated/mimir-distributed/templates/index-cache/index-cache-statefulset.yaml b/operations/helm/tests/test-oss-k8s-1.25-values-generated/mimir-distributed/templates/index-cache/index-cache-statefulset.yaml index 0212f3786a5..37b3bbc9b2e 100644 --- a/operations/helm/tests/test-oss-k8s-1.25-values-generated/mimir-distributed/templates/index-cache/index-cache-statefulset.yaml +++ b/operations/helm/tests/test-oss-k8s-1.25-values-generated/mimir-distributed/templates/index-cache/index-cache-statefulset.yaml @@ -46,7 +46,7 @@ spec: terminationGracePeriodSeconds: 60 containers: - name: memcached - image: memcached:1.6.24-alpine + image: memcached:1.6.25-alpine imagePullPolicy: IfNotPresent resources: limits: diff --git a/operations/helm/tests/test-oss-k8s-1.25-values-generated/mimir-distributed/templates/metadata-cache/metadata-cache-statefulset.yaml b/operations/helm/tests/test-oss-k8s-1.25-values-generated/mimir-distributed/templates/metadata-cache/metadata-cache-statefulset.yaml index 1607a2c7518..97b13b9248e 100644 --- a/operations/helm/tests/test-oss-k8s-1.25-values-generated/mimir-distributed/templates/metadata-cache/metadata-cache-statefulset.yaml +++ b/operations/helm/tests/test-oss-k8s-1.25-values-generated/mimir-distributed/templates/metadata-cache/metadata-cache-statefulset.yaml @@ -46,7 +46,7 @@ spec: terminationGracePeriodSeconds: 60 containers: - name: memcached - image: memcached:1.6.24-alpine + image: memcached:1.6.25-alpine imagePullPolicy: IfNotPresent resources: limits: diff --git a/operations/helm/tests/test-oss-k8s-1.25-values-generated/mimir-distributed/templates/results-cache/results-cache-statefulset.yaml b/operations/helm/tests/test-oss-k8s-1.25-values-generated/mimir-distributed/templates/results-cache/results-cache-statefulset.yaml index e82275bd425..f2382388227 100644 --- a/operations/helm/tests/test-oss-k8s-1.25-values-generated/mimir-distributed/templates/results-cache/results-cache-statefulset.yaml +++ b/operations/helm/tests/test-oss-k8s-1.25-values-generated/mimir-distributed/templates/results-cache/results-cache-statefulset.yaml @@ -46,7 +46,7 @@ spec: terminationGracePeriodSeconds: 60 containers: - name: memcached - image: memcached:1.6.24-alpine + image: memcached:1.6.25-alpine imagePullPolicy: IfNotPresent resources: limits: diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/chunks-cache/chunks-cache-statefulset.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/chunks-cache/chunks-cache-statefulset.yaml index d96cd7c9698..71566b9f2dc 100644 --- a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/chunks-cache/chunks-cache-statefulset.yaml +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/chunks-cache/chunks-cache-statefulset.yaml @@ -47,7 +47,7 @@ spec: terminationGracePeriodSeconds: 60 containers: - name: memcached - image: memcached:1.6.24-alpine + image: memcached:1.6.25-alpine imagePullPolicy: IfNotPresent resources: limits: null diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/index-cache/index-cache-statefulset.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/index-cache/index-cache-statefulset.yaml index 0fb77caf6e9..6106d647c0d 100644 --- a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/index-cache/index-cache-statefulset.yaml +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/index-cache/index-cache-statefulset.yaml @@ -47,7 +47,7 @@ spec: terminationGracePeriodSeconds: 60 containers: - name: memcached - image: memcached:1.6.24-alpine + image: memcached:1.6.25-alpine imagePullPolicy: IfNotPresent resources: limits: null diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/metadata-cache/metadata-cache-statefulset.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/metadata-cache/metadata-cache-statefulset.yaml index 2721ec824aa..f70cae03ab5 100644 --- a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/metadata-cache/metadata-cache-statefulset.yaml +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/metadata-cache/metadata-cache-statefulset.yaml @@ -47,7 +47,7 @@ spec: terminationGracePeriodSeconds: 60 containers: - name: memcached - image: memcached:1.6.24-alpine + image: memcached:1.6.25-alpine imagePullPolicy: IfNotPresent resources: limits: null diff --git a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/results-cache/results-cache-statefulset.yaml b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/results-cache/results-cache-statefulset.yaml index 980e06bad3d..25617e38da6 100644 --- a/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/results-cache/results-cache-statefulset.yaml +++ b/operations/helm/tests/test-oss-values-generated/mimir-distributed/templates/results-cache/results-cache-statefulset.yaml @@ -47,7 +47,7 @@ spec: terminationGracePeriodSeconds: 60 containers: - name: memcached - image: memcached:1.6.24-alpine + image: memcached:1.6.25-alpine imagePullPolicy: IfNotPresent resources: limits: null From 8c201821baff7688ab2275fa196aad67f1514d86 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20=C5=A0tibran=C3=BD?= Date: Mon, 25 Mar 2024 11:19:14 +0100 Subject: [PATCH 7/8] Introduce New function for DefaultCacheKeyGenerator. (#7718) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * Introduce New function for DefaultCacheKeyGenerator. Signed-off-by: Peter Štibraný * Hide interval. Signed-off-by: Peter Štibraný --------- Signed-off-by: Peter Štibraný --- pkg/frontend/querymiddleware/results_cache.go | 15 +++++++++++---- .../querymiddleware/results_cache_test.go | 2 +- pkg/frontend/querymiddleware/roundtrip.go | 2 +- .../querymiddleware/split_and_cache_test.go | 16 ++++++++-------- 4 files changed, 21 insertions(+), 14 deletions(-) diff --git a/pkg/frontend/querymiddleware/results_cache.go b/pkg/frontend/querymiddleware/results_cache.go index 70177c616ef..3870c5697b2 100644 --- a/pkg/frontend/querymiddleware/results_cache.go +++ b/pkg/frontend/querymiddleware/results_cache.go @@ -204,13 +204,20 @@ type CacheKeyGenerator interface { type DefaultCacheKeyGenerator struct { codec Codec - // Interval is a constant split Interval when determining cache keys for QueryRequest. - Interval time.Duration + // interval is a constant split interval when determining cache keys for QueryRequest. + interval time.Duration } -// QueryRequest generates a cache key based on the userID, Request and Interval. +func NewDefaultCacheKeyGenerator(codec Codec, interval time.Duration) DefaultCacheKeyGenerator { + return DefaultCacheKeyGenerator{ + codec: codec, + interval: interval, + } +} + +// QueryRequest generates a cache key based on the userID, Request and interval. func (g DefaultCacheKeyGenerator) QueryRequest(_ context.Context, userID string, r Request) string { - startInterval := r.GetStart() / g.Interval.Milliseconds() + startInterval := r.GetStart() / g.interval.Milliseconds() stepOffset := r.GetStart() % r.GetStep() // Use original format for step-aligned request, so that we can use existing cached results for such requests. diff --git a/pkg/frontend/querymiddleware/results_cache_test.go b/pkg/frontend/querymiddleware/results_cache_test.go index 878d168900a..a1c4115ba2e 100644 --- a/pkg/frontend/querymiddleware/results_cache_test.go +++ b/pkg/frontend/querymiddleware/results_cache_test.go @@ -579,7 +579,7 @@ func TestDefaultSplitter_QueryRequest(t *testing.T) { } for _, tt := range tests { t.Run(fmt.Sprintf("%s - %s", tt.name, tt.interval), func(t *testing.T) { - if got := (DefaultCacheKeyGenerator{codec: codec, Interval: tt.interval}).QueryRequest(ctx, "fake", tt.r); got != tt.want { + if got := (DefaultCacheKeyGenerator{codec: codec, interval: tt.interval}).QueryRequest(ctx, "fake", tt.r); got != tt.want { t.Errorf("generateKey() = %v, want %v", got, tt.want) } }) diff --git a/pkg/frontend/querymiddleware/roundtrip.go b/pkg/frontend/querymiddleware/roundtrip.go index 4a2414e386b..10e261364a2 100644 --- a/pkg/frontend/querymiddleware/roundtrip.go +++ b/pkg/frontend/querymiddleware/roundtrip.go @@ -239,7 +239,7 @@ func newQueryTripperware( cacheKeyGenerator := cfg.CacheKeyGenerator if cacheKeyGenerator == nil { - cacheKeyGenerator = DefaultCacheKeyGenerator{codec: codec, Interval: cfg.SplitQueriesByInterval} + cacheKeyGenerator = NewDefaultCacheKeyGenerator(codec, cfg.SplitQueriesByInterval) } // Inject the middleware to split requests by interval + results cache (if at least one of the two is enabled). diff --git a/pkg/frontend/querymiddleware/split_and_cache_test.go b/pkg/frontend/querymiddleware/split_and_cache_test.go index b577bee30d6..46af8e53aa9 100644 --- a/pkg/frontend/querymiddleware/split_and_cache_test.go +++ b/pkg/frontend/querymiddleware/split_and_cache_test.go @@ -244,7 +244,7 @@ func TestSplitAndCacheMiddleware_ResultsCache(t *testing.T) { mockLimits{maxCacheFreshness: 10 * time.Minute, resultsCacheTTL: resultsCacheTTL, resultsCacheOutOfOrderWindowTTL: resultsCacheLowerTTL}, newTestPrometheusCodec(), cacheBackend, - DefaultCacheKeyGenerator{Interval: day}, + DefaultCacheKeyGenerator{interval: day}, PrometheusResponseExtractor{}, resultsCacheAlwaysEnabled, log.NewNopLogger(), @@ -376,7 +376,7 @@ func TestSplitAndCacheMiddleware_ResultsCache_ShouldNotLookupCacheIfStepIsNotAli mockLimits{maxCacheFreshness: 10 * time.Minute}, newTestPrometheusCodec(), cacheBackend, - DefaultCacheKeyGenerator{Interval: day}, + DefaultCacheKeyGenerator{interval: day}, PrometheusResponseExtractor{}, resultsCacheAlwaysEnabled, log.NewNopLogger(), @@ -492,7 +492,7 @@ func TestSplitAndCacheMiddleware_ResultsCache_EnabledCachingOfStepUnalignedReque limits, newTestPrometheusCodec(), cacheBackend, - DefaultCacheKeyGenerator{Interval: day}, + DefaultCacheKeyGenerator{interval: day}, PrometheusResponseExtractor{}, resultsCacheAlwaysEnabled, log.NewNopLogger(), @@ -643,7 +643,7 @@ func TestSplitAndCacheMiddleware_ResultsCache_ShouldNotCacheRequestEarlierThanMa for testName, testData := range tests { t.Run(testName, func(t *testing.T) { cacheBackend := cache.NewMockCache() - keyGenerator := DefaultCacheKeyGenerator{Interval: day} + keyGenerator := DefaultCacheKeyGenerator{interval: day} reg := prometheus.NewPedanticRegistry() mw := newSplitAndCacheMiddleware( @@ -864,7 +864,7 @@ func TestSplitAndCacheMiddleware_ResultsCacheFuzzy(t *testing.T) { }, newTestPrometheusCodec(), cache.NewMockCache(), - DefaultCacheKeyGenerator{Interval: day}, + DefaultCacheKeyGenerator{interval: day}, PrometheusResponseExtractor{}, resultsCacheAlwaysEnabled, log.NewNopLogger(), @@ -1134,7 +1134,7 @@ func TestSplitAndCacheMiddleware_ResultsCache_ExtentsEdgeCases(t *testing.T) { t.Run(testName, func(t *testing.T) { ctx := user.InjectOrgID(context.Background(), userID) cacheBackend := cache.NewInstrumentedMockCache() - keyGenerator := DefaultCacheKeyGenerator{Interval: day} + keyGenerator := DefaultCacheKeyGenerator{interval: day} mw := newSplitAndCacheMiddleware( false, // No splitting. @@ -1192,7 +1192,7 @@ func TestSplitAndCacheMiddleware_StoreAndFetchCacheExtents(t *testing.T) { }, newTestPrometheusCodec(), cacheBackend, - DefaultCacheKeyGenerator{Interval: day}, + DefaultCacheKeyGenerator{interval: day}, PrometheusResponseExtractor{}, resultsCacheAlwaysEnabled, log.NewNopLogger(), @@ -1273,7 +1273,7 @@ func TestSplitAndCacheMiddleware_WrapMultipleTimes(t *testing.T) { mockLimits{}, newTestPrometheusCodec(), cache.NewMockCache(), - DefaultCacheKeyGenerator{Interval: day}, + DefaultCacheKeyGenerator{interval: day}, PrometheusResponseExtractor{}, resultsCacheAlwaysEnabled, log.NewNopLogger(), From 0020042f621effdfcde2507c902f57a71cfdfea2 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Peter=20=C5=A0tibran=C3=BD?= Date: Mon, 25 Mar 2024 11:42:53 +0100 Subject: [PATCH 8/8] Move query-frontend-codec initialization into separate module. (#7720) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Signed-off-by: Peter Štibraný --- pkg/mimir/modules.go | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/pkg/mimir/modules.go b/pkg/mimir/modules.go index 60d7c3f1451..dcec7a68f5f 100644 --- a/pkg/mimir/modules.go +++ b/pkg/mimir/modules.go @@ -85,6 +85,7 @@ const ( Queryable string = "queryable" StoreQueryable string = "store-queryable" QueryFrontend string = "query-frontend" + QueryFrontendCodec string = "query-frontend-codec" QueryFrontendTripperware string = "query-frontend-tripperware" RulerStorage string = "ruler-storage" Ruler string = "ruler" @@ -703,10 +704,16 @@ func (t *Mimir) initFlusher() (serv services.Service, err error) { return t.Flusher, nil } +// initQueryFrontendCodec initializes query frontend codec. +// NOTE: Grafana Enterprise Metrics depends on this. +func (t *Mimir) initQueryFrontendCodec() (services.Service, error) { + t.QueryFrontendCodec = querymiddleware.NewPrometheusCodec(t.Registerer, t.Cfg.Frontend.QueryMiddleware.QueryResultResponseFormat) + return nil, nil +} + // initQueryFrontendTripperware instantiates the tripperware used by the query frontend // to optimize Prometheus query requests. func (t *Mimir) initQueryFrontendTripperware() (serv services.Service, err error) { - t.QueryFrontendCodec = querymiddleware.NewPrometheusCodec(t.Registerer, t.Cfg.Frontend.QueryMiddleware.QueryResultResponseFormat) promqlEngineRegisterer := prometheus.WrapRegistererWith(prometheus.Labels{"engine": "query-frontend"}, t.Registerer) engineOpts, engineExperimentalFunctionsEnabled := engine.NewPromQLEngineOptions(t.Cfg.Querier.EngineConfig, t.ActivityTracker, util_log.Logger, promqlEngineRegisterer) @@ -1051,6 +1058,7 @@ func (t *Mimir) setupModuleManager() error { mm.RegisterModule(Queryable, t.initQueryable, modules.UserInvisibleModule) mm.RegisterModule(Querier, t.initQuerier) mm.RegisterModule(StoreQueryable, t.initStoreQueryable, modules.UserInvisibleModule) + mm.RegisterModule(QueryFrontendCodec, t.initQueryFrontendCodec, modules.UserInvisibleModule) mm.RegisterModule(QueryFrontendTripperware, t.initQueryFrontendTripperware, modules.UserInvisibleModule) mm.RegisterModule(QueryFrontend, t.initQueryFrontend) mm.RegisterModule(RulerStorage, t.initRulerStorage, modules.UserInvisibleModule) @@ -1085,7 +1093,7 @@ func (t *Mimir) setupModuleManager() error { Queryable: {Overrides, DistributorService, IngesterRing, IngesterPartitionRing, API, StoreQueryable, MemberlistKV}, Querier: {TenantFederation, Vault}, StoreQueryable: {Overrides, MemberlistKV}, - QueryFrontendTripperware: {API, Overrides}, + QueryFrontendTripperware: {API, Overrides, QueryFrontendCodec}, QueryFrontend: {QueryFrontendTripperware, MemberlistKV, Vault}, QueryScheduler: {API, Overrides, MemberlistKV, Vault}, Ruler: {DistributorService, StoreQueryable, RulerStorage, Vault},