From 80a5ce6b157765ac46fcafe0478c28fd5f93570f Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Tue, 16 Jan 2024 12:50:56 +0200 Subject: [PATCH] receive: disable overlapping compaction MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Use the new TSDB flag to disable overlapping compaction to fix OOO samples handling in the Receive component. Signed-off-by: Giedrius Statkevičius --- go.mod | 2 +- go.sum | 4 ++-- pkg/receive/multitsdb.go | 6 ++++++ 3 files changed, 9 insertions(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d5a30ea70d..0faf225636 100644 --- a/go.mod +++ b/go.mod @@ -62,7 +62,7 @@ require ( github.com/prometheus/common v0.45.1-0.20231122191551-832cd6e99f99 github.com/prometheus/exporter-toolkit v0.10.0 // Prometheus maps version 2.x.y to tags v0.x.y. - github.com/prometheus/prometheus v0.48.1-0.20240115084306-17920623e7cd + github.com/prometheus/prometheus v0.49.0-rc.2.0.20240116085416-72a8f1084b29 github.com/sony/gobreaker v0.5.0 github.com/stretchr/testify v1.8.4 github.com/thanos-io/objstore v0.0.0-20231112185854-37752ee64d98 diff --git a/go.sum b/go.sum index 10003653f5..200b7832f1 100644 --- a/go.sum +++ b/go.sum @@ -894,8 +894,8 @@ github.com/prometheus/procfs v0.7.3/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1 github.com/prometheus/procfs v0.8.0/go.mod h1:z7EfXMXOkbkqb9IINtpCn86r/to3BnA0uaxHdg830/4= github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= -github.com/prometheus/prometheus v0.48.1-0.20240115084306-17920623e7cd h1:qq+8W4nYaar4QxY6JtYYVfUfO4AVNQcJ53Jaz3UZCMU= -github.com/prometheus/prometheus v0.48.1-0.20240115084306-17920623e7cd/go.mod h1:NCkyb7Sg5TsJxi6Hx9rCcAXZ2E1Bu7x0dYOQ0jLTfDU= +github.com/prometheus/prometheus v0.49.0-rc.2.0.20240116085416-72a8f1084b29 h1:poqfIBU1Rt6K1lA0IxizRrN/NoOY1bVtTmrQT9HsmOg= +github.com/prometheus/prometheus v0.49.0-rc.2.0.20240116085416-72a8f1084b29/go.mod h1:NCkyb7Sg5TsJxi6Hx9rCcAXZ2E1Bu7x0dYOQ0jLTfDU= github.com/rcrowley/go-metrics v0.0.0-20181016184325-3113b8401b8a/go.mod h1:bCqnVzQkZxMG4s8nGwiZ5l3QUCyqpo9Y+/ZMZ9VjZe4= github.com/redis/rueidis v1.0.14-go1.18 h1:dGir5z8w8X1ex7JWO/Zx2FMBrZgQ8Yjm+lw9fPLSNGw= github.com/redis/rueidis v1.0.14-go1.18/go.mod h1:HGekzV3HbmzFmRK6j0xic8Z9119+ECoGMjeN1TV1NYU= diff --git a/pkg/receive/multitsdb.go b/pkg/receive/multitsdb.go index 3583ce8f3a..754b2ad6e7 100644 --- a/pkg/receive/multitsdb.go +++ b/pkg/receive/multitsdb.go @@ -582,6 +582,12 @@ func (t *MultiTSDB) startTSDB(logger log.Logger, tenantID string, tenant *tenant level.Info(logger).Log("msg", "opening TSDB") opts := *t.tsdbOpts + + // NOTE(GiedriusS): always set to false to properly handle OOO samples - OOO samples are written into the WBL + // which gets later converted into a block. Without setting this flag to false, the block would get compacted + // into other ones. This presents a race between compaction and the shipper (if it is configured to upload compacted blocks). + // Hence, avoid this situation by disabling overlapping compaction. Vertical compaction must be enabled on the compactor. + opts.EnableOverlappingCompaction = false s, err := tsdb.Open( dataDir, logger,