From 41571e2ff7ec3c8f11b55a366b429b9a28f20aa9 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Giedrius=20Statkevi=C4=8Dius?= Date: Fri, 12 Jan 2024 20:54:56 +0200 Subject: [PATCH] Revert "receive: upload compacted blocks if OOO enabled (#6974)" (#7053) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit This reverts commit 7b8eb86c0ff3b1144aa1cea392806afa40a4cdf8. Proper way to handle this is to disable vertical compaction. I am trying to add this functionality here: https://github.com/prometheus/prometheus/pull/13393 Signed-off-by: Giedrius Statkevičius --- cmd/thanos/receive.go | 6 ------ pkg/receive/handler_test.go | 1 - pkg/receive/multitsdb.go | 6 +----- pkg/receive/multitsdb_test.go | 10 ---------- pkg/receive/receive_test.go | 1 - pkg/receive/writer_test.go | 2 -- 6 files changed, 1 insertion(+), 25 deletions(-) diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go index c3b1574a44..93f4425b84 100644 --- a/cmd/thanos/receive.go +++ b/cmd/thanos/receive.go @@ -167,8 +167,6 @@ func runReceive( // Has this thanos receive instance been configured to ingest metrics into a local TSDB? enableIngestion := receiveMode == receive.IngestorOnly || receiveMode == receive.RouterIngestor - isOOOEnabled := tsdbOpts.OutOfOrderTimeWindow > 0 - upload := len(confContentYaml) > 0 if enableIngestion { if upload { @@ -178,9 +176,6 @@ func runReceive( "Compaction needs to be disabled (tsdb.min-block-duration = tsdb.max-block-duration)", tsdbOpts.MaxBlockDuration, tsdbOpts.MinBlockDuration) } level.Warn(logger).Log("msg", "flag to ignore min/max block duration flags differing is being used. If the upload of a 2h block fails and a tsdb compaction happens that block may be missing from your Thanos bucket storage.") - if isOOOEnabled { - level.Warn(logger).Log("msg", "out-of-order support is also enabled which means that Receiver will now upload compacted blocks to not lose any data. Vertical compaction needs to be enabled on Compactor! See https://github.com/prometheus/prometheus/issues/13112") - } } // The background shipper continuously scans the data directory and uploads // new blocks to object storage service. @@ -218,7 +213,6 @@ func runReceive( conf.tenantLabelName, bkt, conf.allowOutOfOrderUpload, - isOOOEnabled, hashFunc, ) writer := receive.NewWriter(log.With(logger, "component", "receive-writer"), dbs, &receive.WriterOptions{ diff --git a/pkg/receive/handler_test.go b/pkg/receive/handler_test.go index de511dc8b6..7eadfee930 100644 --- a/pkg/receive/handler_test.go +++ b/pkg/receive/handler_test.go @@ -966,7 +966,6 @@ func benchmarkHandlerMultiTSDBReceiveRemoteWrite(b testutil.TB) { "tenant_id", nil, false, - false, metadata.NoneFunc, ) defer func() { testutil.Ok(b, m.Close()) }() diff --git a/pkg/receive/multitsdb.go b/pkg/receive/multitsdb.go index 65935f917f..3583ce8f3a 100644 --- a/pkg/receive/multitsdb.go +++ b/pkg/receive/multitsdb.go @@ -61,7 +61,6 @@ type MultiTSDB struct { allowOutOfOrderUpload bool hashFunc metadata.HashFunc hashringConfigs []HashringConfig - uploadCompactedBlocks bool } // NewMultiTSDB creates new MultiTSDB. @@ -75,7 +74,6 @@ func NewMultiTSDB( tenantLabelName string, bucket objstore.Bucket, allowOutOfOrderUpload bool, - allowCompactedUpload bool, hashFunc metadata.HashFunc, ) *MultiTSDB { if l == nil { @@ -93,7 +91,6 @@ func NewMultiTSDB( tenantLabelName: tenantLabelName, bucket: bucket, allowOutOfOrderUpload: allowOutOfOrderUpload, - uploadCompactedBlocks: allowCompactedUpload, hashFunc: hashFunc, } } @@ -599,7 +596,6 @@ func (t *MultiTSDB) startTSDB(logger log.Logger, tenantID string, tenant *tenant return err } var ship *shipper.Shipper - if t.bucket != nil { ship = shipper.New( logger, @@ -608,7 +604,7 @@ func (t *MultiTSDB) startTSDB(logger log.Logger, tenantID string, tenant *tenant t.bucket, func() labels.Labels { return lset }, metadata.ReceiveSource, - func() bool { return t.uploadCompactedBlocks }, + nil, t.allowOutOfOrderUpload, t.hashFunc, shipper.DefaultMetaFilename, diff --git a/pkg/receive/multitsdb_test.go b/pkg/receive/multitsdb_test.go index 1b6bb39af8..cc13b5c6ab 100644 --- a/pkg/receive/multitsdb_test.go +++ b/pkg/receive/multitsdb_test.go @@ -51,7 +51,6 @@ func TestMultiTSDB(t *testing.T) { "tenant_id", nil, false, - false, metadata.NoneFunc, ) defer func() { testutil.Ok(t, m.Close()) }() @@ -136,7 +135,6 @@ func TestMultiTSDB(t *testing.T) { "tenant_id", nil, false, - false, metadata.NoneFunc, ) defer func() { testutil.Ok(t, m.Close()) }() @@ -180,7 +178,6 @@ func TestMultiTSDB(t *testing.T) { "tenant_id", nil, false, - false, metadata.NoneFunc, ) defer func() { testutil.Ok(t, m.Close()) }() @@ -448,7 +445,6 @@ func TestMultiTSDBPrune(t *testing.T) { "tenant_id", test.bucket, false, - false, metadata.NoneFunc, ) defer func() { testutil.Ok(t, m.Close()) }() @@ -510,7 +506,6 @@ func TestMultiTSDBRecreatePrunedTenant(t *testing.T) { "tenant_id", objstore.NewInMemBucket(), false, - false, metadata.NoneFunc, ) defer func() { testutil.Ok(t, m.Close()) }() @@ -572,7 +567,6 @@ func TestAlignedHeadFlush(t *testing.T) { "tenant_id", test.bucket, false, - false, metadata.NoneFunc, ) defer func() { testutil.Ok(t, m.Close()) }() @@ -647,7 +641,6 @@ func TestMultiTSDBStats(t *testing.T) { "tenant_id", nil, false, - false, metadata.NoneFunc, ) defer func() { testutil.Ok(t, m.Close()) }() @@ -677,7 +670,6 @@ func TestMultiTSDBWithNilStore(t *testing.T) { "tenant_id", nil, false, - false, metadata.NoneFunc, ) defer func() { testutil.Ok(t, m.Close()) }() @@ -719,7 +711,6 @@ func TestProxyLabelValues(t *testing.T) { "tenant_id", nil, false, - false, metadata.NoneFunc, ) defer func() { testutil.Ok(t, m.Close()) }() @@ -810,7 +801,6 @@ func BenchmarkMultiTSDB(b *testing.B) { "tenant_id", nil, false, - false, metadata.NoneFunc, ) defer func() { testutil.Ok(b, m.Close()) }() diff --git a/pkg/receive/receive_test.go b/pkg/receive/receive_test.go index 0266923584..55b59d182a 100644 --- a/pkg/receive/receive_test.go +++ b/pkg/receive/receive_test.go @@ -814,7 +814,6 @@ func initializeMultiTSDB(dir string) *MultiTSDB { "tenant_id", bucket, false, - false, metadata.NoneFunc, ) diff --git a/pkg/receive/writer_test.go b/pkg/receive/writer_test.go index 9c2ea60490..34613794b8 100644 --- a/pkg/receive/writer_test.go +++ b/pkg/receive/writer_test.go @@ -343,7 +343,6 @@ func TestWriter(t *testing.T) { "tenant_id", nil, false, - false, metadata.NoneFunc, ) t.Cleanup(func() { testutil.Ok(t, m.Close()) }) @@ -436,7 +435,6 @@ func benchmarkWriter(b *testing.B, labelsNum int, seriesNum int, generateHistogr "tenant_id", nil, false, - false, metadata.NoneFunc, ) b.Cleanup(func() { testutil.Ok(b, m.Close()) })