From 0b2bff92a6235e9990b6ce144e11372c88f0f549 Mon Sep 17 00:00:00 2001 From: Yu Long Date: Wed, 6 Nov 2024 17:11:17 -0800 Subject: [PATCH] Fix API breakage --- cmd/thanos/compact.go | 2 +- cmd/thanos/downsample.go | 2 +- cmd/thanos/receive.go | 2 +- cmd/thanos/rule.go | 2 +- cmd/thanos/sidecar.go | 2 +- cmd/thanos/store.go | 2 +- cmd/thanos/tools_bucket.go | 20 ++++++++++---------- go.mod | 2 +- go.sum | 4 ++-- pkg/replicate/replicator.go | 4 ++-- test/e2e/compact_test.go | 6 +++--- test/e2e/query_test.go | 6 +++--- test/e2e/store_gateway_test.go | 16 ++++++++-------- test/e2e/tools_bucket_web_test.go | 2 +- 14 files changed, 36 insertions(+), 36 deletions(-) diff --git a/cmd/thanos/compact.go b/cmd/thanos/compact.go index ddaf702d85..741fc664ef 100644 --- a/cmd/thanos/compact.go +++ b/cmd/thanos/compact.go @@ -205,7 +205,7 @@ func runCompact( return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.String(), nil) if err != nil { return err } diff --git a/cmd/thanos/downsample.go b/cmd/thanos/downsample.go index a5cd5c38ed..556369b0a1 100644 --- a/cmd/thanos/downsample.go +++ b/cmd/thanos/downsample.go @@ -84,7 +84,7 @@ func RunDownsample( return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Downsample.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Downsample.String(), nil) if err != nil { return err } diff --git a/cmd/thanos/receive.go b/cmd/thanos/receive.go index 569ec1e57e..5db1d51631 100644 --- a/cmd/thanos/receive.go +++ b/cmd/thanos/receive.go @@ -195,7 +195,7 @@ func runReceive( } // The background shipper continuously scans the data directory and uploads // new blocks to object storage service. - bkt, err = client.NewBucket(logger, confContentYaml, comp.String()) + bkt, err = client.NewBucket(logger, confContentYaml, comp.String(), nil) if err != nil { return err } diff --git a/cmd/thanos/rule.go b/cmd/thanos/rule.go index ad793d17a4..62b42761a7 100644 --- a/cmd/thanos/rule.go +++ b/cmd/thanos/rule.go @@ -843,7 +843,7 @@ func runRule( if len(confContentYaml) > 0 { // The background shipper continuously scans the data directory and uploads // new blocks to Google Cloud Storage or an S3-compatible storage service. - bkt, err := client.NewBucket(logger, confContentYaml, component.Rule.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Rule.String(), nil) if err != nil { return err } diff --git a/cmd/thanos/sidecar.go b/cmd/thanos/sidecar.go index 91f7feee54..3df6612259 100644 --- a/cmd/thanos/sidecar.go +++ b/cmd/thanos/sidecar.go @@ -367,7 +367,7 @@ func runSidecar( if uploads { // The background shipper continuously scans the data directory and uploads // new blocks to Google Cloud Storage or an S3-compatible storage service. - bkt, err := client.NewBucket(logger, confContentYaml, component.Sidecar.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Sidecar.String(), nil) if err != nil { return err } diff --git a/cmd/thanos/store.go b/cmd/thanos/store.go index d626fe55d8..0c5eb780a1 100644 --- a/cmd/thanos/store.go +++ b/cmd/thanos/store.go @@ -309,7 +309,7 @@ func runStore( return err } - bkt, err := client.NewBucket(logger, confContentYaml, conf.component.String()) + bkt, err := client.NewBucket(logger, confContentYaml, conf.component.String(), nil) if err != nil { return err } diff --git a/cmd/thanos/tools_bucket.go b/cmd/thanos/tools_bucket.go index 7e711168cf..7875d9c0a0 100644 --- a/cmd/thanos/tools_bucket.go +++ b/cmd/thanos/tools_bucket.go @@ -327,7 +327,7 @@ func registerBucketVerify(app extkingpin.AppClause, objStoreConfig *extflag.Path return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Bucket.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Bucket.String(), nil) if err != nil { return err } @@ -346,7 +346,7 @@ func registerBucketVerify(app extkingpin.AppClause, objStoreConfig *extflag.Path } } else { // nil Prometheus registerer: don't create conflicting metrics. - backupBkt, err = client.NewBucket(logger, backupconfContentYaml, component.Bucket.String()) + backupBkt, err = client.NewBucket(logger, backupconfContentYaml, component.Bucket.String(), nil) if err != nil { return err } @@ -411,7 +411,7 @@ func registerBucketLs(app extkingpin.AppClause, objStoreConfig *extflag.PathOrCo return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Bucket.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Bucket.String(), nil) if err != nil { return err } @@ -519,7 +519,7 @@ func registerBucketInspect(app extkingpin.AppClause, objStoreConfig *extflag.Pat return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Bucket.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Bucket.String(), nil) if err != nil { return err } @@ -629,7 +629,7 @@ func registerBucketWeb(app extkingpin.AppClause, objStoreConfig *extflag.PathOrC return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Bucket.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Bucket.String(), nil) if err != nil { return errors.Wrap(err, "bucket client") } @@ -826,7 +826,7 @@ func registerBucketCleanup(app extkingpin.AppClause, objStoreConfig *extflag.Pat return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Cleanup.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Cleanup.String(), nil) if err != nil { return err } @@ -1083,7 +1083,7 @@ func registerBucketMarkBlock(app extkingpin.AppClause, objStoreConfig *extflag.P return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Mark.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Mark.String(), nil) if err != nil { return err } @@ -1163,7 +1163,7 @@ func registerBucketRewrite(app extkingpin.AppClause, objStoreConfig *extflag.Pat return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Rewrite.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Rewrite.String(), nil) if err != nil { return err } @@ -1371,7 +1371,7 @@ func registerBucketRetention(app extkingpin.AppClause, objStoreConfig *extflag.P return err } - bkt, err := client.NewBucket(logger, confContentYaml, component.Retention.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Retention.String(), nil) if err != nil { return err } @@ -1460,7 +1460,7 @@ func registerBucketUploadBlocks(app extkingpin.AppClause, objStoreConfig *extfla return errors.Wrap(err, "unable to parse objstore config") } - bkt, err := client.NewBucket(logger, confContentYaml, component.Upload.String()) + bkt, err := client.NewBucket(logger, confContentYaml, component.Upload.String(), nil) if err != nil { return errors.Wrap(err, "unable to create bucket") } diff --git a/go.mod b/go.mod index 565c325300..5438621f51 100644 --- a/go.mod +++ b/go.mod @@ -61,7 +61,7 @@ require ( github.com/prometheus/prometheus v0.54.2-0.20240920164404-6f0d6038b7f9 github.com/sony/gobreaker v0.5.0 github.com/stretchr/testify v1.9.0 - github.com/thanos-io/objstore v0.0.0-20240913074259-63feed0da069 + github.com/thanos-io/objstore v0.0.0-20241028150459-cfdd0e50390d github.com/thanos-io/promql-engine v0.0.0-20240921092401-37747eddbd31 github.com/uber/jaeger-client-go v2.30.0+incompatible github.com/uber/jaeger-lib v2.4.1+incompatible // indirect diff --git a/go.sum b/go.sum index 0e75a24782..660b4e37d4 100644 --- a/go.sum +++ b/go.sum @@ -2243,8 +2243,8 @@ github.com/tencentyun/cos-go-sdk-v5 v0.7.40 h1:W6vDGKCHe4wBACI1d2UgE6+50sJFhRWU4 github.com/tencentyun/cos-go-sdk-v5 v0.7.40/go.mod h1:4dCEtLHGh8QPxHEkgq+nFaky7yZxQuYwgSJM87icDaw= github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e h1:f1Zsv7OAU9iQhZwigp50Yl38W10g/vd5NC8Rdk1Jzng= github.com/thanos-community/galaxycache v0.0.0-20211122094458-3a32041a1f1e/go.mod h1:jXcofnrSln/cLI6/dhlBxPQZEEQHVPCcFaH75M+nSzM= -github.com/thanos-io/objstore v0.0.0-20240913074259-63feed0da069 h1:TUPZ6euAh8I62KrpDnBIg7k2C5HjgXQnVHoUUMacGwM= -github.com/thanos-io/objstore v0.0.0-20240913074259-63feed0da069/go.mod h1:Cba80S8NbVBBdyZKzra7San/jXvpAxArbpFymWzIZhg= +github.com/thanos-io/objstore v0.0.0-20241028150459-cfdd0e50390d h1:k+SLTP1mjNqXxsCiq4UYeKCe07le0ieffyuHm/YfmH8= +github.com/thanos-io/objstore v0.0.0-20241028150459-cfdd0e50390d/go.mod h1:/ZMUxFcp/nT6oYV5WslH9k07NU/+86+aibgZRmMMr/4= github.com/thanos-io/promql-engine v0.0.0-20240921092401-37747eddbd31 h1:xPaP58g+3EPohdw4cv+6jv5+LcX6LynhHvQcYwTAMxQ= github.com/thanos-io/promql-engine v0.0.0-20240921092401-37747eddbd31/go.mod h1:wx0JlRZtsB2S10JYUgeg5GqLfMxw31SzArP+28yyE00= github.com/themihai/gomemcache v0.0.0-20180902122335-24332e2d58ab h1:7ZR3hmisBWw77ZpO1/o86g+JV3VKlk3d48jopJxzTjU= diff --git a/pkg/replicate/replicator.go b/pkg/replicate/replicator.go index 668d64afce..dc9e804f56 100644 --- a/pkg/replicate/replicator.go +++ b/pkg/replicate/replicator.go @@ -115,7 +115,7 @@ func RunReplicate( return errors.New("No supported bucket was configured to replicate from") } - bkt, err := client.NewBucket(logger, fromConfContentYaml, component.Replicate.String()) + bkt, err := client.NewBucket(logger, fromConfContentYaml, component.Replicate.String(), nil) if err != nil { return err } @@ -136,7 +136,7 @@ func RunReplicate( return errors.New("No supported bucket was configured to replicate to") } - toBkt, err := client.NewBucket(logger, toConfContentYaml, component.Replicate.String()) + toBkt, err := client.NewBucket(logger, toConfContentYaml, component.Replicate.String(), nil) if err != nil { return err } diff --git a/test/e2e/compact_test.go b/test/e2e/compact_test.go index 54c04fe91a..975edb51c5 100644 --- a/test/e2e/compact_test.go +++ b/test/e2e/compact_test.go @@ -351,7 +351,7 @@ func testCompactWithStoreGateway(t *testing.T, penaltyDedup bool) { testutil.Ok(t, e2e.StartAndWaitReady(m)) bkt, err := s3.NewBucketWithConfig(logger, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) testutil.Ok(t, err) ctx, cancel := context.WithTimeout(context.Background(), 90*time.Second) @@ -896,7 +896,7 @@ func TestCompactorDownsampleIgnoresMarked(t *testing.T) { testutil.Ok(t, e2e.StartAndWaitReady(m)) bktCfg := e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()) - bkt, err := s3.NewBucketWithConfig(logger, bktCfg, "test") + bkt, err := s3.NewBucketWithConfig(logger, bktCfg, "test", nil) testutil.Ok(t, err) downsampledBase := blockDesc{ @@ -944,7 +944,7 @@ func TestCompactorIssue6775(t *testing.T) { testutil.Ok(t, e2e.StartAndWaitReady(m)) bkt, err := s3.NewBucketWithConfig(logger, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) testutil.Ok(t, err) baseBlockDesc := blockDesc{ diff --git a/test/e2e/query_test.go b/test/e2e/query_test.go index 763e5cd9cb..0351cc9bc8 100644 --- a/test/e2e/query_test.go +++ b/test/e2e/query_test.go @@ -619,7 +619,7 @@ func TestQueryStoreMetrics(t *testing.T) { testutil.Ok(t, e2e.StartAndWaitReady(minio)) l := log.NewLogfmtLogger(os.Stdout) - bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, minio.Endpoint("http"), minio.Dir()), "test") + bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, minio.Endpoint("http"), minio.Dir()), "test", nil) testutil.Ok(t, err) // Preparing 3 different blocks for the tests. @@ -807,7 +807,7 @@ func TestQueryStoreDedup(t *testing.T) { testutil.Ok(t, e2e.StartAndWaitReady(minio)) l := log.NewLogfmtLogger(os.Stdout) - bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, minio.Endpoint("http"), minio.Dir()), "test") + bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, minio.Endpoint("http"), minio.Dir()), "test", nil) testutil.Ok(t, err) storeGW := e2ethanos.NewStoreGW( @@ -2035,7 +2035,7 @@ func TestQueryTenancyEnforcement(t *testing.T) { testutil.Ok(t, e2e.StartAndWaitReady(minio)) l := log.NewLogfmtLogger(os.Stdout) - bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, minio.Endpoint("http"), minio.Dir()), "test") + bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, minio.Endpoint("http"), minio.Dir()), "test", nil) testutil.Ok(t, err) // Add series from different tenants diff --git a/test/e2e/store_gateway_test.go b/test/e2e/store_gateway_test.go index 912712f0a8..8977aa46bb 100644 --- a/test/e2e/store_gateway_test.go +++ b/test/e2e/store_gateway_test.go @@ -127,7 +127,7 @@ metafile_content_ttl: 0s`, memcached.InternalEndpoint("memcached")) testutil.Ok(t, err) l := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(l, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) testutil.Ok(t, err) testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id1.String()), id1.String())) @@ -453,7 +453,7 @@ func TestStoreGatewayNoCacheFile(t *testing.T) { testutil.Ok(t, err) l := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(l, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) testutil.Ok(t, err) testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id1.String()), id1.String())) @@ -672,7 +672,7 @@ blocks_iter_ttl: 0s`, memcached.InternalEndpoint("memcached")) l := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(l, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) testutil.Ok(t, err) testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id.String()), id.String())) @@ -807,7 +807,7 @@ metafile_content_ttl: 0s` testutil.Ok(t, err) l := log.NewLogfmtLogger(os.Stdout) - bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") + bkt, err := s3.NewBucketWithConfig(l, e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) testutil.Ok(t, err) testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id.String()), id.String())) @@ -934,7 +934,7 @@ config: testutil.Ok(t, err) l := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(l, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) testutil.Ok(t, err) testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id1.String()), id1.String())) @@ -1068,7 +1068,7 @@ config: l := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(l, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) testutil.Ok(t, err) testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id.String()), id.String())) @@ -1180,7 +1180,7 @@ func TestStoreGatewayLazyExpandedPostingsEnabled(t *testing.T) { l := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(l, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) testutil.Ok(t, err) testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id.String()), id.String())) @@ -1354,7 +1354,7 @@ func TestStoreGatewayLazyExpandedPostingsPromQLSmithFuzz(t *testing.T) { l := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(l, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed") + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "test-feed", nil) testutil.Ok(t, err) testutil.Ok(t, objstore.UploadDir(ctx, l, bkt, path.Join(dir, id.String()), id.String())) diff --git a/test/e2e/tools_bucket_web_test.go b/test/e2e/tools_bucket_web_test.go index a72a14a3db..e679d2b924 100644 --- a/test/e2e/tools_bucket_web_test.go +++ b/test/e2e/tools_bucket_web_test.go @@ -156,7 +156,7 @@ func TestToolsBucketWebWithTimeAndRelabelFilter(t *testing.T) { // Create bucket. logger := log.NewLogfmtLogger(os.Stdout) bkt, err := s3.NewBucketWithConfig(logger, - e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "tools") + e2ethanos.NewS3Config(bucket, m.Endpoint("http"), m.Dir()), "tools", nil) testutil.Ok(t, err) // Create share dir for upload.