diff --git a/CHANGELOG.md b/CHANGELOG.md index 4cbba1de2da..02feafd1065 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -247,6 +247,7 @@ * [ENHANCEMENT] `tsdb-series`: Added `-min-time` and `-max-time` options to filter samples that are used for computing data-points per minute. #8844 * [ENHANCEMENT] `mimir-rules-action`: Added new input to support matching target namespaces by regex. #9244 * [ENHANCEMENT] `mimir-rules-action`: Added new inputs to support ignoring namespaces and ignoring namespaces by regex. #9258 #9324 +* [BUGFIX] `copyblocks`, `undelete-blocks`, `copyprefix`: use a multipart upload to server-side copy objects greater than 5GiB in size on S3. #9357 ## 2.13.0 diff --git a/pkg/util/objtools/s3.go b/pkg/util/objtools/s3.go index 1344fac4028..f03d710b59e 100644 --- a/pkg/util/objtools/s3.go +++ b/pkg/util/objtools/s3.go @@ -77,22 +77,40 @@ func (bkt *s3Bucket) Get(ctx context.Context, objectName string, options GetOpti return obj, nil } +const maxSingleCopySize int64 = 5 * (1024 * 1024 * 1024) // 5 GiB + func (bkt *s3Bucket) ServerSideCopy(ctx context.Context, objectName string, dstBucket Bucket, options CopyOptions) error { d, ok := dstBucket.(*s3Bucket) if !ok { return errors.New("destination Bucket wasn't an S3 Bucket") } - _, err := d.client.CopyObject(ctx, - minio.CopyDestOptions{ - Bucket: d.bucketName, - Object: options.destinationObjectName(objectName), - }, - minio.CopySrcOptions{ - Bucket: bkt.bucketName, - Object: objectName, - VersionID: options.SourceVersionID, - }, - ) + + stat, err := bkt.client.StatObject(ctx, bkt.bucketName, objectName, minio.StatObjectOptions{ + VersionID: options.SourceVersionID, + }) + if err != nil { + return err + } + + dstOptions := minio.CopyDestOptions{ + Bucket: d.bucketName, + Object: options.destinationObjectName(objectName), + } + + srcOptions := minio.CopySrcOptions{ + Bucket: bkt.bucketName, + Object: objectName, + VersionID: options.SourceVersionID, + } + + if stat.Size <= maxSingleCopySize { + _, err := d.client.CopyObject(ctx, dstOptions, srcOptions) + return err + } + + // Uses a multi-part upload + // Due to https://github.com/minio/minio-go/issues/1683 this likely does not work cross-region + _, err = d.client.ComposeObject(ctx, dstOptions, srcOptions) return err }