diff --git a/go.mod b/go.mod index d8f426233..58219424a 100644 --- a/go.mod +++ b/go.mod @@ -8,7 +8,7 @@ require ( github.com/moby/patternmatcher v0.6.0 github.com/spf13/cobra v1.8.1 github.com/stretchr/testify v1.9.0 - gocloud.dev v0.39.0 + gocloud.dev v0.40.0 gotest.tools/v3 v3.5.1 ) diff --git a/go.sum b/go.sum index 3fcea6c25..213cb38a9 100644 --- a/go.sum +++ b/go.sum @@ -208,8 +208,8 @@ go.opentelemetry.io/otel/sdk v1.28.0 h1:b9d7hIry8yZsgtbmM0DKyPWMMUMlK9NEKuIG4aBq go.opentelemetry.io/otel/sdk v1.28.0/go.mod h1:oYj7ClPUA7Iw3m+r7GeEjz0qckQRJK2B8zjcZEfu7Pg= go.opentelemetry.io/otel/trace v1.29.0 h1:J/8ZNK4XgR7a21DZUAsbF8pZ5Jcw1VhACmnYt39JTi4= go.opentelemetry.io/otel/trace v1.29.0/go.mod h1:eHl3w0sp3paPkYstJOmAimxhiFXPg+MMTlEh3nsQgWQ= -gocloud.dev v0.39.0 h1:EYABYGhAalPUaMrbSKOr5lejxoxvXj99nE8XFtsDgds= -gocloud.dev v0.39.0/go.mod h1:drz+VyYNBvrMTW0KZiBAYEdl8lbNZx+OQ7oQvdrFmSQ= +gocloud.dev v0.40.0 h1:f8LgP+4WDqOG/RXoUcyLpeIAGOcAbZrZbDQCUee10ng= +gocloud.dev v0.40.0/go.mod h1:drz+VyYNBvrMTW0KZiBAYEdl8lbNZx+OQ7oQvdrFmSQ= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPhUkYOs5KpL4U8rLKemX1yGLhDgUto= golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= diff --git a/vendor/gocloud.dev/aws/aws.go b/vendor/gocloud.dev/aws/aws.go index 21d4ec5af..4ee177c3b 100644 --- a/vendor/gocloud.dev/aws/aws.go +++ b/vendor/gocloud.dev/aws/aws.go @@ -22,10 +22,14 @@ import ( "strconv" awsv2 "github.com/aws/aws-sdk-go-v2/aws" + "github.com/aws/aws-sdk-go-v2/aws/ratelimit" + "github.com/aws/aws-sdk-go-v2/aws/retry" + "github.com/aws/aws-sdk-go-v2/config" awsv2cfg "github.com/aws/aws-sdk-go-v2/config" "github.com/aws/aws-sdk-go/aws" "github.com/aws/aws-sdk-go/aws/client" "github.com/aws/aws-sdk-go/aws/credentials" + "github.com/aws/aws-sdk-go/aws/endpoints" "github.com/aws/aws-sdk-go/aws/session" "github.com/google/wire" ) @@ -79,8 +83,10 @@ func (co ConfigOverrider) ClientConfig(serviceName string, cfgs ...*aws.Config) // The following query options are supported: // - region: The AWS region for requests; sets aws.Config.Region. // - endpoint: The endpoint URL (hostname only or fully qualified URI); sets aws.Config.Endpoint. -// - disableSSL: A value of "true" disables SSL when sending requests; sets aws.Config.DisableSSL. -// - s3ForcePathStyle: A value of "true" forces the request to use path-style addressing; sets aws.Config.S3ForcePathStyle. +// - disable_ssl (or disableSSL): A value of "true" disables SSL when sending requests; sets aws.Config.DisableSSL. +// - s3_force_path_style (or s3ForcePathStyle): A value of "true" forces the request to use path-style addressing; sets aws.Config.S3ForcePathStyle. +// - dualstack: A value of "true" enables dual stack (IPv4 and IPv6) endpoints +// - fips: A value of "true" enables the use of FIPS endpoints func ConfigFromURLParams(q url.Values) (*aws.Config, error) { var cfg aws.Config for param, values := range q { @@ -90,18 +96,32 @@ func ConfigFromURLParams(q url.Values) (*aws.Config, error) { cfg.Region = aws.String(value) case "endpoint": cfg.Endpoint = aws.String(value) - case "disableSSL": + case "disable_ssl", "disableSSL": b, err := strconv.ParseBool(value) if err != nil { return nil, fmt.Errorf("invalid value for query parameter %q: %v", param, err) } cfg.DisableSSL = aws.Bool(b) - case "s3ForcePathStyle": + case "s3_force_path_style", "s3ForcePathStyle": b, err := strconv.ParseBool(value) if err != nil { return nil, fmt.Errorf("invalid value for query parameter %q: %v", param, err) } cfg.S3ForcePathStyle = aws.Bool(b) + case "dualstack": + b, err := strconv.ParseBool(value) + if err != nil { + return nil, fmt.Errorf("invalid value for query parameter %q: %v", param, err) + } + cfg.UseDualStack = aws.Bool(b) + case "fips": + b, err := strconv.ParseBool(value) + if err != nil { + return nil, fmt.Errorf("invalid value for query parameter %q: %v", param, err) + } + if b { + cfg.UseFIPSEndpoint = endpoints.FIPSEndpointStateEnabled + } case "awssdk": // ignore, should be handled before this default: @@ -177,30 +197,83 @@ func NewDefaultV2Config(ctx context.Context) (awsv2.Config, error) { // - region: The AWS region for requests; sets WithRegion. // - profile: The shared config profile to use; sets SharedConfigProfile. // - endpoint: The AWS service endpoint to send HTTP request. +// - hostname_immutable: Make the hostname immutable, only works if endpoint is also set. +// - dualstack: A value of "true" enables dual stack (IPv4 and IPv6) endpoints. +// - fips: A value of "true" enables the use of FIPS endpoints. +// - rate_limiter_capacity: A integer value configures the capacity of a token bucket used +// in client-side rate limits. If no value is set, the client-side rate limiting is disabled. +// See https://aws.github.io/aws-sdk-go-v2/docs/configuring-sdk/retries-timeouts/#client-side-rate-limiting. func V2ConfigFromURLParams(ctx context.Context, q url.Values) (awsv2.Config, error) { + var endpoint string + var hostnameImmutable bool + var rateLimitCapacity int64 var opts []func(*awsv2cfg.LoadOptions) error for param, values := range q { value := values[0] switch param { + case "hostname_immutable": + var err error + hostnameImmutable, err = strconv.ParseBool(value) + if err != nil { + return awsv2.Config{}, fmt.Errorf("invalid value for hostname_immutable: %w", err) + } case "region": opts = append(opts, awsv2cfg.WithRegion(value)) case "endpoint": - customResolver := awsv2.EndpointResolverWithOptionsFunc( - func(service, region string, options ...interface{}) (awsv2.Endpoint, error) { - return awsv2.Endpoint{ - PartitionID: "aws", - URL: value, - SigningRegion: region, - }, nil - }) - opts = append(opts, awsv2cfg.WithEndpointResolverWithOptions(customResolver)) + endpoint = value case "profile": opts = append(opts, awsv2cfg.WithSharedConfigProfile(value)) + case "dualstack": + dualStack, err := strconv.ParseBool(value) + if err != nil { + return awsv2.Config{}, fmt.Errorf("invalid value for dualstack: %w", err) + } + if dualStack { + opts = append(opts, awsv2cfg.WithUseDualStackEndpoint(awsv2.DualStackEndpointStateEnabled)) + } + case "fips": + fips, err := strconv.ParseBool(value) + if err != nil { + return awsv2.Config{}, fmt.Errorf("invalid value for fips: %w", err) + } + if fips { + opts = append(opts, awsv2cfg.WithUseFIPSEndpoint(awsv2.FIPSEndpointStateEnabled)) + } + case "rate_limiter_capacity": + var err error + rateLimitCapacity, err = strconv.ParseInt(value, 10, 32) + if err != nil { + return awsv2.Config{}, fmt.Errorf("invalid value for capacity: %w", err) + } case "awssdk": // ignore, should be handled before this default: return awsv2.Config{}, fmt.Errorf("unknown query parameter %q", param) } } + if endpoint != "" { + customResolver := awsv2.EndpointResolverWithOptionsFunc( + func(service, region string, options ...interface{}) (awsv2.Endpoint, error) { + return awsv2.Endpoint{ + PartitionID: "aws", + URL: endpoint, + SigningRegion: region, + HostnameImmutable: hostnameImmutable, + }, nil + }) + opts = append(opts, awsv2cfg.WithEndpointResolverWithOptions(customResolver)) + } + + var rateLimiter retry.RateLimiter + rateLimiter = ratelimit.None + if rateLimitCapacity > 0 { + rateLimiter = ratelimit.NewTokenRateLimit(uint(rateLimitCapacity)) + } + opts = append(opts, config.WithRetryer(func() awsv2.Retryer { + return retry.NewStandard(func(so *retry.StandardOptions) { + so.RateLimiter = rateLimiter + }) + })) + return awsv2cfg.LoadDefaultConfig(ctx, opts...) } diff --git a/vendor/gocloud.dev/blob/blob.go b/vendor/gocloud.dev/blob/blob.go index b0237fcc2..59624e5de 100644 --- a/vendor/gocloud.dev/blob/blob.go +++ b/vendor/gocloud.dev/blob/blob.go @@ -514,11 +514,16 @@ func (w *Writer) uploadAndClose(r io.Reader) (err error) { // Shouldn't happen. return gcerr.Newf(gcerr.Internal, nil, "blob: uploadAndClose must be the first write") } - driverUploader, ok := w.w.(driver.Uploader) - if ok { - err = driverUploader.Upload(r) - } else { + // When ContentMD5 is being checked, we can't use Upload. + if len(w.contentMD5) > 0 { _, err = w.ReadFrom(r) + } else { + driverUploader, ok := w.w.(driver.Uploader) + if ok { + err = driverUploader.Upload(r) + } else { + _, err = w.ReadFrom(r) + } } cerr := w.Close() if err == nil && cerr != nil { @@ -688,7 +693,8 @@ var NewBucket = newBucket // function; see the package documentation for details. func newBucket(b driver.Bucket) *Bucket { return &Bucket{ - b: b, + b: b, + ioFSCallback: func() (context.Context, *ReaderOptions) { return context.Background(), nil }, tracer: &oc.Tracer{ Package: pkgName, Provider: oc.ProviderName(b), diff --git a/vendor/gocloud.dev/blob/blob_fs.go b/vendor/gocloud.dev/blob/blob_fs.go index 3b75a8e31..22b34d171 100644 --- a/vendor/gocloud.dev/blob/blob_fs.go +++ b/vendor/gocloud.dev/blob/blob_fs.go @@ -158,17 +158,15 @@ func (d *iofsDir) openOnce() error { // // fn should return a context.Context and *ReaderOptions that can be used in // calls to List and NewReader on b. It may be called more than once. +// +// If SetIOFSCallback is never called, io.FS functions will use context.Background +// and a default ReaderOptions. func (b *Bucket) SetIOFSCallback(fn func() (context.Context, *ReaderOptions)) { b.ioFSCallback = fn } // Open implements fs.FS.Open (https://pkg.go.dev/io/fs#FS). -// -// SetIOFSCallback must be called prior to calling this function. func (b *Bucket) Open(path string) (fs.File, error) { - if b.ioFSCallback == nil { - return nil, gcerr.Newf(gcerr.InvalidArgument, nil, "blob: Open -- SetIOFSCallback must be called before Open") - } if !fs.ValidPath(path) { return nil, &fs.PathError{Op: "open", Path: path, Err: fs.ErrInvalid} } diff --git a/vendor/gocloud.dev/blob/gcsblob/gcsblob.go b/vendor/gocloud.dev/blob/gcsblob/gcsblob.go index 9bc44a102..55e3091be 100644 --- a/vendor/gocloud.dev/blob/gcsblob/gcsblob.go +++ b/vendor/gocloud.dev/blob/gcsblob/gcsblob.go @@ -284,6 +284,9 @@ type Options struct { // MakeSignBytes is a factory for functions that are being used in place of an empty SignBytes. // If your implementation of 'SignBytes' needs a request context, set this instead. MakeSignBytes func(requestCtx context.Context) SignBytesFunc + + // ClientOptions are passed when constructing the storage.Client. + ClientOptions []option.ClientOption } // clear clears all the fields of o. @@ -306,6 +309,7 @@ func openBucket(ctx context.Context, client *gcp.HTTPClient, bucketName string, return nil, errors.New("gcsblob.OpenBucket: bucketName is required") } + // We wrap the provided http.Client to add a Go CDK User-Agent. clientOpts := []option.ClientOption{option.WithHTTPClient(useragent.HTTPClient(&client.Client, "blob"))} if host := os.Getenv("STORAGE_EMULATOR_HOST"); host != "" { clientOpts = []option.ClientOption{ @@ -314,15 +318,14 @@ func openBucket(ctx context.Context, client *gcp.HTTPClient, bucketName string, option.WithHTTPClient(http.DefaultClient), } } - - // We wrap the provided http.Client to add a Go CDK User-Agent. + if opts == nil { + opts = &Options{} + } + clientOpts = append(clientOpts, opts.ClientOptions...) c, err := storage.NewClient(ctx, clientOpts...) if err != nil { return nil, err } - if opts == nil { - opts = &Options{} - } return &bucket{name: bucketName, client: c, opts: opts}, nil } diff --git a/vendor/gocloud.dev/blob/s3blob/s3blob.go b/vendor/gocloud.dev/blob/s3blob/s3blob.go index 3447c9fd3..f575649b7 100644 --- a/vendor/gocloud.dev/blob/s3blob/s3blob.go +++ b/vendor/gocloud.dev/blob/s3blob/s3blob.go @@ -47,7 +47,7 @@ // - ListOptions.BeforeList: (V1) *s3.ListObjectsV2Input or *s3.ListObjectsInput // when Options.UseLegacyList == true; (V2) *s3v2.ListObjectsV2Input or *[]func(*s3v2.Options), or *s3v2.ListObjectsInput // when Options.UseLegacyList == true -// - Reader: (V1) s3.GetObjectOutput; (V2) s3v2.GetObjectInput +// - Reader: (V1) s3.GetObjectOutput; (V2) s3v2.GetObjectOutput // - ReaderOptions.BeforeRead: (V1) *s3.GetObjectInput; (V2) *s3v2.GetObjectInput or *[]func(*s3v2.Options) // - Attributes: (V1) s3.HeadObjectOutput; (V2)s3v2.HeadObjectOutput // - CopyOptions.BeforeCopy: *(V1) s3.CopyObjectInput; (V2) s3v2.CopyObjectInput @@ -130,6 +130,11 @@ const Scheme = "s3" // Use "awssdk=v1" to force using AWS SDK v1, "awssdk=v2" to force using AWS SDK v2, // or anything else to accept the default. // +// The following S3-specific query options are also supported: +// - ssetype: The type of server side encryption used (AES256, aws:kms, aws:kms:dsse) +// - kmskeyid: The KMS key ID for server side encryption +// - accelerate: A value of "true" uses the S3 Transfer Accleration endpoints +// // For V1, see gocloud.dev/aws/ConfigFromURLParams for supported query parameters // for overriding the aws.Session from the URL. // For V2, see gocloud.dev/aws/V2ConfigFromURLParams. @@ -145,8 +150,11 @@ type URLOpener struct { } const ( - sseTypeParamKey = "ssetype" - kmsKeyIdParamKey = "kmskeyid" + sseTypeParamKey = "ssetype" + kmsKeyIdParamKey = "kmskeyid" + accelerateParamKey = "accelerate" + usePathStyleParamkey = "use_path_style" + disableHTTPSParamKey = "disable_https" ) func toServerSideEncryptionType(value string) (typesv2.ServerSideEncryption, error) { @@ -178,12 +186,48 @@ func (o *URLOpener) OpenBucketURL(ctx context.Context, u *url.URL) (*blob.Bucket o.Options.KMSEncryptionID = kmsKeyID } + accelerate := false + if accelerateParam := q.Get(accelerateParamKey); accelerateParam != "" { + q.Del(accelerateParamKey) + var err error + accelerate, err = strconv.ParseBool(accelerateParam) + if err != nil { + return nil, fmt.Errorf("invalid value for %q: %v", accelerateParamKey, err) + } + } + if o.UseV2 { + opts := []func(*s3v2.Options){ + func(o *s3v2.Options) { + o.UseAccelerate = accelerate + }, + } + if disableHTTPSParam := q.Get(disableHTTPSParamKey); disableHTTPSParam != "" { + q.Del(disableHTTPSParamKey) + value, err := strconv.ParseBool(disableHTTPSParam) + if err != nil { + return nil, fmt.Errorf("invalid value for %q: %v", disableHTTPSParamKey, err) + } + opts = append(opts, func(o *s3v2.Options) { + o.EndpointOptions.DisableHTTPS = value + }) + } + if usePathStyleParam := q.Get(usePathStyleParamkey); usePathStyleParam != "" { + q.Del(usePathStyleParamkey) + value, err := strconv.ParseBool(usePathStyleParam) + if err != nil { + return nil, fmt.Errorf("invalid value for %q: %v", usePathStyleParamkey, err) + } + opts = append(opts, func(o *s3v2.Options) { + o.UsePathStyle = value + }) + } + cfg, err := gcaws.V2ConfigFromURLParams(ctx, q) if err != nil { return nil, fmt.Errorf("open bucket %v: %v", u, err) } - clientV2 := s3v2.NewFromConfig(cfg) + clientV2 := s3v2.NewFromConfig(cfg, opts...) return OpenBucketV2(ctx, clientV2, u.Host, &o.Options) } @@ -194,6 +238,8 @@ func (o *URLOpener) OpenBucketURL(ctx context.Context, u *url.URL) (*blob.Bucket if err != nil { return nil, fmt.Errorf("open bucket %v: %v", u, err) } + + overrideCfg.S3UseAccelerate = &accelerate configProvider.Configs = append(configProvider.Configs, overrideCfg) return OpenBucket(ctx, configProvider, u.Host, &o.Options) @@ -380,7 +426,6 @@ func (w *writer) open(r io.Reader, closePipeOnError bool) { if err != nil { if closePipeOnError { w.pr.CloseWithError(err) - w.pr = nil } w.err = err } @@ -971,9 +1016,6 @@ func escapeKey(key string) string { // For "../", escape the trailing slash. case i > 1 && c == '/' && r[i-1] == '.' && r[i-2] == '.': return true - // Escape \. - case c == '\\': - return true } return false }) diff --git a/vendor/gocloud.dev/internal/useragent/useragent.go b/vendor/gocloud.dev/internal/useragent/useragent.go index 4eea9344d..3ef6bac98 100644 --- a/vendor/gocloud.dev/internal/useragent/useragent.go +++ b/vendor/gocloud.dev/internal/useragent/useragent.go @@ -26,7 +26,7 @@ import ( const ( prefix = "go-cloud" - version = "0.39.0" + version = "0.40.0" ) // ClientOption returns an option.ClientOption that sets a Go CDK User-Agent. diff --git a/vendor/modules.txt b/vendor/modules.txt index 5d06987da..64fe2ab7c 100644 --- a/vendor/modules.txt +++ b/vendor/modules.txt @@ -461,7 +461,7 @@ go.opentelemetry.io/otel/metric/noop ## explicit; go 1.21 go.opentelemetry.io/otel/trace go.opentelemetry.io/otel/trace/embedded -# gocloud.dev v0.39.0 +# gocloud.dev v0.40.0 ## explicit; go 1.21.0 gocloud.dev/aws gocloud.dev/blob