From 7fbcf39c6a8a3abaa717696757a6074c5b4483ff Mon Sep 17 00:00:00 2001 From: Nicolas JUHEL Date: Mon, 17 Jan 2022 16:30:56 +0100 Subject: [PATCH] AWS : add method to update metadata (copy object) AWS : add method for website object (update redirection & acl) AWS : fix bug with missing content-type (all defined as application/octet-stream) --- aws/object/find.go | 4 +-- aws/object/interface.go | 29 +++++++++-------- aws/object/multipart.go | 14 ++++++-- aws/object/object.go | 72 +++++++++++++++++++++++++++++++++-------- go.mod | 42 ++++++++++++------------ mail/config.go | 9 +++--- 6 files changed, 114 insertions(+), 56 deletions(-) diff --git a/aws/object/find.go b/aws/object/find.go index 4beadb64..45e316f2 100644 --- a/aws/object/find.go +++ b/aws/object/find.go @@ -28,10 +28,10 @@ package object import ( "regexp" - "github.com/nabbar/golib/errors" + liberr "github.com/nabbar/golib/errors" ) -func (cli *client) Find(pattern string) ([]string, errors.Error) { +func (cli *client) Find(pattern string) ([]string, liberr.Error) { var ( result = make([]string, 0) token = "" diff --git a/aws/object/interface.go b/aws/object/interface.go index d5f35974..84a9cc30 100644 --- a/aws/object/interface.go +++ b/aws/object/interface.go @@ -32,33 +32,36 @@ import ( sdkiam "github.com/aws/aws-sdk-go-v2/service/iam" sdksss "github.com/aws/aws-sdk-go-v2/service/s3" sdktps "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/nabbar/golib/aws/helper" - "github.com/nabbar/golib/errors" + libhlp "github.com/nabbar/golib/aws/helper" + liberr "github.com/nabbar/golib/errors" ) type client struct { - helper.Helper + libhlp.Helper iam *sdkiam.Client s3 *sdksss.Client } type Object interface { - Find(pattern string) ([]string, errors.Error) - Size(object string) (size int64, err errors.Error) + Find(pattern string) ([]string, liberr.Error) + Size(object string) (size int64, err liberr.Error) - List(continuationToken string) ([]sdktps.Object, string, int64, errors.Error) - Head(object string) (*sdksss.HeadObjectOutput, errors.Error) - Get(object string) (*sdksss.GetObjectOutput, errors.Error) - Put(object string, body io.Reader) errors.Error - Delete(object string) errors.Error + List(continuationToken string) ([]sdktps.Object, string, int64, liberr.Error) + Head(object string) (*sdksss.HeadObjectOutput, liberr.Error) + Get(object string) (*sdksss.GetObjectOutput, liberr.Error) + Put(object string, body io.Reader) liberr.Error + Delete(object string) liberr.Error - MultipartPut(object string, body io.Reader) errors.Error - MultipartPutCustom(partSize helper.PartSize, object string, body io.Reader) errors.Error + MultipartPut(object string, body io.Reader) liberr.Error + MultipartPutCustom(partSize libhlp.PartSize, object string, body io.Reader) liberr.Error + + UpdateMetadata(meta *sdksss.CopyObjectInput) liberr.Error + SetWebsite(object, redirect string) liberr.Error } func New(ctx context.Context, bucket, region string, iam *sdkiam.Client, s3 *sdksss.Client) Object { return &client{ - Helper: helper.New(ctx, bucket, region), + Helper: libhlp.New(ctx, bucket, region), iam: iam, s3: s3, } diff --git a/aws/object/multipart.go b/aws/object/multipart.go index f870afbd..cccc1553 100644 --- a/aws/object/multipart.go +++ b/aws/object/multipart.go @@ -31,6 +31,8 @@ import ( "crypto/md5" "encoding/base64" "io" + "mime" + "path/filepath" //nolint #gci "os" @@ -55,6 +57,7 @@ func (cli *client) MultipartPutCustom(partSize libhlp.PartSize, object string, b rio libhlp.ReaderPartSize upl *sdksss.CreateMultipartUploadOutput err error + tpe *string ) defer func() { @@ -63,9 +66,16 @@ func (cli *client) MultipartPutCustom(partSize libhlp.PartSize, object string, b } }() + if t := mime.TypeByExtension(filepath.Ext(object)); t == "" { + tpe = sdkaws.String("application/octet-stream") + } else { + tpe = sdkaws.String(t) + } + upl, err = cli.s3.CreateMultipartUpload(cli.GetContext(), &sdksss.CreateMultipartUploadInput{ - Key: sdkaws.String(object), - Bucket: sdkaws.String(cli.GetBucketName()), + Key: sdkaws.String(object), + Bucket: sdkaws.String(cli.GetBucketName()), + ContentType: tpe, }) if err != nil { diff --git a/aws/object/object.go b/aws/object/object.go index b0c1927b..a3050fc2 100644 --- a/aws/object/object.go +++ b/aws/object/object.go @@ -27,15 +27,17 @@ package object import ( "io" + "mime" + "path/filepath" sdkaws "github.com/aws/aws-sdk-go-v2/aws" sdksss "github.com/aws/aws-sdk-go-v2/service/s3" sdktps "github.com/aws/aws-sdk-go-v2/service/s3/types" - "github.com/nabbar/golib/aws/helper" - "github.com/nabbar/golib/errors" + libhlp "github.com/nabbar/golib/aws/helper" + liberr "github.com/nabbar/golib/errors" ) -func (cli *client) List(continuationToken string) ([]sdktps.Object, string, int64, errors.Error) { +func (cli *client) List(continuationToken string) ([]sdktps.Object, string, int64, liberr.Error) { in := sdksss.ListObjectsV2Input{ Bucket: cli.GetBucketAws(), } @@ -55,7 +57,7 @@ func (cli *client) List(continuationToken string) ([]sdktps.Object, string, int6 } } -func (cli *client) Get(object string) (*sdksss.GetObjectOutput, errors.Error) { +func (cli *client) Get(object string) (*sdksss.GetObjectOutput, liberr.Error) { out, err := cli.s3.GetObject(cli.GetContext(), &sdksss.GetObjectInput{ Bucket: cli.GetBucketAws(), Key: sdkaws.String(object), @@ -69,13 +71,13 @@ func (cli *client) Get(object string) (*sdksss.GetObjectOutput, errors.Error) { }() return nil, cli.GetError(err) } else if out.Body == nil { - return nil, helper.ErrorResponse.Error(nil) + return nil, libhlp.ErrorResponse.Error(nil) } else { return out, nil } } -func (cli *client) Head(object string) (*sdksss.HeadObjectOutput, errors.Error) { +func (cli *client) Head(object string) (*sdksss.HeadObjectOutput, liberr.Error) { out, e := cli.s3.HeadObject(cli.GetContext(), &sdksss.HeadObjectInput{ Bucket: cli.GetBucketAws(), Key: sdkaws.String(object), @@ -84,13 +86,13 @@ func (cli *client) Head(object string) (*sdksss.HeadObjectOutput, errors.Error) if e != nil { return nil, cli.GetError(e) } else if out.ETag == nil { - return nil, helper.ErrorResponse.Error(nil) + return nil, libhlp.ErrorResponse.Error(nil) } else { return out, nil } } -func (cli *client) Size(object string) (size int64, err errors.Error) { +func (cli *client) Size(object string) (size int64, err liberr.Error) { var ( h *sdksss.HeadObjectOutput ) @@ -102,23 +104,32 @@ func (cli *client) Size(object string) (size int64, err errors.Error) { } } -func (cli *client) Put(object string, body io.Reader) errors.Error { +func (cli *client) Put(object string, body io.Reader) liberr.Error { + var tpe *string + + if t := mime.TypeByExtension(filepath.Ext(object)); t == "" { + tpe = sdkaws.String("application/octet-stream") + } else { + tpe = sdkaws.String(t) + } + out, err := cli.s3.PutObject(cli.GetContext(), &sdksss.PutObjectInput{ - Bucket: cli.GetBucketAws(), - Key: sdkaws.String(object), - Body: body, + Bucket: cli.GetBucketAws(), + Key: sdkaws.String(object), + Body: body, + ContentType: tpe, }) if err != nil { return cli.GetError(err) } else if out.ETag == nil { - return helper.ErrorResponse.Error(nil) + return libhlp.ErrorResponse.Error(nil) } return nil } -func (cli *client) Delete(object string) errors.Error { +func (cli *client) Delete(object string) liberr.Error { if _, err := cli.Head(object); err != nil { return err } @@ -130,3 +141,36 @@ func (cli *client) Delete(object string) errors.Error { return cli.GetError(err) } + +func (cli *client) UpdateMetadata(meta *sdksss.CopyObjectInput) liberr.Error { + _, err := cli.s3.CopyObject(cli.GetContext(), meta) + + return cli.GetError(err) +} + +func (cli *client) SetWebsite(object, redirect string) liberr.Error { + var err error + + _, err = cli.s3.PutObjectAcl(cli.GetContext(), &sdksss.PutObjectAclInput{ + Bucket: cli.GetBucketAws(), + Key: sdkaws.String(object), + ACL: sdktps.ObjectCannedACLPublicRead, + }) + + if err != nil { + return cli.GetError(err) + } + + if redirect == "" { + return nil + } + + meta := &sdksss.CopyObjectInput{ + Bucket: cli.GetBucketAws(), + CopySource: sdkaws.String(cli.GetBucketName() + "/" + object), + Key: sdkaws.String(object), + WebsiteRedirectLocation: sdkaws.String(redirect), + } + + return cli.UpdateMetadata(meta) +} diff --git a/go.mod b/go.mod index 5a70bce9..aee76219 100644 --- a/go.mod +++ b/go.mod @@ -3,11 +3,11 @@ module github.com/nabbar/golib go 1.17 require ( - github.com/aws/aws-sdk-go-v2 v1.12.0 - github.com/aws/aws-sdk-go-v2/config v1.12.0 - github.com/aws/aws-sdk-go-v2/credentials v1.7.0 - github.com/aws/aws-sdk-go-v2/service/iam v1.15.0 - github.com/aws/aws-sdk-go-v2/service/s3 v1.23.0 + github.com/aws/aws-sdk-go-v2 v1.13.0 + github.com/aws/aws-sdk-go-v2/config v1.13.0 + github.com/aws/aws-sdk-go-v2/credentials v1.8.0 + github.com/aws/aws-sdk-go-v2/service/iam v1.16.0 + github.com/aws/aws-sdk-go-v2/service/s3 v1.24.0 github.com/c-bata/go-prompt v0.2.6 github.com/fatih/color v1.13.0 github.com/fxamacker/cbor/v2 v2.4.0 @@ -20,11 +20,11 @@ require ( github.com/hashicorp/go-retryablehttp v0.7.0 github.com/hashicorp/go-uuid v1.0.2 github.com/hashicorp/go-version v1.4.0 - github.com/lni/dragonboat/v3 v3.3.4 + github.com/lni/dragonboat/v3 v3.3.5 github.com/matcornic/hermes/v2 v2.1.0 github.com/mattn/go-colorable v0.1.12 - github.com/nats-io/jwt/v2 v2.2.0 - github.com/nats-io/nats-server/v2 v2.6.6 + github.com/nats-io/jwt/v2 v2.2.1-0.20220113022732-58e87895b296 + github.com/nats-io/nats-server/v2 v2.7.0 github.com/nats-io/nats.go v1.13.1-0.20211122170419-d7c1d78a50fc github.com/onsi/ginkgo/v2 v2.0.0 github.com/onsi/gomega v1.17.0 @@ -36,10 +36,10 @@ require ( github.com/xhit/go-simple-mail v2.2.2+incompatible github.com/xujiajun/nutsdb v0.6.0 github.com/xujiajun/utils v0.0.0-20190123093513-8bf096c4f53b - golang.org/x/net v0.0.0-20220111093109-d55c255bac03 + golang.org/x/net v0.0.0-20220114011407-0dd24b26b47d golang.org/x/oauth2 v0.0.0-20211104180415-d3ed0bb246c8 golang.org/x/sync v0.0.0-20210220032951-036812b2e83c - golang.org/x/sys v0.0.0-20220111092808-5a964db01320 + golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 golang.org/x/term v0.0.0-20210927222741-03fcf44c2211 ) @@ -55,17 +55,17 @@ require ( github.com/andybalholm/cascadia v1.3.1 // indirect github.com/aokoli/goutils v1.1.1 // indirect github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da // indirect - github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.1.0 // indirect - github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.9.0 // indirect - github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.3 // indirect - github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.1.0 // indirect - github.com/aws/aws-sdk-go-v2/internal/ini v1.3.3 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.6.0 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.6.0 // indirect - github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.10.0 // indirect - github.com/aws/aws-sdk-go-v2/service/sso v1.8.0 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.13.0 // indirect - github.com/aws/smithy-go v1.9.1 // indirect + github.com/aws/aws-sdk-go-v2/aws/protocol/eventstream v1.2.0 // indirect + github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.10.0 // indirect + github.com/aws/aws-sdk-go-v2/internal/configsources v1.1.4 // indirect + github.com/aws/aws-sdk-go-v2/internal/endpoints/v2 v2.2.0 // indirect + github.com/aws/aws-sdk-go-v2/internal/ini v1.3.4 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.7.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.7.0 // indirect + github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.11.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sso v1.9.0 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.14.0 // indirect + github.com/aws/smithy-go v1.10.0 // indirect github.com/bwmarrin/snowflake v0.3.0 // indirect github.com/cockroachdb/errors v1.7.5 // indirect github.com/cockroachdb/logtags v0.0.0-20190617123548-eb05cc24525f // indirect diff --git a/mail/config.go b/mail/config.go index 181c1bb5..dcecf61f 100644 --- a/mail/config.go +++ b/mail/config.go @@ -28,10 +28,11 @@ package mail import ( "fmt" + "net/textproto" + libval "github.com/go-playground/validator/v10" liberr "github.com/nabbar/golib/errors" - "github.com/nabbar/golib/ioutils" - "net/textproto" + libiot "github.com/nabbar/golib/ioutils" ) type Config struct { @@ -140,7 +141,7 @@ func (c Config) NewMailer() (Mail, liberr.Error) { if len(c.Attach) > 0 { for _, f := range c.Attach { - if h, e := ioutils.NewFileProgressPathRead(f.Path, 0); e != nil { + if h, e := libiot.NewFileProgressPathRead(f.Path, 0); e != nil { return nil, e } else { m.AddAttachment(f.Name, f.Mime, h, false) @@ -150,7 +151,7 @@ func (c Config) NewMailer() (Mail, liberr.Error) { if len(c.Inline) > 0 { for _, f := range c.Inline { - if h, e := ioutils.NewFileProgressPathRead(f.Path, 0); e != nil { + if h, e := libiot.NewFileProgressPathRead(f.Path, 0); e != nil { return nil, e } else { m.AddAttachment(f.Name, f.Mime, h, true)