Merge pull request #74 from nabbar/issue_73

 # Fix Issue #73 and other bugs:
    - Boolean value for EndpointOption
    - Optimize CustomConfig IsHTTPS function
    - Bucket Creation return
    - Bucket Create contraint Region Location
    - CustomConfig missing Signing Region
    - some AWS errors

 # Fix Github/Action :
    - Add some linter/test as blocked task

.github/workflows/go.yml

@@ -42,19 +42,12 @@ jobs:
         run: ./scripts/ci_depend
 
       - name: Check goFmt & goImport
-        continue-on-error: true
+        continue-on-error: false
         run: ./scripts/ci_format
 
       - name: Check Missing License
-        continue-on-error: true
-        run: |
-          for PKG in $(find . -type f -name "*.go" | grep -v 'vendor/' | grep -v 'version/license_.*.go');
-          do
-            if [ "$(head -n 25 $PKG | grep -c -P '[*\s]*MIT License')" != 1 ];
-            then
-              echo "Missing Licence: $PKG";
-            fi;
-          done;
+        continue-on-error: false
+        run: ./scripts/ci_license
 
       - name: Check goLinter
         continue-on-error: true
@@ -64,15 +57,15 @@ jobs:
           CGO_ENABLED: 0
 
       - name: Check goSecu + snyk.io
-        continue-on-error: true
+        continue-on-error: false
         run: ./scripts/ci_secu $SNYK_TOKEN
         env:
           GOOS: linux
           CGO_ENABLED: 0
           SNYK_TOKEN: ${{ secrets.SNYK_TOKEN }}
 
       - name: Check tests
-        continue-on-error: true
+        continue-on-error: false
         run: |
           go version
           ginkgo version
@@ -87,17 +80,22 @@ jobs:
           CGO_ENABLED: 0
 
       - name: Add ming-w32 (x86 + win64)
+        continue-on-error: false
         run: sudo apt-get install gcc-multilib gcc-mingw-w64
 
       - name: Test Build Linux/amd64 with suffix
+        continue-on-error: false
         run: CGO_ENABLED=0 GOOS=linux GOARCH=amd64 go build -a -v -installsuffix cgo -ldflags "-w -s -extldflags '-static' " ./...
 
       - name: Test Build Linux/386 with suffix
+        continue-on-error: false
         run: CGO_ENABLED=0 GOOS=linux GOARCH=386 go build -a -v -installsuffix cgo -ldflags "-w -s -extldflags '-static' " ./...
 
       - name: Test Build Windows/amd64 with CGO
+        continue-on-error: false
         run: CC=/usr/bin/x86_64-w64-mingw32-gcc CGO_ENABLED=1 GOOS=windows GOARCH=amd64 go build -a -v -ldflags "-w -s -extldflags '-static' " ./...
 
       - name: Test Build Windows/386 with CGO
+        continue-on-error: false
         run: CC=/usr/bin/i686-w64-mingw32-gcc CGO_ENABLED=1 GOOS=windows GOARCH=386 go build -a -v -ldflags "-w -s -extldflags '-static' " ./...
 

aws/bucket/bucket.go

@@ -50,9 +50,16 @@ func (cli *client) Check() liberr.Error {
 	return nil
 }
 
-func (cli *client) Create() liberr.Error {
+func (cli *client) Create(RegionConstraint string) liberr.Error {
+	cnt := &sdkstp.CreateBucketConfiguration{}
+
+	if RegionConstraint != "" {
+		cnt.LocationConstraint = sdkstp.BucketLocationConstraint(RegionConstraint)
+	}
+
 	out, err := cli.s3.CreateBucket(cli.GetContext(), &sdksss.CreateBucketInput{
-		Bucket: cli.GetBucketAws(),
+		Bucket:                    cli.GetBucketAws(),
+		CreateBucketConfiguration: cnt,
 	})
 
 	if err != nil {
@@ -61,7 +68,7 @@ func (cli *client) Create() liberr.Error {
 		return libhlp.ErrorResponse.Error(nil)
 	}
 
-	return cli.GetError(err)
+	return nil
 }
 
 func (cli *client) Delete() liberr.Error {

aws/bucket/interface.go

@@ -45,7 +45,7 @@ type Bucket interface {
 	Check() ligerr.Error
 
 	List() ([]*sdkstp.Bucket, ligerr.Error)
-	Create() ligerr.Error
+	Create(RegionConstraint string) ligerr.Error
 	Delete() ligerr.Error
 
 	//FindObject(pattern string) ([]string, errors.Error)

aws/bucket_test.go

@@ -36,7 +36,7 @@ var _ = Describe("Bucket", func() {
 			Expect(cli.Bucket().Check()).ToNot(Succeed())
 		})
 		It("Must be possible to create a bucket", func() {
-			Expect(cli.Bucket().Create()).To(Succeed())
+			Expect(cli.Bucket().Create("")).To(Succeed())
 			Expect(cli.Bucket().Check()).To(Succeed())
 		})
 	})

aws/configCustom/models.go

@@ -216,13 +216,17 @@ func (c awsModel) GetEndpoint() *url.URL {
 func (c *awsModel) ResolveEndpoint(service, region string) (sdkaws.Endpoint, error) {
 	if e, ok := c.mapRegion[region]; ok {
 		return sdkaws.Endpoint{
-			URL: strings.TrimSuffix(e.String(), "/"),
+			URL:           strings.TrimSuffix(e.String(), "/"),
+			SigningRegion: region,
+			SigningName:   service,
 		}, nil
 	}
 
 	if c.Endpoint != "" {
 		return sdkaws.Endpoint{
-			URL: strings.TrimSuffix(c.Endpoint, "/"),
+			URL:           strings.TrimSuffix(c.Endpoint, "/"),
+			SigningRegion: region,
+			SigningName:   service,
 		}, nil
 	}
 
@@ -231,7 +235,7 @@ func (c *awsModel) ResolveEndpoint(service, region string) (sdkaws.Endpoint, err
 }
 
 func (c *awsModel) IsHTTPs() bool {
-	return c.endpoint.Scheme == "https"
+	return strings.HasSuffix(strings.ToLower(c.endpoint.Scheme), "s")
 }
 
 func (c *awsModel) SetRetryer(retryer sdkaws.Retryer) {

aws/helper/partSize.go

@@ -28,6 +28,7 @@ package helper
 import (
 	"errors"
 	"io"
+	"strings"
 
 	sdkaws "github.com/aws/aws-sdk-go-v2/aws"
 	sdktps "github.com/aws/aws-sdk-go-v2/service/s3/types"
@@ -91,7 +92,7 @@ func NewReaderPartSize(rd io.Reader, p PartSize) ReaderPartSize {
 	return &readerPartSize{
 		b: rd,
 		p: p.Int64(),
-		i: 0,
+		i: 1,
 		j: 0,
 		e: false,
 		c: nil,
@@ -125,7 +126,7 @@ func (r *readerPartSize) NextPart(eTag *string) {
 	}
 
 	r.c.Parts = append(r.c.Parts, &sdktps.CompletedPart{
-		ETag:       eTag,
+		ETag:       sdkaws.String(strings.Replace(*eTag, "\"", "", -1)),
 		PartNumber: sdkaws.Int32(int32(r.i)),
 	})
 

aws/interface.go

@@ -145,7 +145,7 @@ func (cli *client) newClientIAM(httpClient *http.Client) (*sdkiam.Client, errors
 		APIOptions:  c.APIOptions,
 		Credentials: c.Credentials,
 		EndpointOptions: sdkiam.EndpointResolverOptions{
-			DisableHTTPS: cli.c.IsHTTPs(),
+			DisableHTTPS: !cli.c.IsHTTPs(),
 		},
 		EndpointResolver: sdkiam.WithEndpointResolver(c.EndpointResolver, nil),
 		HTTPSignerV4:     sdksv4.NewSigner(),
@@ -176,7 +176,7 @@ func (cli *client) newClientS3(httpClient *http.Client) (*sdksss.Client, errors.
 		APIOptions:  c.APIOptions,
 		Credentials: c.Credentials,
 		EndpointOptions: sdksss.EndpointResolverOptions{
-			DisableHTTPS: cli.c.IsHTTPs(),
+			DisableHTTPS: !cli.c.IsHTTPs(),
 		},
 		EndpointResolver: sdksss.WithEndpointResolver(c.EndpointResolver, nil),
 		HTTPSignerV4:     sdksv4.NewSigner(),

aws/object/multipart.go

@@ -26,11 +26,14 @@
 package object
 
 import (
+	"crypto/md5"
+	"encoding/base64"
 	"io"
 	"os"
 
 	sdkaws "github.com/aws/aws-sdk-go-v2/aws"
 	sdksss "github.com/aws/aws-sdk-go-v2/service/s3"
+	sdktyp "github.com/aws/aws-sdk-go-v2/service/s3/types"
 	libhlp "github.com/nabbar/golib/aws/helper"
 	liberr "github.com/nabbar/golib/errors"
 	libiou "github.com/nabbar/golib/ioutils"
@@ -95,13 +98,25 @@ func (cli *client) MultipartPutCustom(partSize libhlp.PartSize, object string, b
 			return cli.multipartCancel(err, upl.UploadId, object)
 		}
 
+		h := md5.New()
+		if _, err := tmp.WriteTo(h); err != nil {
+			return cli.multipartCancel(err, upl.UploadId, object)
+		}
+
+		_, err = tmp.Seek(0, io.SeekStart)
+		if err != nil {
+			return cli.multipartCancel(err, upl.UploadId, object)
+		}
+
 		prt, err = cli.s3.UploadPart(cli.GetContext(), &sdksss.UploadPartInput{
 			Bucket:        sdkaws.String(cli.GetBucketName()),
 			Body:          tmp,
 			PartNumber:    sdkaws.Int32(rio.CurrPart()),
 			UploadId:      upl.UploadId,
 			Key:           sdkaws.String(object),
 			ContentLength: sdkaws.Int64(inf.Size()),
+			RequestPayer:  sdktyp.RequestPayerRequester,
+			ContentMD5:    sdkaws.String(base64.StdEncoding.EncodeToString(h.Sum(nil))),
 		})
 
 		_ = tmp.Close()
@@ -118,10 +133,11 @@ func (cli *client) MultipartPutCustom(partSize libhlp.PartSize, object string, b
 
 	var prt *sdksss.CompleteMultipartUploadOutput
 	prt, err = cli.s3.CompleteMultipartUpload(cli.GetContext(), &sdksss.CompleteMultipartUploadInput{
-		UploadId:        upl.UploadId,
-		MultipartUpload: rio.CompPart(),
 		Bucket:          sdkaws.String(cli.GetBucketName()),
 		Key:             sdkaws.String(object),
+		UploadId:        upl.UploadId,
+		MultipartUpload: rio.CompPart(),
+		RequestPayer:    sdktyp.RequestPayerRequester,
 	})
 
 	if err != nil {

go.mod

@@ -46,7 +46,7 @@ require (
 	golang.org/x/net v0.0.0-20201110031124-69a78807bb2b
 	golang.org/x/oauth2 v0.0.0-20201109201403-9fd604954f58
 	golang.org/x/sync v0.0.0-20201020160332-67f06af15bc9
-	golang.org/x/sys v0.0.0-20201117222635-ba5294a509c7 // indirect
+	golang.org/x/sys v0.0.0-20201119102817-f84b799fce68 // indirect
 	golang.org/x/text v0.3.4 // indirect
 	golang.org/x/time v0.0.0-20200630173020-3af7569d3a1e // indirect
 	google.golang.org/appengine v1.6.7 // indirect