Merge pull request #170 from nabbar/fix_2023-11-20

Fix 2023-11-21

Package Archives:
    - fix security arbitrary path
    - fix linter
    - fix issue #171 
    - fix issue #172 
    
Package AWS:
    - implement resolver v2
    - force default bucket region constraint to prevent SDK error
    
Package Cobra:
    - fix linter
    
Package Config/component:
    - fix linter
    
Package Context/Config:
    - Add function to set context
    
Package Database/KV...:
    - Fix error
    - Fix collision pointer
    - Fix models
    - Fix circular dependencies
    - Add function Delete on driver, table and item
    - Add function new on drvier to prevent collision data when create item on table get / walk
    
Package Duration:
    - Add type Duration based on time.Duration to allow transform duration to string instead of int64 nanosecond
    - Add function to parse in json, yaml, toml, text, cbor
    - Add function to allow convert type into mapstructure (spf13 viper, cobra...)
    
Package File/Perm:
    - Add type Perm based on os.FileMode to allow marshall / unmashall it into octal form instead of string representation (-rwxrwxrwx)
    - Add function to marshall / unmarshall in json, yaml, toml, text, cbor
    - Add function to allow convert type into mapstructure (spf13 viper, cobra...)
    
Package File/progress:
    - Fix linter
    
Package HTTPServer :
    - Fix linter
    - Fix security by adding a default value if not set on config
    
Package ioutils:
    - Fix Linter
    
Package LDAP:
    - Add Clone function
    - Fix linter
    
Package logger/hookfile:
    - Fix linter
    
Package nats:
    - Fix linter
    
Package Network/Protocol:
    - Fix bug with quote / Dbl Quote on unmarshall
    
Package Password:
    - Replace password with crypto rand instead of math rand
    
Package Size:
    - Fix potential overflow
    - Add function to format value into Int32, Int, Uint32, Uint, Float32
    - Add function to parse Float64 into type Size
    
Package Socket:
    - change config uint32 to golib Size, time.Duration to golib Duration
    - add TLS managment to server TCP, discard for UDP & Unix file Local Domain
    - add function Info Server to print information of server when listen is starting
    
Other:
    - bump dependencies
    - fix CVE-2023-46129

.github/workflows/go.yml

@@ -96,17 +96,17 @@ jobs:
         continue-on-error: false
         run: |
           IGNORE_BUILD=$(sed '/^[[:space:]]*$/d' "build.$(go env | grep GOARCH | cut -d'=' -f2 | tr -d '"' | tr -d "'")" | tr '\n' '|')
-          CGO_ENABLED=1 GOOS=linux GOARCH=amd64 GOAMD64=v4 go build -a -v -race -installsuffix cgo -ldflags "-w -s -extldflags '-static' " $(go list ./... | grep -vPi ${IGNORE_BUILD::-1})
+          CGO_ENABLED=1 GOOS=linux GOARCH=amd64 GOAMD64=v4 go build -a -race -installsuffix cgo -ldflags "-w -s -extldflags '-static' " $(go list ./... | grep -vPi ${IGNORE_BUILD::-1})
 
       - name: Test Build Windows/amd64 with CGO
         continue-on-error: false
         run: |
           IGNORE_BUILD=$(sed '/^[[:space:]]*$/d' "build.$(go env | grep GOARCH | cut -d'=' -f2 | tr -d '"' | tr -d "'")" | tr '\n' '|')
-          CC=/usr/bin/x86_64-w64-mingw32-gcc CGO_ENABLED=1 GOOS=windows GOARCH=amd64 GOAMD64=v4 go build -a -v -ldflags "-w -s -extldflags '-static' " $(go list ./... | grep -vPi ${IGNORE_BUILD::-1})
+          CC=/usr/bin/x86_64-w64-mingw32-gcc CGO_ENABLED=1 GOOS=windows GOARCH=amd64 GOAMD64=v4 go build -a -ldflags "-w -s -extldflags '-static' " $(go list ./... | grep -vPi ${IGNORE_BUILD::-1})
 
       - name: Test Build Darwin/arm64 with suffix/CGO
         continue-on-error: false
         run: |
           IGNORE_BUILD=$(sed '/^[[:space:]]*$/d' "build.$(go env | grep GOARCH | cut -d'=' -f2 | tr -d '"' | tr -d "'")" | tr '\n' '|')
-          CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 GOAMD64=v1 go build -a -v -ldflags "-w -s -extldflags '-static' " $(go list ./... | grep -vPi ${IGNORE_BUILD::-1})
+          CGO_ENABLED=0 GOOS=darwin GOARCH=arm64 GOAMD64=v1 go build -a -ldflags "-w -s -extldflags '-static' " $(go list ./... | grep -vPi ${IGNORE_BUILD::-1})
 

archive/bz2/reader.go

@@ -41,9 +41,10 @@ func GetFile(src io.ReadSeeker, dst io.WriteSeeker) errors.Error {
 
 	r := bzip2.NewReader(src)
 
-	//nolint #nosec
-	/* #nosec */
-	if _, e := io.Copy(dst, r); e != nil {
+	// #nosec
+	_, e := io.Copy(dst, r)
+
+	if e != nil {
 		return ErrorIOCopy.Error(e)
 	} else if _, e = dst.Seek(0, io.SeekStart); e != nil {
 		return ErrorFileSeek.Error(e)

archive/gzip/reader.go

@@ -29,9 +29,8 @@ import (
 	gz "compress/gzip"
 	"io"
 
-	libfpg "github.com/nabbar/golib/file/progress"
-
 	"github.com/nabbar/golib/errors"
+	libfpg "github.com/nabbar/golib/file/progress"
 )
 
 func GetFile(src io.ReadSeeker, dst io.WriteSeeker) errors.Error {
@@ -50,9 +49,10 @@ func GetFile(src io.ReadSeeker, dst io.WriteSeeker) errors.Error {
 		_ = r.Close()
 	}()
 
-	//nolint #nosec
-	/* #nosec */
-	if _, e = io.Copy(dst, r); e != nil {
+	// #nosec
+	_, e = io.Copy(dst, r)
+
+	if e != nil {
 		return ErrorIOCopy.Error(e)
 	} else if _, e = dst.Seek(0, io.SeekStart); e != nil {
 		return ErrorFileSeek.Error(e)
@@ -88,6 +88,7 @@ func GetGunZipSize(src io.ReadSeeker) int64 {
 	}
 
 	var n int64
+	// #nosec
 	n, e = io.Copy(io.Discard, r)
 
 	if e != nil {

archive/tar/reader.go

@@ -33,10 +33,9 @@ import (
 	"runtime"
 	"strings"
 
-	libfpg "github.com/nabbar/golib/file/progress"
-
 	libarc "github.com/nabbar/golib/archive/archive"
 	liberr "github.com/nabbar/golib/errors"
+	libfpg "github.com/nabbar/golib/file/progress"
 )
 
 func GetFile(src, dst libfpg.Progress, filenameContain, filenameRegex string) liberr.Error {
@@ -93,9 +92,11 @@ func GetAll(src io.ReadSeeker, outputFolder string, defaultDirPerm os.FileMode)
 			return ErrorTarNext.Error(e)
 		}
 
+		dst := filepath.Join(outputFolder, libarc.CleanPath(strings.Replace(h.Name, ".."+string(filepath.Separator), "", -1)))
+
 		//nolint #nosec
 		/* #nosec */
-		if err := writeContent(r, h, filepath.Join(outputFolder, libarc.CleanPath(h.Name)), defaultDirPerm); err != nil {
+		if err := writeContent(r, h, dst, defaultDirPerm); err != nil {
 			return err
 		}
 	}

archive/zip/reader.go

@@ -30,11 +30,11 @@ import (
 	"io"
 	"os"
 	"path/filepath"
-
-	libfpg "github.com/nabbar/golib/file/progress"
+	"strings"
 
 	arcmod "github.com/nabbar/golib/archive/archive"
 	liberr "github.com/nabbar/golib/errors"
+	libfpg "github.com/nabbar/golib/file/progress"
 )
 
 func GetFile(src, dst libfpg.Progress, filenameContain, filenameRegex string) liberr.Error {
@@ -119,9 +119,10 @@ func GetAll(src libfpg.Progress, outputFolder string, defaultDirPerm os.FileMode
 			continue
 		}
 
-		//nolint #nosec
-		/* #nosec */
-		if err := writeContent(f, filepath.Join(outputFolder, arcmod.CleanPath(f.Name)), defaultDirPerm); err != nil {
+		dst := filepath.Join(outputFolder, arcmod.CleanPath(strings.Replace(f.Name, ".."+string(filepath.Separator), "..", -1)))
+
+		// #nosec
+		if err := writeContent(f, dst, defaultDirPerm); err != nil {
 			return err
 		}
 	}
@@ -175,9 +176,10 @@ func writeContent(f *zip.File, out string, defaultDirPerm os.FileMode) (err libe
 		return ErrorZipFileOpen.Error(e)
 	}
 
-	//nolint #nosec
-	/* #nosec */
-	if _, e = io.Copy(dst, r); e != nil {
+	// #nosec
+	_, e = io.Copy(dst, r)
+
+	if e != nil {
 		return ErrorIOCopy.Error(e)
 	} else if e = dst.Close(); e != nil {
 		return ErrorFileClose.Error(e)

aws/bucket/bucket.go

@@ -28,6 +28,7 @@ package bucket
 import (
 	"fmt"
 
+	sdkaws "github.com/aws/aws-sdk-go-v2/aws"
 	sdksss "github.com/aws/aws-sdk-go-v2/service/s3"
 	sdkstp "github.com/aws/aws-sdk-go-v2/service/s3/types"
 	libhlp "github.com/nabbar/golib/aws/helper"
@@ -64,10 +65,12 @@ func (cli *client) _create(RegionConstraint string, lockEnable bool) error {
 
 	if RegionConstraint != "" {
 		in.CreateBucketConfiguration.LocationConstraint = sdkstp.BucketLocationConstraint(RegionConstraint)
+	} else {
+		in.CreateBucketConfiguration.LocationConstraint = sdkstp.BucketLocationConstraint(cli.GetRegion())
 	}
 
 	if lockEnable {
-		in.ObjectLockEnabledForBucket = true
+		in.ObjectLockEnabledForBucket = sdkaws.Bool(true)
 	}
 
 	out, err := cli.s3.CreateBucket(cli.GetContext(), in)

aws/model.go

@@ -118,11 +118,13 @@ func (c *client) _NewClientIAM(ctx context.Context, httpClient *http.Client) (*s
 		EndpointOptions: sdkiam.EndpointResolverOptions{
 			DisableHTTPS: !c.c.IsHTTPs(),
 		},
-		EndpointResolver: c._NewIAMResolver(cfg),
-		HTTPSignerV4:     sig,
-		Region:           cfg.Region,
-		Retryer:          ret,
-		HTTPClient:       httpClient,
+		BaseEndpoint:       sdkaws.String(c.c.GetEndpoint().String()),
+		EndpointResolver:   c._NewIAMResolver(cfg),
+		EndpointResolverV2: c._NewIAMResolverV2(c.c),
+		HTTPSignerV4:       sig,
+		Region:             cfg.Region,
+		Retryer:            ret,
+		HTTPClient:         httpClient,
 	})
 
 	return iam, nil
@@ -161,12 +163,14 @@ func (c *client) _NewClientS3(ctx context.Context, httpClient *http.Client) (*sd
 		EndpointOptions: sdksss.EndpointResolverOptions{
 			DisableHTTPS: !c.c.IsHTTPs(),
 		},
-		EndpointResolver: c._NewS3Resolver(cfg),
-		HTTPSignerV4:     sig,
-		Region:           cfg.Region,
-		Retryer:          ret,
-		HTTPClient:       httpClient,
-		UsePathStyle:     c.p,
+		BaseEndpoint:       sdkaws.String(c.c.GetEndpoint().String()),
+		EndpointResolver:   c._NewS3Resolver(cfg),
+		EndpointResolverV2: c._NewS3ResolverV2(c.c),
+		HTTPSignerV4:       sig,
+		Region:             cfg.Region,
+		Retryer:            ret,
+		HTTPClient:         httpClient,
+		UsePathStyle:       c.p,
 	})
 
 	return sss, nil

aws/multipart/copy.go

@@ -65,7 +65,7 @@ func (m *mpu) Copy(fromBucket, fromObject, fromVersionId string) error {
 			Bucket:          sdkaws.String(bck),
 			CopySource:      sdkaws.String(src),
 			Key:             sdkaws.String(obj),
-			PartNumber:      m.Counter() + 1,
+			PartNumber:      sdkaws.Int32(m.Counter() + 1),
 			UploadId:        sdkaws.String(mid),
 			CopySourceRange: sdkaws.String("bytes=" + p),
 			RequestPayer:    sdktyp.RequestPayerRequester,
@@ -121,7 +121,9 @@ func (m *mpu) getCopyPart(fromBucket, fromObject, fromVersionId string) []string
 		return res
 	} else if hdo == nil || hdo.ETag == nil || len(*hdo.ETag) < 1 {
 		return res
-	} else if size := hdo.ContentLength; size < 1 {
+	} else if s := hdo.ContentLength; s == nil {
+		return res
+	} else if size := *s; size < 1 {
 		return res
 	} else {
 		var i int64 = 0

aws/multipart/part.go

@@ -98,7 +98,7 @@ func (m *mpu) RegisterPart(etag string) {
 	m.n++
 	m.l = append(m.l, sdktyp.CompletedPart{
 		ETag:       sdkaws.String(strings.Replace(etag, "\"", "", -1)),
-		PartNumber: m.n,
+		PartNumber: sdkaws.Int32(m.n),
 	})
 }
 
@@ -154,8 +154,8 @@ func (m *mpu) AddPart(r io.Reader) (n int64, e error) {
 		Bucket:        sdkaws.String(bck),
 		Key:           sdkaws.String(obj),
 		UploadId:      sdkaws.String(mid),
-		PartNumber:    m.Counter() + 1,
-		ContentLength: n,
+		PartNumber:    sdkaws.Int32(m.Counter() + 1),
+		ContentLength: sdkaws.Int64(n),
 		Body:          tmp,
 		RequestPayer:  sdktyp.RequestPayerRequester,
 		ContentMD5:    sdkaws.String(hss),

aws/object/lifecycle.go

@@ -75,7 +75,7 @@ func (cli *client) SetRetention(object, version string, bypass bool, until time.
 	}
 
 	if bypass {
-		in.BypassGovernanceRetention = true
+		in.BypassGovernanceRetention = sdkaws.Bool(true)
 	}
 
 	switch {

aws/object/multipart.go

@@ -42,7 +42,7 @@ import (
 func (cli *client) MultipartList(keyMarker, markerId string) (uploads []sdktyp.MultipartUpload, nextKeyMarker string, nextIdMarker string, count int64, e error) {
 	in := &sdksss.ListMultipartUploadsInput{
 		Bucket:     sdkaws.String(cli.GetBucketName()),
-		MaxUploads: 1000,
+		MaxUploads: sdkaws.Int32(1000),
 	}
 
 	if keyMarker != "" && markerId != "" {
@@ -54,10 +54,17 @@ func (cli *client) MultipartList(keyMarker, markerId string) (uploads []sdktyp.M
 
 	if err != nil {
 		return nil, "", "", 0, cli.GetError(err)
-	} else if out.IsTruncated {
-		return out.Uploads, *out.NextKeyMarker, *out.NextUploadIdMarker, int64(out.MaxUploads), nil
+	}
+
+	var maxKeys int32
+	if out != nil && out.MaxUploads != nil {
+		maxKeys = *out.MaxUploads
+	}
+
+	if out != nil && out.IsTruncated != nil && *out.IsTruncated {
+		return out.Uploads, *out.NextKeyMarker, *out.NextUploadIdMarker, int64(maxKeys), nil
 	} else {
-		return out.Uploads, "", "", int64(out.MaxUploads), nil
+		return out.Uploads, "", "", int64(maxKeys), nil
 	}
 }
 

aws/object/object.go

@@ -30,9 +30,8 @@ import (
 	"mime"
 	"path/filepath"
 
-	sdksss "github.com/aws/aws-sdk-go-v2/service/s3"
-
 	sdkaws "github.com/aws/aws-sdk-go-v2/aws"
+	sdksss "github.com/aws/aws-sdk-go-v2/service/s3"
 	sdktps "github.com/aws/aws-sdk-go-v2/service/s3/types"
 	libhlp "github.com/nabbar/golib/aws/helper"
 )
@@ -58,10 +57,17 @@ func (cli *client) ListPrefix(continuationToken string, prefix string) ([]sdktps
 
 	if err != nil {
 		return nil, "", 0, cli.GetError(err)
-	} else if out.IsTruncated {
-		return out.Contents, *out.NextContinuationToken, int64(out.KeyCount), nil
+	}
+
+	var maxKeys int32
+	if out != nil && out.MaxKeys != nil {
+		maxKeys = *out.MaxKeys
+	}
+
+	if out != nil && out.IsTruncated != nil && *out.IsTruncated {
+		return out.Contents, *out.NextContinuationToken, int64(maxKeys), nil
 	} else {
-		return out.Contents, "", int64(out.KeyCount), nil
+		return out.Contents, "", int64(maxKeys), nil
 	}
 }
 
@@ -104,7 +110,7 @@ func (cli *client) WalkPrefix(prefix string, f WalkFunc) error {
 			}
 		}
 
-		if out.IsTruncated {
+		if out != nil && out.IsTruncated != nil && *out.IsTruncated {
 			t = out.NextContinuationToken
 		} else {
 			return e

aws/object/version.go

@@ -38,7 +38,7 @@ import (
 func (cli *client) VersionList(prefix, keyMarker, markerId string) (version []sdktps.ObjectVersion, delMarker []sdktps.DeleteMarkerEntry, nextKeyMarker, nextMarkerId string, count int64, err error) {
 	in := sdksss.ListObjectVersionsInput{
 		Bucket:  cli.GetBucketAws(),
-		MaxKeys: 1000,
+		MaxKeys: sdkaws.Int32(1000),
 	}
 
 	if prefix != "" {
@@ -54,10 +54,17 @@ func (cli *client) VersionList(prefix, keyMarker, markerId string) (version []sd
 
 	if e != nil {
 		return nil, nil, "", "", 0, cli.GetError(e)
-	} else if out.IsTruncated {
-		return out.Versions, out.DeleteMarkers, *out.NextKeyMarker, *out.NextVersionIdMarker, int64(out.MaxKeys), nil
+	}
+
+	var maxKeys int32
+	if out != nil && out.MaxKeys != nil {
+		maxKeys = *out.MaxKeys
+	}
+
+	if out != nil && out.IsTruncated != nil && *out.IsTruncated {
+		return out.Versions, out.DeleteMarkers, *out.NextKeyMarker, *out.NextVersionIdMarker, int64(maxKeys), nil
 	} else {
-		return out.Versions, out.DeleteMarkers, "", "", int64(out.MaxKeys), nil
+		return out.Versions, out.DeleteMarkers, "", "", int64(maxKeys), nil
 	}
 }
 
@@ -68,7 +75,7 @@ func (cli *client) VersionWalk(fv VersionWalkFunc, fd DelMakWalkFunc) error {
 func (cli *client) VersionWalkPrefix(prefix string, fv VersionWalkFunc, fd DelMakWalkFunc) error {
 	in := sdksss.ListObjectVersionsInput{
 		Bucket:  cli.GetBucketAws(),
-		MaxKeys: 1000,
+		MaxKeys: sdkaws.Int32(1000),
 	}
 
 	if prefix != "" {
@@ -117,7 +124,7 @@ func (cli *client) VersionWalkPrefix(prefix string, fv VersionWalkFunc, fd DelMa
 			}
 		}
 
-		if out.IsTruncated {
+		if out != nil && out.IsTruncated != nil && *out.IsTruncated {
 			km = out.NextKeyMarker
 			mi = out.NextVersionIdMarker
 		} else {
@@ -181,8 +188,10 @@ func (cli *client) VersionSize(object, version string) (size int64, err error) {
 
 	if h, err = cli.VersionHead(object, version); err != nil {
 		return
+	} else if h != nil && h.ContentLength != nil {
+		return *h.ContentLength, nil
 	} else {
-		return h.ContentLength, nil
+		return 0, nil
 	}
 }
 
@@ -207,7 +216,7 @@ func (cli *client) VersionDeleteLock(check bool, object, version string, byPassG
 	}
 
 	if byPassGovernance {
-		in.BypassGovernanceRetention = true
+		in.BypassGovernanceRetention = sdkaws.Bool(true)
 	}
 
 	_, err := cli.s3.DeleteObject(cli.GetContext(), &in)