From fd8ecc0eef55b7bd3fd31c893adee2214e73ca31 Mon Sep 17 00:00:00 2001 From: nabbar Date: Mon, 14 Nov 2022 16:42:31 +0100 Subject: [PATCH] Package IOUtils : - windows : cannot set the max open file number over the windows hard limit (8192) Package Request : - windows : add path function fix to not using the os.DirSeparator for url path Other : - Bump dependencies --- go.mod | 17 +++++++++-------- ioutils/fileDescriptor_ko.go | 14 ++++++++++++-- request/model.go | 24 ++++++++++++++++-------- 3 files changed, 37 insertions(+), 18 deletions(-) diff --git a/go.mod b/go.mod index 1f88b9bd..2db14483 100644 --- a/go.mod +++ b/go.mod @@ -4,10 +4,10 @@ go 1.19 require ( github.com/aws/aws-sdk-go-v2 v1.17.1 - github.com/aws/aws-sdk-go-v2/config v1.17.10 - github.com/aws/aws-sdk-go-v2/credentials v1.12.23 + github.com/aws/aws-sdk-go-v2/config v1.18.0 + github.com/aws/aws-sdk-go-v2/credentials v1.13.0 github.com/aws/aws-sdk-go-v2/service/iam v1.18.23 - github.com/aws/aws-sdk-go-v2/service/s3 v1.29.1 + github.com/aws/aws-sdk-go-v2/service/s3 v1.29.2 github.com/bits-and-blooms/bitset v1.3.3 github.com/c-bata/go-prompt v0.2.6 github.com/fatih/color v1.13.0 @@ -28,7 +28,7 @@ require ( github.com/mitchellh/go-homedir v1.1.0 github.com/nats-io/jwt/v2 v2.3.0 github.com/nats-io/nats-server/v2 v2.9.6 - github.com/nats-io/nats.go v1.19.1 + github.com/nats-io/nats.go v1.20.0 github.com/onsi/ginkgo/v2 v2.5.0 github.com/onsi/gomega v1.24.1 github.com/pelletier/go-toml v1.9.5 @@ -42,9 +42,9 @@ require ( github.com/vbauerster/mpb/v5 v5.4.0 github.com/xanzy/go-gitlab v0.74.0 github.com/xhit/go-simple-mail v2.2.2+incompatible - github.com/xujiajun/nutsdb v0.11.0 + github.com/xujiajun/nutsdb v0.11.1 github.com/xujiajun/utils v0.0.0-20220904132955-5f7c5b914235 - golang.org/x/exp v0.0.0-20221109205753-fc8884afc316 + golang.org/x/exp v0.0.0-20221111204811-129d8d6c17ab golang.org/x/net v0.2.0 golang.org/x/oauth2 v0.2.0 golang.org/x/sync v0.1.0 @@ -52,7 +52,7 @@ require ( golang.org/x/term v0.2.0 gopkg.in/yaml.v3 v3.0.1 gorm.io/driver/clickhouse v0.5.0 - gorm.io/driver/mysql v1.4.3 + gorm.io/driver/mysql v1.4.4 gorm.io/driver/postgres v1.4.5 gorm.io/driver/sqlite v1.4.3 gorm.io/driver/sqlserver v1.4.1 @@ -63,6 +63,7 @@ require ( github.com/Azure/go-ntlmssp v0.0.0-20220621081337-cb9428e4ac1e // indirect github.com/ClickHouse/ch-go v0.49.0 // indirect github.com/ClickHouse/clickhouse-go/v2 v2.3.0 // indirect + github.com/DataDog/zstd v1.5.2 // indirect github.com/Masterminds/goutils v1.1.1 // indirect github.com/Masterminds/semver v1.5.0 // indirect github.com/Masterminds/sprig v2.22.0+incompatible // indirect @@ -85,7 +86,7 @@ require ( github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.13.19 // indirect github.com/aws/aws-sdk-go-v2/service/sso v1.11.25 // indirect github.com/aws/aws-sdk-go-v2/service/ssooidc v1.13.8 // indirect - github.com/aws/aws-sdk-go-v2/service/sts v1.17.1 // indirect + github.com/aws/aws-sdk-go-v2/service/sts v1.17.2 // indirect github.com/aws/smithy-go v1.13.4 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/bwmarrin/snowflake v0.3.0 // indirect diff --git a/ioutils/fileDescriptor_ko.go b/ioutils/fileDescriptor_ko.go index 303744f5..55ed1c3e 100644 --- a/ioutils/fileDescriptor_ko.go +++ b/ioutils/fileDescriptor_ko.go @@ -33,12 +33,22 @@ import ( "github.com/nabbar/golib/ioutils/maxstdio" ) +const ( + //cf https://learn.microsoft.com/en-us/cpp/c-runtime-library/reference/setmaxstdio + winDefaultMaxStdio = 512 + winHardLimitMaxStdio = 8192 +) + func systemFileDescriptor(newValue int) (current int, max int, err Error) { rLimit := maxstdio.GetMaxStdio() if rLimit < 0 { // default windows value - rLimit = 512 + rLimit = winDefaultMaxStdio + } + + if newValue > winHardLimitMaxStdio { + newValue = winHardLimitMaxStdio } if newValue > rLimit { @@ -46,6 +56,6 @@ func systemFileDescriptor(newValue int) (current int, max int, err Error) { return SystemFileDescriptor(0) } - return rLimit, rLimit, nil + return rLimit, winHardLimitMaxStdio, nil // return 0, 0, fmt.Errorf("rLimit is nor implemented in current system") } diff --git a/request/model.go b/request/model.go index 1c2e4b7a..9cedf57b 100644 --- a/request/model.go +++ b/request/model.go @@ -35,6 +35,7 @@ import ( "io" "net/http" "net/url" + "os" "path/filepath" "runtime" "strings" @@ -445,10 +446,15 @@ func (r *request) AddPath(raw bool, path ...string) { return } + var str string + if raw { + str = strings.Replace(r.u.RawPath, "/", string(os.PathSeparator), -1) + } else { + str = strings.Replace(r.u.Path, "/", string(os.PathSeparator), -1) + } + for i := range path { - if raw && strings.HasSuffix(r.u.RawPath, "/") && strings.HasPrefix(path[i], "/") { - path[i] = strings.TrimPrefix(path[i], "/") - } else if !raw && strings.HasSuffix(r.u.Path, "/") && strings.HasPrefix(path[i], "/") { + if strings.HasSuffix(str, "/") && strings.HasPrefix(path[i], "/") { path[i] = strings.TrimPrefix(path[i], "/") } @@ -456,11 +462,13 @@ func (r *request) AddPath(raw bool, path ...string) { path[i] = strings.TrimSuffix(path[i], "/") } - if raw { - r.u.RawPath = filepath.Join(r.u.RawPath, path[i]) - } else { - r.u.Path = filepath.Join(r.u.Path, path[i]) - } + str = filepath.Join(str, path[i]) + } + + if raw { + r.u.RawPath = strings.Replace(str, string(os.PathSeparator), "/", -1) + } else { + r.u.Path = strings.Replace(str, string(os.PathSeparator), "/", -1) } }