diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 959f312c..85ce866b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -69,6 +69,7 @@ jobs: name: Test GatewayD Plugins runs-on: ubuntu-latest needs: test + timeout-minutes: 3 services: postgres: image: postgres diff --git a/.gitignore b/.gitignore index 1934d8c4..5e633aab 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ dist/ # Editor files .vscode +.idea # Logs *.log diff --git a/Makefile b/Makefile index b66c1e09..1e1fb437 100644 --- a/Makefile +++ b/Makefile @@ -110,6 +110,9 @@ clean: test: @go test -v ./... +test-race: + @go test -race -v ./... + benchmark: @go test -bench=. -benchmem -run=^# ./... diff --git a/cmd/run.go b/cmd/run.go index 4d7bb407..286ec85f 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -30,7 +30,6 @@ import ( usage "github.com/gatewayd-io/gatewayd/usagereport/v1" "github.com/getsentry/sentry-go" "github.com/go-co-op/gocron" - "github.com/panjf2000/gnet/v2" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/rs/zerolog" @@ -71,7 +70,6 @@ var ( func StopGracefully( runCtx context.Context, - pluginTimeoutCtx context.Context, sig os.Signal, metricsMerger *metrics.Merger, metricsServer *http.Server, @@ -88,6 +86,10 @@ func StopGracefully( logger.Info().Msg("Notifying the plugins that the server is shutting down") if pluginRegistry != nil { + pluginTimeoutCtx, cancel := context.WithTimeout(context.Background(), conf.Plugin.Timeout) + defer cancel() + + //nolint:contextcheck _, err := pluginRegistry.Run( pluginTimeoutCtx, map[string]interface{}{"signal": signal}, @@ -99,11 +101,12 @@ func StopGracefully( } } - logger.Info().Msg("Stopping GatewayD") - span.AddEvent("Stopping GatewayD", trace.WithAttributes( + logger.Info().Msg("GatewayD is shutting down") + span.AddEvent("GatewayD is shutting down", trace.WithAttributes( attribute.String("signal", signal), )) if healthCheckScheduler != nil { + healthCheckScheduler.Stop() healthCheckScheduler.Clear() logger.Info().Msg("Stopped health check scheduler") span.AddEvent("Stopped health check scheduler") @@ -269,7 +272,7 @@ var runCmd = &cobra.Command{ startDelay := time.Now().Add(conf.Plugin.HealthCheckPeriod) if _, err := healthCheckScheduler.Every( conf.Plugin.HealthCheckPeriod).SingletonMode().StartAt(startDelay).Do(func() { - _, span = otel.Tracer(config.TracerName).Start(ctx, "Run plugin health check") + _, span := otel.Tracer(config.TracerName).Start(ctx, "Run plugin health check") defer span.End() var plugins []string @@ -461,6 +464,9 @@ var runCmd = &cobra.Command{ }(conf.Global.Metrics[config.Default], logger) // This is a notification hook, so we don't care about the result. + pluginTimeoutCtx, cancel = context.WithTimeout(context.Background(), conf.Plugin.Timeout) + defer cancel() + if data, ok := conf.GlobalKoanf.Get("loggers").(map[string]interface{}); ok { _, err = pluginRegistry.Run( pluginTimeoutCtx, data, v1.HookName_HOOK_NAME_ON_NEW_LOGGER) @@ -527,6 +533,10 @@ var runCmd = &cobra.Command{ span.AddEvent("Create client", eventOptions) + pluginTimeoutCtx, cancel = context.WithTimeout( + context.Background(), conf.Plugin.Timeout) + defer cancel() + clientCfg := map[string]interface{}{ "id": client.ID, "network": client.Network, @@ -571,6 +581,10 @@ var runCmd = &cobra.Command{ os.Exit(gerr.FailedToInitializePool) } + pluginTimeoutCtx, cancel = context.WithTimeout( + context.Background(), conf.Plugin.Timeout) + defer cancel() + _, err = pluginRegistry.Run( pluginTimeoutCtx, map[string]interface{}{"name": name, "size": cfg.GetSize()}, @@ -610,6 +624,10 @@ var runCmd = &cobra.Command{ attribute.String("healthCheckPeriod", cfg.HealthCheckPeriod.String()), )) + pluginTimeoutCtx, cancel = context.WithTimeout( + context.Background(), conf.Plugin.Timeout) + defer cancel() + if data, ok := conf.GlobalKoanf.Get("proxies").(map[string]interface{}); ok { _, err = pluginRegistry.Run( pluginTimeoutCtx, data, v1.HookName_HOOK_NAME_ON_NEW_PROXY) @@ -633,30 +651,9 @@ var runCmd = &cobra.Command{ cfg.Network, cfg.Address, cfg.GetTickInterval(), - []gnet.Option{ - // Scheduling options - gnet.WithMulticore(cfg.MultiCore), - gnet.WithLockOSThread(cfg.LockOSThread), - // NumEventLoop overrides Multicore option. - // gnet.WithNumEventLoop(1), - + network.Option{ // Can be used to send keepalive messages to the client. - gnet.WithTicker(cfg.EnableTicker), - - // Internal event-loop load balancing options - gnet.WithLoadBalancing(cfg.GetLoadBalancer()), - - // Buffer options - gnet.WithReadBufferCap(cfg.ReadBufferCap), - gnet.WithWriteBufferCap(cfg.WriteBufferCap), - gnet.WithSocketRecvBuffer(cfg.SocketRecvBuffer), - gnet.WithSocketSendBuffer(cfg.SocketSendBuffer), - - // TCP options - gnet.WithReuseAddr(cfg.ReuseAddress), - gnet.WithReusePort(cfg.ReusePort), - gnet.WithTCPKeepAlive(cfg.TCPKeepAlive), - gnet.WithTCPNoDelay(cfg.GetTCPNoDelay()), + EnableTicker: cfg.EnableTicker, }, proxies[name], logger, @@ -669,21 +666,13 @@ var runCmd = &cobra.Command{ attribute.String("network", cfg.Network), attribute.String("address", cfg.Address), attribute.String("tickInterval", cfg.TickInterval.String()), - attribute.Bool("multiCore", cfg.MultiCore), - attribute.Bool("lockOSThread", cfg.LockOSThread), - attribute.Bool("enableTicker", cfg.EnableTicker), - attribute.String("loadBalancer", cfg.LoadBalancer), - attribute.Int("readBufferCap", cfg.ReadBufferCap), - attribute.Int("writeBufferCap", cfg.WriteBufferCap), - attribute.Int("socketRecvBuffer", cfg.SocketRecvBuffer), - attribute.Int("socketSendBuffer", cfg.SocketSendBuffer), - attribute.Bool("reuseAddress", cfg.ReuseAddress), - attribute.Bool("reusePort", cfg.ReusePort), - attribute.String("tcpKeepAlive", cfg.TCPKeepAlive.String()), - attribute.Bool("tcpNoDelay", cfg.TCPNoDelay), attribute.String("pluginTimeout", conf.Plugin.Timeout.String()), )) + pluginTimeoutCtx, cancel = context.WithTimeout( + context.Background(), conf.Plugin.Timeout) + defer cancel() + if data, ok := conf.GlobalKoanf.Get("servers").(map[string]interface{}); ok { _, err = pluginRegistry.Run( pluginTimeoutCtx, data, v1.HookName_HOOK_NAME_ON_NEW_SERVER) @@ -786,13 +775,15 @@ var runCmd = &cobra.Command{ go func(pluginRegistry *plugin.Registry, logger zerolog.Logger, servers map[string]*network.Server, + metricsMerger *metrics.Merger, + metricsServer *http.Server, + stopChan chan struct{}, ) { for sig := range signalsCh { for _, s := range signals { if sig != s { StopGracefully( runCtx, - pluginTimeoutCtx, sig, metricsMerger, metricsServer, @@ -805,13 +796,14 @@ var runCmd = &cobra.Command{ } } } - }(pluginRegistry, logger, servers) + }(pluginRegistry, logger, servers, metricsMerger, metricsServer, stopChan) _, span = otel.Tracer(config.TracerName).Start(runCtx, "Start servers") // Start the server. for name, server := range servers { logger := loggers[name] go func( + span trace.Span, server *network.Server, logger zerolog.Logger, healthCheckScheduler *gocron.Scheduler, @@ -831,7 +823,7 @@ var runCmd = &cobra.Command{ pluginRegistry.Shutdown() os.Exit(gerr.FailedToStartServer) } - }(server, logger, healthCheckScheduler, metricsMerger, pluginRegistry) + }(span, server, logger, healthCheckScheduler, metricsMerger, pluginRegistry) } span.End() diff --git a/cmd/run_test.go b/cmd/run_test.go index 0969e6e8..40761ca6 100644 --- a/cmd/run_test.go +++ b/cmd/run_test.go @@ -1,6 +1,7 @@ package cmd import ( + "context" "os" "sync" "testing" @@ -29,8 +30,7 @@ func Test_runCmd(t *testing.T) { time.Sleep(100 * time.Millisecond) StopGracefully( - runCmd.Context(), - runCmd.Context(), + context.Background(), nil, nil, nil, @@ -88,8 +88,7 @@ func Test_runCmdWithMultiTenancy(t *testing.T) { time.Sleep(500 * time.Millisecond) StopGracefully( - runCmd.Context(), - runCmd.Context(), + context.Background(), nil, nil, nil, @@ -165,11 +164,10 @@ func Test_runCmdWithCachePlugin(t *testing.T) { var waitGroup sync.WaitGroup waitGroup.Add(1) go func(waitGroup *sync.WaitGroup) { - time.Sleep(500 * time.Millisecond) + time.Sleep(time.Second) StopGracefully( - runCmd.Context(), - runCmd.Context(), + context.Background(), nil, nil, nil, diff --git a/config/config.go b/config/config.go index e2d38f9b..34b97279 100644 --- a/config/config.go +++ b/config/config.go @@ -129,21 +129,10 @@ func (c *Config) LoadDefaults(ctx context.Context) { } defaultServer := Server{ - Network: DefaultListenNetwork, - Address: DefaultListenAddress, - EnableTicker: false, - TickInterval: DefaultTickInterval, - MultiCore: true, - LockOSThread: false, - ReuseAddress: true, - ReusePort: true, - LoadBalancer: DefaultLoadBalancer, - ReadBufferCap: DefaultBufferSize, - WriteBufferCap: DefaultBufferSize, - SocketRecvBuffer: DefaultBufferSize, - SocketSendBuffer: DefaultBufferSize, - TCPKeepAlive: DefaultTCPKeepAliveDuration, - TCPNoDelay: DefaultTCPNoDelay, + Network: DefaultListenNetwork, + Address: DefaultListenAddress, + EnableTicker: false, + TickInterval: DefaultTickInterval, } c.globalDefaults = GlobalConfig{ diff --git a/config/constants.go b/config/constants.go index a40175e8..9cbafb4e 100644 --- a/config/constants.go +++ b/config/constants.go @@ -117,6 +117,7 @@ const ( DefaultTCPKeepAliveDuration = 3 * time.Second DefaultLoadBalancer = "roundrobin" DefaultTCPNoDelay = true + DefaultEngineStopTimeout = 5 * time.Second // Utility constants. DefaultSeed = 1000 diff --git a/config/getters.go b/config/getters.go index 11a8aa3b..0d09ee35 100644 --- a/config/getters.go +++ b/config/getters.go @@ -5,7 +5,6 @@ import ( "path/filepath" "time" - "github.com/panjf2000/gnet/v2" "github.com/rs/zerolog" ) @@ -28,11 +27,6 @@ var ( "continue": Continue, "stop": Stop, } - loadBalancers = map[string]gnet.LoadBalancing{ - "roundrobin": gnet.RoundRobin, - "leastconnections": gnet.LeastConnections, - "sourceaddrhash": gnet.SourceAddrHash, - } logOutputs = map[string]LogOutput{ "console": Console, "stdout": Stdout, @@ -166,23 +160,6 @@ func (s Server) GetTickInterval() time.Duration { return s.TickInterval } -// GetLoadBalancer returns the load balancing algorithm to use. -func (s Server) GetLoadBalancer() gnet.LoadBalancing { - if lb, ok := loadBalancers[s.LoadBalancer]; ok { - return lb - } - return gnet.RoundRobin -} - -// GetTCPNoDelay returns the TCP no delay option from config file. -func (s Server) GetTCPNoDelay() gnet.TCPSocketOpt { - if s.TCPNoDelay { - return gnet.TCPNoDelay - } - - return gnet.TCPDelay -} - // GetSize returns the pool size from config file. func (p Pool) GetSize() int { if p.Size == 0 { diff --git a/config/getters_test.go b/config/getters_test.go index 2d485c39..4c3d6b71 100644 --- a/config/getters_test.go +++ b/config/getters_test.go @@ -4,7 +4,6 @@ import ( "testing" "time" - "github.com/panjf2000/gnet/v2" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" ) @@ -75,18 +74,6 @@ func TestGetTickInterval(t *testing.T) { assert.Equal(t, DefaultTickInterval, server.GetTickInterval()) } -// TestGetLoadBalancer tests the GetLoadBalancer function. -func TestGetLoadBalancer(t *testing.T) { - server := Server{} - assert.Equal(t, gnet.RoundRobin, server.GetLoadBalancer()) -} - -// TestGetTCPNoDelay tests the GetTCPNoDelay function. -func TestGetTCPNoDelay(t *testing.T) { - server := Server{} - assert.Equal(t, gnet.TCPDelay, server.GetTCPNoDelay()) -} - // TestGetSize tests the GetSize function. func TestGetSize(t *testing.T) { pool := Pool{} diff --git a/config/types.go b/config/types.go index 5924e855..d1a499b3 100644 --- a/config/types.go +++ b/config/types.go @@ -5,16 +5,6 @@ import ( "time" ) -// // getPath returns the path to the referenced config value. -// func getPath(cfg *koanf.Koanf, path string) string { -// ref := cfg.String(path) -// if cfg.Exists(path) && cfg.StringMap(ref) != nil { -// return ref -// } - -// return path -// } - type Plugin struct { Name string `json:"name" jsonschema:"required"` Enabled bool `json:"enabled"` @@ -88,21 +78,10 @@ type Proxy struct { } type Server struct { - EnableTicker bool `json:"enableTicker"` - MultiCore bool `json:"multiCore"` - LockOSThread bool `json:"lockOSThread"` //nolint:tagliatelle - ReuseAddress bool `json:"reuseAddress"` - ReusePort bool `json:"reusePort"` - TCPNoDelay bool `json:"tcpNoDelay"` - ReadBufferCap int `json:"readBufferCap"` - WriteBufferCap int `json:"writeBufferCap"` - SocketRecvBuffer int `json:"socketRecvBuffer"` - SocketSendBuffer int `json:"socketSendBuffer"` - TCPKeepAlive time.Duration `json:"tcpKeepAlive" jsonschema:"oneof_type=string;integer"` - TickInterval time.Duration `json:"tickInterval" jsonschema:"oneof_type=string;integer"` - Network string `json:"network" jsonschema:"enum=tcp,enum=udp,enum=unix"` - Address string `json:"address"` - LoadBalancer string `json:"loadBalancer" jsonschema:"enum=roundrobin,enum=leastconnections,enum=sourceaddrhash"` + EnableTicker bool `json:"enableTicker"` + TickInterval time.Duration `json:"tickInterval" jsonschema:"oneof_type=string;integer"` + Network string `json:"network" jsonschema:"enum=tcp,enum=udp,enum=unix"` + Address string `json:"address"` } type API struct { diff --git a/errors/errors.go b/errors/errors.go index d884d184..e5b1c1a5 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -9,7 +9,6 @@ const ( ErrCodeNetworkNotSupported ErrCodeResolveFailed ErrCodePoolExhausted - ErrCodeStartServerFailed ErrCodePluginNotFound ErrCodePluginNotReady ErrCodeStartPluginFailed @@ -21,7 +20,12 @@ const ( ErrCodeClientSendFailed ErrCodeServerReceiveFailed ErrCodeServerSendFailed + ErrCodeServerListenFailed + ErrCodeSplitHostPortFailed + ErrCodeAcceptFailed + ErrCodeReadFailed ErrCodePutFailed + ErrCodeNilPointer ErrCodeCastFailed ErrCodeHookVerificationFailed ErrCodeHookReturnedError @@ -50,8 +54,6 @@ var ( ErrCodeResolveFailed, "failed to resolve address", nil) ErrPoolExhausted = NewGatewayDError( ErrCodePoolExhausted, "pool is exhausted", nil) - ErrFailedToStartServer = NewGatewayDError( - ErrCodeStartServerFailed, "failed to start server", nil) ErrPluginNotFound = NewGatewayDError( ErrCodePluginNotFound, "plugin not found", nil) @@ -77,9 +79,20 @@ var ( ErrCodeServerSendFailed, "couldn't send data to the client", nil) ErrServerReceiveFailed = NewGatewayDError( ErrCodeServerReceiveFailed, "couldn't receive data from the client", nil) + ErrServerListenFailed = NewGatewayDError( + ErrCodeServerListenFailed, "couldn't listen on the server", nil) + ErrSplitHostPortFailed = NewGatewayDError( + ErrCodeSplitHostPortFailed, "failed to split host:port", nil) + ErrAcceptFailed = NewGatewayDError( + ErrCodeAcceptFailed, "failed to accept connection", nil) + + ErrReadFailed = NewGatewayDError( + ErrCodeReadFailed, "failed to read from the client", nil) ErrPutFailed = NewGatewayDError( ErrCodePutFailed, "failed to put in pool", nil) + ErrNilPointer = NewGatewayDError( + ErrCodeNilPointer, "nil pointer", nil) ErrCastFailed = NewGatewayDError( ErrCodeCastFailed, "failed to cast", nil) diff --git a/gatewayd.yaml b/gatewayd.yaml index f0032f2d..d30880dc 100644 --- a/gatewayd.yaml +++ b/gatewayd.yaml @@ -56,17 +56,6 @@ servers: address: 0.0.0.0:15432 enableTicker: False tickInterval: 5s # duration - multiCore: True - lockOSThread: False - loadBalancer: roundrobin - readBufferCap: 134217728 - writeBufferCap: 134217728 - socketRecvBuffer: 134217728 - socketSendBuffer: 134217728 - reuseAddress: True - reusePort: True - tcpKeepAlive: 3s # duration - tcpNoDelay: True api: enabled: True diff --git a/go.mod b/go.mod index 1d614dc6..135b4932 100644 --- a/go.mod +++ b/go.mod @@ -8,44 +8,45 @@ require ( github.com/codingsince1985/checksum v1.3.0 github.com/envoyproxy/protoc-gen-validate v1.0.2 github.com/gatewayd-io/gatewayd-plugin-sdk v0.1.1 - github.com/getsentry/sentry-go v0.24.1 - github.com/go-co-op/gocron v1.34.0 - github.com/google/go-cmp v0.5.9 + github.com/getsentry/sentry-go v0.25.0 + github.com/go-co-op/gocron v1.35.2 + github.com/google/go-cmp v0.6.0 github.com/google/go-github/v53 v53.2.0 github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0 github.com/hashicorp/go-hclog v1.5.0 - github.com/hashicorp/go-plugin v1.5.1 - github.com/invopop/jsonschema v0.8.0 + github.com/hashicorp/go-plugin v1.5.2 + github.com/invopop/jsonschema v0.12.0 github.com/knadh/koanf v1.5.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/panjf2000/gnet/v2 v2.3.2 - github.com/prometheus/client_golang v1.16.0 - github.com/prometheus/client_model v0.4.0 + github.com/prometheus/client_golang v1.17.0 + github.com/prometheus/client_model v0.5.0 github.com/prometheus/common v0.44.0 - github.com/rs/zerolog v1.30.0 + github.com/rs/zerolog v1.31.0 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.8.4 github.com/zenizh/go-capturer v0.0.0-20211219060012-52ea6c8fed04 - go.opentelemetry.io/otel v1.18.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.18.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.18.0 - go.opentelemetry.io/otel/sdk v1.18.0 - go.opentelemetry.io/otel/trace v1.18.0 - golang.org/x/exp v0.0.0-20230905200255-921286631fa9 - google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb - google.golang.org/grpc v1.58.1 + go.opentelemetry.io/otel v1.19.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 + go.opentelemetry.io/otel/sdk v1.19.0 + go.opentelemetry.io/otel/trace v1.19.0 + golang.org/x/exp v0.0.0-20231006140011-7918f672742d + google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b + google.golang.org/grpc v1.58.3 google.golang.org/protobuf v1.31.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v3 v3.0.1 ) require ( - github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect + github.com/ProtonMail/go-crypto v0.0.0-20230923063757-afb1ddc0824c // indirect + github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/buger/jsonparser v1.1.1 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/cloudflare/circl v1.3.3 // indirect + github.com/cloudflare/circl v1.3.5 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/fatih/color v1.15.0 // indirect github.com/fatih/structs v1.1.0 // indirect @@ -56,33 +57,30 @@ require ( github.com/google/go-querystring v1.1.0 // indirect github.com/google/uuid v1.3.1 // indirect github.com/hashicorp/yamux v0.1.1 // indirect - github.com/iancoleman/orderedmap v0.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/oklog/run v1.1.0 // indirect - github.com/pelletier/go-toml v1.9.4 // indirect + github.com/pelletier/go-toml v1.9.5 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/procfs v0.11.1 // indirect + github.com/prometheus/procfs v0.12.0 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/valyala/bytebufferpool v1.0.0 // indirect - go.opentelemetry.io/otel/metric v1.18.0 // indirect + github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect + go.opentelemetry.io/otel/metric v1.19.0 // indirect go.opentelemetry.io/proto/otlp v1.0.0 // indirect go.uber.org/atomic v1.11.0 // indirect - go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.26.0 // indirect - golang.org/x/crypto v0.13.0 // indirect - golang.org/x/net v0.15.0 // indirect - golang.org/x/oauth2 v0.12.0 // indirect - golang.org/x/sync v0.3.0 // indirect - golang.org/x/sys v0.12.0 // indirect + golang.org/x/crypto v0.14.0 // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/oauth2 v0.13.0 // indirect + golang.org/x/sys v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230913181813-007df8e322eb // indirect + google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect ) diff --git a/go.sum b/go.sum index 83411919..a198162b 100644 --- a/go.sum +++ b/go.sum @@ -5,8 +5,8 @@ github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0 github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= -github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 h1:kkhsdkhsCvIsutKu5zLMgWtgh9YxGCNAw8Ad8hjwfYg= -github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= +github.com/ProtonMail/go-crypto v0.0.0-20230923063757-afb1ddc0824c h1:kMFnB0vCcX7IL/m9Y5LO+KQYv+t1CQOiFe6+SV2J7bE= +github.com/ProtonMail/go-crypto v0.0.0-20230923063757-afb1ddc0824c/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -27,6 +27,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.2/go.mod h1:72H github.com/aws/aws-sdk-go-v2/service/sso v1.4.2/go.mod h1:NBvT9R1MEF+Ud6ApJKM0G+IkPchKS7p7c2YPKwHmBOk= github.com/aws/aws-sdk-go-v2/service/sts v1.7.2/go.mod h1:8EzeIqfWt2wWT4rJVu3f21TfrhJ8AEMzVybRNSb/b4g= github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -34,6 +36,8 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA= github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= @@ -42,8 +46,9 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs= github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= +github.com/cloudflare/circl v1.3.5 h1:g+wWynZqVALYAlpSQFAa7TscDnUK8mKYtrxMpw6AUKo= +github.com/cloudflare/circl v1.3.5/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/codingsince1985/checksum v1.3.0 h1:kqqIqWBwjidGmt/pO4yXCEX+np7HACGx72EB+MkKcVY= @@ -76,11 +81,11 @@ github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/gatewayd-io/gatewayd-plugin-sdk v0.1.1 h1:ujlh6TSDFmG2e9VtY6hZk8BW0p7i0AEQL3BpMMLzxwc= github.com/gatewayd-io/gatewayd-plugin-sdk v0.1.1/go.mod h1:B4oWVHf7NeSCs7szN8nrlIO6tkznV1F3ZMqE9VxDtKY= -github.com/getsentry/sentry-go v0.24.1 h1:W6/0GyTy8J6ge6lVCc94WB6Gx2ZuLrgopnn9w8Hiwuk= -github.com/getsentry/sentry-go v0.24.1/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= +github.com/getsentry/sentry-go v0.25.0 h1:q6Eo+hS+yoJlTO3uu/azhQadsD8V+jQn2D8VvX1eOyI= +github.com/getsentry/sentry-go v0.25.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-co-op/gocron v1.34.0 h1:/rcOZjJWUYnGR0ZDKozPXEnJ+wJt220FSLo2/hxZvV0= -github.com/go-co-op/gocron v1.34.0/go.mod h1:NLi+bkm4rRSy1F8U7iacZOz0xPseMoIOnvabGoSe/no= +github.com/go-co-op/gocron v1.35.2 h1:lG3rdA9TqBBC/PtT2ukQqgLm6jEepnAzz3+OQetvPTE= +github.com/go-co-op/gocron v1.35.2/go.mod h1:NLi+bkm4rRSy1F8U7iacZOz0xPseMoIOnvabGoSe/no= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -132,8 +137,8 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github/v53 v53.2.0 h1:wvz3FyF53v4BK+AsnvCmeNhf8AkTaeh2SoYu/XUvTtI= github.com/google/go-github/v53 v53.2.0/go.mod h1:XhFRObz+m/l+UCm9b7KSIC3lT3NWSXGt7mOsAWEloao= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= @@ -161,8 +166,8 @@ github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iP github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-plugin v1.0.1/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY= -github.com/hashicorp/go-plugin v1.5.1 h1:oGm7cWBaYIp3lJpx1RUEfLWophprE2EV/KUeqBYo+6k= -github.com/hashicorp/go-plugin v1.5.1/go.mod h1:w1sAEES3g3PuV/RzUrgow20W2uErMly84hhD3um1WL4= +github.com/hashicorp/go-plugin v1.5.2 h1:aWv8eimFqWlsEiMrYZdPYl+FdHaBJSN4AWwGWfT1G2Y= +github.com/hashicorp/go-plugin v1.5.2/go.mod h1:w1sAEES3g3PuV/RzUrgow20W2uErMly84hhD3um1WL4= github.com/hashicorp/go-retryablehttp v0.5.4/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-rootcerts v1.0.1/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= @@ -188,19 +193,17 @@ github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= github.com/hjson/hjson-go/v4 v4.0.0 h1:wlm6IYYqHjOdXH1gHev4VoXCaW20HdQAGCxdOEEg2cs= github.com/hjson/hjson-go/v4 v4.0.0/go.mod h1:KaYt3bTw3zhBjYqnXkYywcYctk0A2nxeEFTse3rH13E= -github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA= -github.com/iancoleman/orderedmap v0.3.0 h1:5cbR2grmZR/DiVt+VJopEhtVs9YGInGIxAoMJn+Ichc= -github.com/iancoleman/orderedmap v0.3.0/go.mod h1:XuLcCUkdL5owUCQeF2Ue9uuw1EptkJDkXXS7VoV7XGE= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/invopop/jsonschema v0.8.0 h1:9Vblm5uNqURXUSaX0QUYcI/Hcu5rrvOz5MbpWgw0VkM= -github.com/invopop/jsonschema v0.8.0/go.mod h1:O9uiLokuu0+MGFlyiaqtWxwqJm41/+8Nj0lD7A36YH0= +github.com/invopop/jsonschema v0.12.0 h1:6ovsNSuvn9wEQVOyc72aycBMVQFKz7cPdMJn10CvzRI= +github.com/invopop/jsonschema v0.12.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0= github.com/jhump/protoreflect v1.15.1 h1:HUMERORf3I3ZdX05WaQ6MIpd/NJ434hTp5YiKgfCL6c= github.com/jhump/protoreflect v1.15.1/go.mod h1:jD/2GMKKE6OqX8qTjhADU1e6DShO+gavG9e0Q693nKo= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -224,6 +227,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -238,8 +243,9 @@ github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOA github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -274,15 +280,11 @@ github.com/npillmayer/nestext v0.1.3/go.mod h1:h2lrijH8jpicr25dFY+oAJLyzlya6jhnu github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA= github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= -github.com/panjf2000/ants/v2 v2.8.1 h1:C+n/f++aiW8kHCExKlpX6X+okmxKXP7DWLutxuAPuwQ= -github.com/panjf2000/ants/v2 v2.8.1/go.mod h1:KIBmYG9QQX5U2qzFP/yQJaq/nSb6rahS9iEHkrCMgM8= -github.com/panjf2000/gnet/v2 v2.3.2 h1:cwzq4S2fZbHvBaGriMlZTDtiFL/EzaaVny2V03wOuj0= -github.com/panjf2000/gnet/v2 v2.3.2/go.mod h1:jQ0+i/ZSs4wxUKl06sgjWE0bL/yWI1d5LkNdqulZXFc= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= -github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= -github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= +github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= @@ -299,14 +301,14 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= -github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= +github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= +github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= -github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= @@ -316,8 +318,8 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= -github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/rhnvrm/simples3 v0.6.1/go.mod h1:Y+3vYm2V7Y4VijFoJHHTrja6OgPrJ2cBti8dPGkC3sA= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= @@ -327,8 +329,8 @@ github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c= -github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w= +github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A= +github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= @@ -349,7 +351,6 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -359,8 +360,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= -github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= +github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= +github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= @@ -370,18 +371,18 @@ github.com/zenizh/go-capturer v0.0.0-20211219060012-52ea6c8fed04/go.mod h1:FiwNQ go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A= go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v3 v3.5.4/go.mod h1:ZaRkVgBZC+L+dLCjTcF1hRXpgZXQPOvnA/Ak/gq3kiY= -go.opentelemetry.io/otel v1.18.0 h1:TgVozPGZ01nHyDZxK5WGPFB9QexeTMXEH7+tIClWfzs= -go.opentelemetry.io/otel v1.18.0/go.mod h1:9lWqYO0Db579XzVuCKFNPDl4s73Voa+zEck3wHaAYQI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.18.0 h1:IAtl+7gua134xcV3NieDhJHjjOVeJhXAnYf/0hswjUY= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.18.0/go.mod h1:w+pXobnBzh95MNIkeIuAKcHe/Uu/CX2PKIvBP6ipKRA= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.18.0 h1:yE32ay7mJG2leczfREEhoW3VfSZIvHaB+gvVo1o8DQ8= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.18.0/go.mod h1:G17FHPDLt74bCI7tJ4CMitEk4BXTYG4FW6XUpkPBXa4= -go.opentelemetry.io/otel/metric v1.18.0 h1:JwVzw94UYmbx3ej++CwLUQZxEODDj/pOuTCvzhtRrSQ= -go.opentelemetry.io/otel/metric v1.18.0/go.mod h1:nNSpsVDjWGfb7chbRLUNW+PBNdcSTHD4Uu5pfFMOI0k= -go.opentelemetry.io/otel/sdk v1.18.0 h1:e3bAB0wB3MljH38sHzpV/qWrOTCFrdZF2ct9F8rBkcY= -go.opentelemetry.io/otel/sdk v1.18.0/go.mod h1:1RCygWV7plY2KmdskZEDDBs4tJeHG92MdHZIluiYs/M= -go.opentelemetry.io/otel/trace v1.18.0 h1:NY+czwbHbmndxojTEKiSMHkG2ClNH2PwmcHrdo0JY10= -go.opentelemetry.io/otel/trace v1.18.0/go.mod h1:T2+SGJGuYZY3bjj5rgh/hN7KIrlpWC5nS8Mjvzckz+0= +go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= +go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 h1:3d+S281UTjM+AbF31XSOYn1qXn3BgIdWl8HNEpx08Jk= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0/go.mod h1:0+KuTDyKL4gjKCF75pHOX4wuzYDUZYfAQdSu43o+Z2I= +go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE= +go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8= +go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o= +go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A= +go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg= +go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -391,11 +392,7 @@ go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0 go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= -go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= -go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= -go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= @@ -404,11 +401,11 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= -golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -440,13 +437,13 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= -golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= -golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= +golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY= +golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -458,8 +455,6 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= @@ -502,8 +497,9 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= @@ -550,12 +546,12 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb h1:XFBgcDwm7irdHTbz4Zk2h7Mh+eis4nfJEFQFYzJzuIA= -google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= -google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb h1:lK0oleSc7IQsUxO3U5TjL9DWlsxpEBemh+zpB7IqhWI= -google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230913181813-007df8e322eb h1:Isk1sSH7bovx8Rti2wZK0UZF6oraBDK74uoyLEEVFN0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230913181813-007df8e322eb/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= +google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b h1:+YaDE2r2OG8t/z5qmsh7Y+XXwCbvadxxZ0YY6mTdrVA= +google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:CgAqfJo+Xmu0GwA0411Ht3OU3OntXwsGmrmjI8ioGXI= +google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b h1:CIC2YMXmIhYw6evmhPxBKJ4fmLbOFtXQN/GV3XOZR8k= +google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:IBQ646DjkDkvUIsVq/cc03FUFQ9wbZu7yE396YcL870= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b h1:ZlWIi1wSK56/8hn4QcBp/j9M7Gt3U/3hZw3mC7vDICo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:swOH3j0KzcDDgGUWr+SNpyTen5YrXjS3eyPzFYKc6lc= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= @@ -564,8 +560,8 @@ google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQ google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.58.1 h1:OL+Vz23DTtrrldqHK49FUOPHyY75rvFqJfXC84NYW58= -google.golang.org/grpc v1.58.1/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= +google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= +google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= diff --git a/metrics/builtins.go b/metrics/builtins.go index e56b978d..2e588a58 100644 --- a/metrics/builtins.go +++ b/metrics/builtins.go @@ -75,10 +75,15 @@ var ( Name: "proxied_connections", Help: "Number of proxy connects", }) - ProxyPassThroughs = promauto.NewCounter(prometheus.CounterOpts{ + ProxyPassThroughsToClient = promauto.NewCounter(prometheus.CounterOpts{ Namespace: Namespace, - Name: "proxy_passthroughs_total", - Help: "Number of successful proxy passthroughs", + Name: "proxy_passthroughs_to_client_total", + Help: "Number of successful proxy passthroughs from server to client", + }) + ProxyPassThroughsToServer = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: Namespace, + Name: "proxy_passthroughs_to_server_total", + Help: "Number of successful proxy passthroughs from client to server", }) ProxyPassThroughTerminations = promauto.NewCounter(prometheus.CounterOpts{ Namespace: Namespace, diff --git a/network/client.go b/network/client.go index 7526c0d1..05408ff4 100644 --- a/network/client.go +++ b/network/client.go @@ -5,6 +5,8 @@ import ( "context" "fmt" "net" + "sync" + "sync/atomic" "time" "github.com/gatewayd-io/gatewayd/config" @@ -17,6 +19,7 @@ import ( type IClient interface { Send(data []byte) (int, *gerr.GatewayDError) Receive() (int, []byte, *gerr.GatewayDError) + Reconnect() error Close() IsConnected() bool RemoteAddr() string @@ -24,10 +27,11 @@ type IClient interface { } type Client struct { - net.Conn - - logger zerolog.Logger - ctx context.Context //nolint:containedctx + conn net.Conn + logger zerolog.Logger + ctx context.Context //nolint:containedctx + connected atomic.Bool + mu sync.Mutex TCPKeepAlive bool TCPKeepAlivePeriod time.Duration @@ -53,6 +57,7 @@ func NewClient(ctx context.Context, clientConfig *config.Client, logger zerolog. return nil } + client.connected.Store(false) client.logger = logger // Try to resolve the address and log an error if it can't be resolved. @@ -65,6 +70,7 @@ func NewClient(ctx context.Context, clientConfig *config.Client, logger zerolog. // Create a resolved client. client = Client{ ctx: clientCtx, + mu: sync.Mutex{}, Network: clientConfig.Network, Address: addr, } @@ -86,13 +92,14 @@ func NewClient(ctx context.Context, clientConfig *config.Client, logger zerolog. return nil } - client.Conn = conn + client.conn = conn + client.connected.Store(true) // Set the TCP keep alive. client.TCPKeepAlive = clientConfig.TCPKeepAlive client.TCPKeepAlivePeriod = clientConfig.TCPKeepAlivePeriod - if c, ok := client.Conn.(*net.TCPConn); ok { + if c, ok := client.conn.(*net.TCPConn); ok { if err := c.SetKeepAlive(client.TCPKeepAlive); err != nil { logger.Error().Err(err).Msg("Failed to set keep alive") span.RecordError(err) @@ -107,7 +114,7 @@ func NewClient(ctx context.Context, clientConfig *config.Client, logger zerolog. // Set the receive deadline (timeout). client.ReceiveDeadline = clientConfig.ReceiveDeadline if client.ReceiveDeadline > 0 { - if err := client.Conn.SetReadDeadline(time.Now().Add(client.ReceiveDeadline)); err != nil { + if err := client.conn.SetReadDeadline(time.Now().Add(client.ReceiveDeadline)); err != nil { logger.Error().Err(err).Msg("Failed to set receive deadline") span.RecordError(err) } else { @@ -119,7 +126,7 @@ func NewClient(ctx context.Context, clientConfig *config.Client, logger zerolog. // Set the send deadline (timeout). client.SendDeadline = clientConfig.SendDeadline if client.SendDeadline > 0 { - if err := client.Conn.SetWriteDeadline(time.Now().Add(client.SendDeadline)); err != nil { + if err := client.conn.SetWriteDeadline(time.Now().Add(client.SendDeadline)); err != nil { logger.Error().Err(err).Msg("Failed to set send deadline") span.RecordError(err) } else { @@ -146,12 +153,28 @@ func (c *Client) Send(data []byte) (int, *gerr.GatewayDError) { _, span := otel.Tracer(config.TracerName).Start(c.ctx, "Send") defer span.End() - sent, err := c.Conn.Write(data) - if err != nil { - c.logger.Error().Err(err).Msg("Couldn't send data to the server") - span.RecordError(err) - return 0, gerr.ErrClientSendFailed.Wrap(err) + if !c.connected.Load() { + span.RecordError(gerr.ErrClientNotConnected) + return 0, gerr.ErrClientNotConnected + } + + sent := 0 + received := len(data) + for { + if sent >= received { + break + } + + written, err := c.conn.Write(data) + if err != nil { + c.logger.Error().Err(err).Msg("Couldn't send data to the server") + span.RecordError(err) + return 0, gerr.ErrClientSendFailed.Wrap(err) + } + + sent += written } + c.logger.Debug().Fields( map[string]interface{}{ "length": sent, @@ -159,8 +182,6 @@ func (c *Client) Send(data []byte) (int, *gerr.GatewayDError) { }, ).Msg("Sent data to server") - metrics.BytesSentToServer.Observe(float64(sent)) - return sent, nil } @@ -169,6 +190,11 @@ func (c *Client) Receive() (int, []byte, *gerr.GatewayDError) { _, span := otel.Tracer(config.TracerName).Start(c.ctx, "Receive") defer span.End() + if !c.connected.Load() { + span.RecordError(gerr.ErrClientNotConnected) + return 0, nil, gerr.ErrClientNotConnected + } + var ctx context.Context var cancel context.CancelFunc if c.ReceiveTimeout > 0 { @@ -181,40 +207,89 @@ func (c *Client) Receive() (int, []byte, *gerr.GatewayDError) { var received int buffer := bytes.NewBuffer(nil) // Read the data in chunks. - select { //nolint:gosimple - case <-time.After(time.Millisecond): - for ctx.Err() == nil { - chunk := make([]byte, c.ReceiveChunkSize) - read, err := c.Conn.Read(chunk) - if err != nil { - c.logger.Error().Err(err).Msg("Couldn't receive data from the server") - span.RecordError(err) - metrics.BytesReceivedFromServer.Observe(float64(received)) - return received, buffer.Bytes(), gerr.ErrClientReceiveFailed.Wrap(err) - } - received += read - buffer.Write(chunk[:read]) + for ctx.Err() == nil { + chunk := make([]byte, c.ReceiveChunkSize) + read, err := c.conn.Read(chunk) + if err != nil { + c.logger.Error().Err(err).Msg("Couldn't receive data from the server") + span.RecordError(err) + return received, buffer.Bytes(), gerr.ErrClientReceiveFailed.Wrap(err) + } + received += read + buffer.Write(chunk[:read]) - if read == 0 || read < c.ReceiveChunkSize { - break - } + if read == 0 || read < c.ReceiveChunkSize { + break } } - metrics.BytesReceivedFromServer.Observe(float64(received)) return received, buffer.Bytes(), nil } +// Reconnect reconnects to the server. +func (c *Client) Reconnect() error { + _, span := otel.Tracer(config.TracerName).Start(c.ctx, "Reconnect") + defer span.End() + + // Save the current address and network. + address := c.Address + network := c.Network + + if c.conn != nil { + c.Close() + } else { + metrics.ServerConnections.Dec() + } + c.connected.Store(false) + + // Restore the address and network. + c.Address = address + c.Network = network + + conn, err := net.Dial(c.Network, c.Address) + if err != nil { + c.logger.Error().Err(err).Msg("Failed to reconnect") + span.RecordError(err) + return gerr.ErrClientConnectionFailed.Wrap(err) + } + + c.conn = conn + c.ID = GetID( + conn.LocalAddr().Network(), conn.LocalAddr().String(), config.DefaultSeed, c.logger) + c.connected.Store(true) + c.logger.Debug().Str("address", c.Address).Msg("Reconnected to server") + metrics.ServerConnections.Inc() + + return nil +} + // Close closes the connection to the server. func (c *Client) Close() { _, span := otel.Tracer(config.TracerName).Start(c.ctx, "Close") defer span.End() + c.mu.Lock() + defer c.mu.Unlock() + + // Set the deadline to now so that the connection is closed immediately. + // This will stop all the Conn.Read() and Conn.Write() calls. + // Ref: https://groups.google.com/g/golang-nuts/c/VPVWFrpIEyo + if c.conn != nil { + if err := c.conn.SetDeadline(time.Now()); err != nil { + c.logger.Error().Err(err).Msg("Failed to set deadline") + span.RecordError(err) + } + } + + c.connected.Store(false) c.logger.Debug().Str("address", c.Address).Msg("Closing connection to server") - if c.Conn != nil { - c.Conn.Close() + if c.conn != nil { + if err := c.conn.Close(); err != nil { + c.logger.Error().Err(err).Msg("Failed to close connection") + span.RecordError(err) + } } c.ID = "" - c.Conn = nil + c.conn = nil c.Address = "" c.Network = "" @@ -237,7 +312,7 @@ func (c *Client) IsConnected() bool { return false } - if c != nil && c.Conn == nil || c.ID == "" { + if c != nil && c.conn == nil || c.ID == "" { c.logger.Debug().Fields( map[string]interface{}{ "address": c.Address, @@ -246,22 +321,20 @@ func (c *Client) IsConnected() bool { return false } - if n, err := c.Read([]byte{}); n == 0 && err != nil { - c.logger.Debug().Fields( - map[string]interface{}{ - "address": c.Address, - "reason": "read 0 bytes", - }).Msg("Connection to server is closed") - return false - } - - return true + return c.connected.Load() } // RemoteAddr returns the remote address of the client safely. func (c *Client) RemoteAddr() string { - if c.Conn != nil && c.Conn.RemoteAddr() != nil { - return c.Conn.RemoteAddr().String() + if !c.connected.Load() { + return "" + } + + c.mu.Lock() + defer c.mu.Unlock() + + if c.conn != nil && c.conn.RemoteAddr() != nil { + return c.conn.RemoteAddr().String() } return "" @@ -269,8 +342,15 @@ func (c *Client) RemoteAddr() string { // LocalAddr returns the local address of the client safely. func (c *Client) LocalAddr() string { - if c.Conn != nil && c.Conn.LocalAddr() != nil { - return c.Conn.LocalAddr().String() + if !c.connected.Load() { + return "" + } + + c.mu.Lock() + defer c.mu.Unlock() + + if c.conn != nil && c.conn.LocalAddr() != nil { + return c.conn.LocalAddr().String() } return "" diff --git a/network/client_test.go b/network/client_test.go index 91789295..8896415e 100644 --- a/network/client_test.go +++ b/network/client_test.go @@ -45,10 +45,11 @@ func TestNewClient(t *testing.T) { client := CreateNewClient(t) defer client.Close() + assert.NotNil(t, client) assert.Equal(t, "tcp", client.Network) assert.Equal(t, "127.0.0.1:5432", client.Address) assert.NotEmpty(t, client.ID) - assert.NotNil(t, client.Conn) + assert.NotNil(t, client.conn) } // TestSend tests the Send function. @@ -95,7 +96,7 @@ func TestClose(t *testing.T) { assert.Equal(t, "", client.ID) assert.Equal(t, "", client.Network) assert.Equal(t, "", client.Address) - assert.Nil(t, client.Conn) + assert.Nil(t, client.conn) } // TestIsConnected tests the IsConnected function. @@ -107,6 +108,24 @@ func TestIsConnected(t *testing.T) { assert.False(t, client.IsConnected()) } +func TestReconnect(t *testing.T) { + client := CreateNewClient(t) + defer client.Close() + + assert.NotNil(t, client) + assert.NotNil(t, client.conn) + assert.NotEmpty(t, client.ID) + localAddr := client.LocalAddr() + assert.NotEmpty(t, localAddr) + + assert.NoError(t, client.Reconnect()) + assert.NotNil(t, client) + assert.NotNil(t, client.conn) + assert.NotEmpty(t, client.ID) + assert.NotEmpty(t, client.LocalAddr()) + assert.NotEqual(t, localAddr, client.LocalAddr()) // This is a new connection. +} + func BenchmarkNewClient(b *testing.B) { cfg := logging.LoggerConfig{ Output: []config.LogOutput{config.Console}, diff --git a/network/engine.go b/network/engine.go new file mode 100644 index 00000000..6f8192ee --- /dev/null +++ b/network/engine.go @@ -0,0 +1,69 @@ +package network + +import ( + "context" + "net" + "sync" + "sync/atomic" + "time" + + "github.com/gatewayd-io/gatewayd/config" + "github.com/rs/zerolog" +) + +// Engine is the network engine. +// TODO: Move this to the Server struct. +type Engine struct { + listener net.Listener + host string + port int + connections uint32 + logger zerolog.Logger + running *atomic.Bool + stopServer chan struct{} + mu *sync.RWMutex +} + +// CountConnections returns the current number of connections. +func (engine *Engine) CountConnections() int { + engine.mu.RLock() + defer engine.mu.RUnlock() + return int(engine.connections) +} + +// Stop stops the engine. +func (engine *Engine) Stop(ctx context.Context) error { + _, cancel := context.WithDeadline(ctx, time.Now().Add(config.DefaultEngineStopTimeout)) + defer cancel() + + var err error + engine.running.Store(false) + if engine.listener != nil { + if err = engine.listener.Close(); err != nil { + engine.logger.Error().Err(err).Msg("Failed to close listener") + } + } else { + engine.logger.Error().Msg("Listener is not initialized") + } + + select { + case <-engine.stopServer: + engine.logger.Info().Msg("Server stopped") + return err //nolint:wrapcheck + default: + engine.stopServer <- struct{}{} + close(engine.stopServer) + return err //nolint:wrapcheck + } +} + +// NewEngine creates a new engine. +func NewEngine(logger zerolog.Logger) Engine { + return Engine{ + connections: 0, + logger: logger, + running: &atomic.Bool{}, + stopServer: make(chan struct{}), + mu: &sync.RWMutex{}, + } +} diff --git a/network/engine_test.go b/network/engine_test.go new file mode 100644 index 00000000..a9ee7e2e --- /dev/null +++ b/network/engine_test.go @@ -0,0 +1,49 @@ +package network + +import ( + "context" + "testing" + "time" + + "github.com/gatewayd-io/gatewayd/config" + "github.com/gatewayd-io/gatewayd/logging" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/zenizh/go-capturer" +) + +func TestEngine(t *testing.T) { + output := capturer.CaptureOutput(func() { + logger := logging.NewLogger(context.Background(), logging.LoggerConfig{ + Output: []config.LogOutput{config.Console}, + TimeFormat: zerolog.TimeFormatUnix, + ConsoleTimeFormat: time.RFC3339, + Level: zerolog.WarnLevel, + NoColor: true, + }) + engine := NewEngine(logger) + assert.NotNil(t, engine) + assert.NotNil(t, engine.logger) + assert.Zero(t, engine.connections) + assert.Zero(t, engine.CountConnections()) + assert.Empty(t, engine.host) + assert.Empty(t, engine.port) + assert.False(t, engine.running.Load()) + + go func(engine Engine) { + v := <-engine.stopServer + // Empty struct is expected to be received and + // it means that the server is stopped. + assert.Equal(t, v, struct{}{}) + }(engine) + + err := engine.Stop(context.Background()) + // This is expected to fail because the engine is not running. + assert.Nil(t, err) + assert.False(t, engine.running.Load()) + assert.Zero(t, engine.connections) + }) + + // Expected output: + assert.Contains(t, output, "ERR Listener is not initialized") +} diff --git a/network/network_helpers_test.go b/network/network_helpers_test.go index 766c0315..12fc7ea6 100644 --- a/network/network_helpers_test.go +++ b/network/network_helpers_test.go @@ -91,8 +91,10 @@ func CollectAndComparePrometheusMetrics(t *testing.T) { # TYPE gatewayd_proxy_health_checks_total counter # HELP gatewayd_proxy_passthrough_terminations_total Number of proxy passthrough terminations by plugins # TYPE gatewayd_proxy_passthrough_terminations_total counter - # HELP gatewayd_proxy_passthroughs_total Number of successful proxy passthroughs - # TYPE gatewayd_proxy_passthroughs_total counter + # HELP gatewayd_proxy_passthroughs_to_client_total Number of successful proxy passthroughs + # TYPE gatewayd_proxy_passthroughs_to_client_total counter + # HELP gatewayd_proxy_passthroughs_to_server_total Number of successful proxy passthroughs + # TYPE gatewayd_proxy_passthroughs_to_server_total counter # HELP gatewayd_server_connections Number of server connections # TYPE gatewayd_server_connections gauge # HELP gatewayd_server_ticks_fired_total Total number of server ticks fired @@ -105,12 +107,12 @@ func CollectAndComparePrometheusMetrics(t *testing.T) { want = metadata + ` gatewayd_bytes_received_from_client_sum 67 gatewayd_bytes_received_from_client_count 1 - gatewayd_bytes_received_from_server_sum 96 - gatewayd_bytes_received_from_server_count 4 + gatewayd_bytes_received_from_server_sum 24 + gatewayd_bytes_received_from_server_count 1 gatewayd_bytes_sent_to_client_sum 24 gatewayd_bytes_sent_to_client_count 1 - gatewayd_bytes_sent_to_server_sum 282 - gatewayd_bytes_sent_to_server_count 5 + gatewayd_bytes_sent_to_server_sum 67 + gatewayd_bytes_sent_to_server_count 1 gatewayd_client_connections 1 gatewayd_plugin_hooks_executed_total 11 gatewayd_plugin_hooks_registered_total 0 @@ -118,7 +120,8 @@ func CollectAndComparePrometheusMetrics(t *testing.T) { gatewayd_proxied_connections 1 gatewayd_proxy_health_checks_total 0 gatewayd_proxy_passthrough_terminations_total 0 - gatewayd_proxy_passthroughs_total 1 + gatewayd_proxy_passthroughs_to_client_total 1 + gatewayd_proxy_passthroughs_to_server_total 1 gatewayd_server_connections 5 gatewayd_traffic_bytes_sum 182 gatewayd_traffic_bytes_count 4 diff --git a/network/proxy.go b/network/proxy.go index fbd04e8d..56e9d799 100644 --- a/network/proxy.go +++ b/network/proxy.go @@ -1,7 +1,11 @@ package network import ( + "bytes" "context" + "errors" + "io" + "net" "time" v1 "github.com/gatewayd-io/gatewayd-plugin-sdk/plugin/v1" @@ -12,16 +16,16 @@ import ( "github.com/gatewayd-io/gatewayd/pool" "github.com/getsentry/sentry-go" "github.com/go-co-op/gocron" - "github.com/panjf2000/gnet/v2" "github.com/rs/zerolog" "go.opentelemetry.io/otel" ) type IProxy interface { - Connect(gconn gnet.Conn) *gerr.GatewayDError - Disconnect(gconn gnet.Conn) *gerr.GatewayDError - PassThrough(gconn gnet.Conn) *gerr.GatewayDError - IsHealty(cl *Client) (*Client, *gerr.GatewayDError) + Connect(conn net.Conn) *gerr.GatewayDError + Disconnect(conn net.Conn) *gerr.GatewayDError + PassThroughToServer(conn net.Conn, stack *Stack) *gerr.GatewayDError + PassThroughToClient(conn net.Conn, stack *Stack) *gerr.GatewayDError + IsHealthy(cl *Client) (*Client, *gerr.GatewayDError) IsExhausted() bool Shutdown() AvailableConnections() []string @@ -123,7 +127,7 @@ func NewProxy( // Connect maps a server connection from the available connection pool to a incoming connection. // It returns an error if the pool is exhausted. If the pool is elastic, it creates a new client // and maps it to the incoming connection. -func (pr *Proxy) Connect(gconn gnet.Conn) *gerr.GatewayDError { +func (pr *Proxy) Connect(conn net.Conn) *gerr.GatewayDError { _, span := otel.Tracer(config.TracerName).Start(pr.ctx, "Connect") defer span.End() @@ -156,13 +160,13 @@ func (pr *Proxy) Connect(gconn gnet.Conn) *gerr.GatewayDError { } } - client, err := pr.IsHealty(client) + client, err := pr.IsHealthy(client) if err != nil { pr.logger.Error().Err(err).Msg("Failed to connect to the client") span.RecordError(err) } - if err := pr.busyConnections.Put(gconn, client); err != nil { + if err := pr.busyConnections.Put(conn, client); err != nil { // This should never happen. span.RecordError(err) return err @@ -173,7 +177,7 @@ func (pr *Proxy) Connect(gconn gnet.Conn) *gerr.GatewayDError { fields := map[string]interface{}{ "function": "proxy.connect", "client": "unknown", - "server": RemoteAddr(gconn), + "server": RemoteAddr(conn), } if client.ID != "" { fields["client"] = client.ID[:7] @@ -198,39 +202,43 @@ func (pr *Proxy) Connect(gconn gnet.Conn) *gerr.GatewayDError { // Disconnect removes the client from the busy connection pool and tries to recycle // the server connection. -func (pr *Proxy) Disconnect(gconn gnet.Conn) *gerr.GatewayDError { +func (pr *Proxy) Disconnect(conn net.Conn) *gerr.GatewayDError { _, span := otel.Tracer(config.TracerName).Start(pr.ctx, "Disconnect") defer span.End() - client := pr.busyConnections.Pop(gconn) + client := pr.busyConnections.Pop(conn) + if client == nil { + // If this ever happens, it means that the client connection + // is pre-empted from the busy connections pool. + pr.logger.Debug().Msg("Client connection is pre-empted from the busy connections pool") + span.RecordError(gerr.ErrClientNotFound) + return gerr.ErrClientNotFound + } + //nolint:nestif - if client != nil { - if client, ok := client.(*Client); ok { - if (pr.Elastic && pr.ReuseElasticClients) || !pr.Elastic { - _, err := pr.IsHealty(client) - if err != nil { - pr.logger.Error().Err(err).Msg("Failed to reconnect to the client") - span.RecordError(err) - } - // If the client is not in the pool, put it back. - err = pr.availableConnections.Put(client.ID, client) - if err != nil { - pr.logger.Error().Err(err).Msg("Failed to put the client back in the pool") - span.RecordError(err) - } - } else { - span.RecordError(gerr.ErrClientNotConnected) - return gerr.ErrClientNotConnected + if client, ok := client.(*Client); ok { + if (pr.Elastic && pr.ReuseElasticClients) || !pr.Elastic { + // Recycle the server connection by reconnecting. + if err := client.Reconnect(); err != nil { + pr.logger.Error().Err(err).Msg("Failed to reconnect to the client") + span.RecordError(err) + } + + // If the client is not in the pool, put it back. + if err := pr.availableConnections.Put(client.ID, client); err != nil { + pr.logger.Error().Err(err).Msg("Failed to put the client back in the pool") + span.RecordError(err) } } else { - // This should never happen, but if it does, - // then there are some serious issues with the pool. - span.RecordError(gerr.ErrCastFailed) - return gerr.ErrCastFailed + span.RecordError(gerr.ErrClientNotConnected) + return gerr.ErrClientNotConnected } } else { - span.RecordError(gerr.ErrClientNotFound) - return gerr.ErrClientNotFound + // This should never happen, but if it does, + // then there are some serious issues with the pool. + pr.logger.Error().Msg("Failed to cast the client to the Client type") + span.RecordError(gerr.ErrCastFailed) + return gerr.ErrCastFailed } metrics.ProxiedConnections.Dec() @@ -251,24 +259,20 @@ func (pr *Proxy) Disconnect(gconn gnet.Conn) *gerr.GatewayDError { return nil } -// PassThrough sends the data from the client to the server and vice versa. -func (pr *Proxy) PassThrough(gconn gnet.Conn) *gerr.GatewayDError { +// PassThroughToServer sends the data from the client to the server. +func (pr *Proxy) PassThroughToServer(conn net.Conn, stack *Stack) *gerr.GatewayDError { _, span := otel.Tracer(config.TracerName).Start(pr.ctx, "PassThrough") defer span.End() - // TODO: Handle bi-directional traffic - // Currently the passthrough is a one-way street from the client to the server, that is, - // the client can send data to the server and receive the response back, but the server - // cannot take initiative and send data to the client. So, there should be another event-loop - // that listens for data from the server and sends it to the client. var client *Client - if pr.busyConnections.Get(gconn) == nil { + // Check if the proxy has a egress client for the incoming connection. + if pr.busyConnections.Get(conn) == nil { span.RecordError(gerr.ErrClientNotFound) return gerr.ErrClientNotFound } // Get the client from the busy connection pool. - if cl, ok := pr.busyConnections.Get(gconn).(*Client); ok { + if cl, ok := pr.busyConnections.Get(conn).(*Client); ok { client = cl } else { span.RecordError(gerr.ErrCastFailed) @@ -276,17 +280,22 @@ func (pr *Proxy) PassThrough(gconn gnet.Conn) *gerr.GatewayDError { } span.AddEvent("Got the client from the busy connection pool") + if !client.IsConnected() { + return gerr.ErrClientNotConnected + } + // Receive the request from the client. - request, origErr := pr.receiveTrafficFromClient(gconn) + request, origErr := pr.receiveTrafficFromClient(conn) span.AddEvent("Received traffic from client") + // Run the OnTrafficFromClient hooks. pluginTimeoutCtx, cancel := context.WithTimeout(context.Background(), pr.pluginTimeout) defer cancel() - // Run the OnTrafficFromClient hooks. + result, err := pr.pluginRegistry.Run( pluginTimeoutCtx, trafficData( - gconn, + conn, client, []Field{ { @@ -302,16 +311,29 @@ func (pr *Proxy) PassThrough(gconn gnet.Conn) *gerr.GatewayDError { } span.AddEvent("Ran the OnTrafficFromClient hooks") + if origErr != nil && errors.Is(origErr, io.EOF) { + // Client closed the connection. + span.AddEvent("Client closed the connection") + return gerr.ErrClientNotConnected.Wrap(origErr) + } + + // Push the client's request to the stack. + stack.Push(&Request{Data: request}) + // If the hook wants to terminate the connection, do it. if pr.shouldTerminate(result) { if modResponse, modReceived := pr.getPluginModifiedResponse(result); modResponse != nil { - metrics.ProxyPassThroughs.Inc() + metrics.ProxyPassThroughsToClient.Inc() metrics.ProxyPassThroughTerminations.Inc() metrics.BytesSentToClient.Observe(float64(modReceived)) metrics.TotalTrafficBytes.Observe(float64(modReceived)) span.AddEvent("Terminating connection") - return pr.sendTrafficToClient(gconn, modResponse, modReceived) + + // Remove the request from the stack if the response is modified. + stack.PopLastRequest() + + return pr.sendTrafficToClient(conn, modResponse, modReceived) } span.RecordError(gerr.ErrHookTerminatedConnection) return gerr.ErrHookTerminatedConnection @@ -322,15 +344,20 @@ func (pr *Proxy) PassThrough(gconn gnet.Conn) *gerr.GatewayDError { span.AddEvent("Plugin(s) modified the request") } + stack.UpdateLastRequest(&Request{Data: request}) + // Send the request to the server. _, err = pr.sendTrafficToServer(client, request) span.AddEvent("Sent traffic to server") + pluginTimeoutCtx, cancel = context.WithTimeout(context.Background(), pr.pluginTimeout) + defer cancel() + // Run the OnTrafficToServer hooks. _, err = pr.pluginRegistry.Run( pluginTimeoutCtx, trafficData( - gconn, + conn, client, []Field{ { @@ -346,52 +373,73 @@ func (pr *Proxy) PassThrough(gconn gnet.Conn) *gerr.GatewayDError { } span.AddEvent("Ran the OnTrafficToServer hooks") + metrics.ProxyPassThroughsToServer.Inc() + + return nil +} + +// PassThroughToClient sends the data from the server to the client. +func (pr *Proxy) PassThroughToClient(conn net.Conn, stack *Stack) *gerr.GatewayDError { + _, span := otel.Tracer(config.TracerName).Start(pr.ctx, "PassThrough") + defer span.End() + + var client *Client + // Check if the proxy has a egress client for the incoming connection. + if pr.busyConnections.Get(conn) == nil { + span.RecordError(gerr.ErrClientNotFound) + return gerr.ErrClientNotFound + } + + // Get the client from the busy connection pool. + if cl, ok := pr.busyConnections.Get(conn).(*Client); ok { + client = cl + } else { + span.RecordError(gerr.ErrCastFailed) + return gerr.ErrCastFailed + } + span.AddEvent("Got the client from the busy connection pool") + + if !client.IsConnected() { + return gerr.ErrClientNotConnected + } + // Receive the response from the server. received, response, err := pr.receiveTrafficFromServer(client) span.AddEvent("Received traffic from server") - // The connection to the server is closed, so we MUST reconnect, - // otherwise the client will be stuck. - // TODO: Fix bug in handling connection close - // See: https://github.com/gatewayd-io/gatewayd/issues/219 - if IsConnClosed(received, err) || IsConnTimedOut(err) { - pr.logger.Debug().Fields( - map[string]interface{}{ - "function": "proxy.passthrough", - "local": client.LocalAddr(), - "remote": client.RemoteAddr(), - }).Msg("Client disconnected") - - client.Close() - client = NewClient(pr.ctx, pr.ClientConfig, pr.logger) - pr.busyConnections.Remove(gconn) - if err := pr.busyConnections.Put(gconn, client); err != nil { - span.RecordError(err) - // This should never happen - return err - } - } - // If the response is empty, don't send anything, instead just close the ingress connection. - // TODO: Fix bug in handling connection close - // See: https://github.com/gatewayd-io/gatewayd/issues/219 - if received == 0 { - pr.logger.Debug().Fields( - map[string]interface{}{ - "function": "proxy.passthrough", - "local": client.LocalAddr(), - "remote": client.RemoteAddr(), - }).Msg("No data to send to client") + if received == 0 || err != nil { + fields := map[string]interface{}{"function": "proxy.passthrough"} + if client.LocalAddr() != "" { + fields["local_addr"] = client.LocalAddr() + } + if client.RemoteAddr() != "" { + fields["remote_addr"] = client.RemoteAddr() + } + pr.logger.Debug().Fields(fields).Msg("No data to send to client") span.AddEvent("No data to send to client") span.RecordError(err) + + stack.PopLastRequest() + return err } + pluginTimeoutCtx, cancel := context.WithTimeout(context.Background(), pr.pluginTimeout) + defer cancel() + + // Get the last request from the stack. + lastRequest := stack.PopLastRequest() + request := make([]byte, 0) + if lastRequest != nil { + request = lastRequest.Data + } + // Run the OnTrafficFromServer hooks. - result, err = pr.pluginRegistry.Run( + result, err := pr.pluginRegistry.Run( pluginTimeoutCtx, trafficData( - gconn, + conn, client, []Field{ { @@ -419,14 +467,17 @@ func (pr *Proxy) PassThrough(gconn gnet.Conn) *gerr.GatewayDError { } // Send the response to the client. - errVerdict := pr.sendTrafficToClient(gconn, response, received) + errVerdict := pr.sendTrafficToClient(conn, response, received) span.AddEvent("Sent traffic to client") // Run the OnTrafficToClient hooks. + pluginTimeoutCtx, cancel = context.WithTimeout(context.Background(), pr.pluginTimeout) + defer cancel() + _, err = pr.pluginRegistry.Run( pluginTimeoutCtx, trafficData( - gconn, + conn, client, []Field{ { @@ -438,7 +489,7 @@ func (pr *Proxy) PassThrough(gconn gnet.Conn) *gerr.GatewayDError { Value: response[:received], }, }, - err, + nil, ), v1.HookName_HOOK_NAME_ON_TRAFFIC_TO_CLIENT) if err != nil { @@ -450,14 +501,14 @@ func (pr *Proxy) PassThrough(gconn gnet.Conn) *gerr.GatewayDError { span.RecordError(errVerdict) } - metrics.ProxyPassThroughs.Inc() + metrics.ProxyPassThroughsToClient.Inc() return errVerdict } -// IsHealty checks if the pool is exhausted or the client is disconnected. -func (pr *Proxy) IsHealty(client *Client) (*Client, *gerr.GatewayDError) { - _, span := otel.Tracer(config.TracerName).Start(pr.ctx, "IsHealty") +// IsHealthy checks if the pool is exhausted or the client is disconnected. +func (pr *Proxy) IsHealthy(client *Client) (*Client, *gerr.GatewayDError) { + _, span := otel.Tracer(config.TracerName).Start(pr.ctx, "IsHealthy") defer span.End() if pr.IsExhausted() { @@ -492,8 +543,10 @@ func (pr *Proxy) Shutdown() { defer span.End() pr.availableConnections.ForEach(func(key, value interface{}) bool { - if cl, ok := value.(*Client); ok { - cl.Close() + if client, ok := value.(*Client); ok { + if client.IsConnected() { + client.Close() + } } return true }) @@ -501,15 +554,26 @@ func (pr *Proxy) Shutdown() { pr.logger.Debug().Msg("All available connections have been closed") pr.busyConnections.ForEach(func(key, value interface{}) bool { - if gconn, ok := key.(gnet.Conn); ok { - gconn.Close() + if conn, ok := key.(net.Conn); ok { + // This will stop all the Conn.Read() and Conn.Write() calls. + if err := conn.SetDeadline(time.Now()); err != nil { + pr.logger.Error().Err(err).Msg("Error setting the deadline") + span.RecordError(err) + } + if err := conn.Close(); err != nil { + pr.logger.Error().Err(err).Msg("Failed to close the connection") + span.RecordError(err) + } } - if cl, ok := value.(*Client); ok { - cl.Close() + if client, ok := value.(*Client); ok { + if client != nil { + client.Close() + } } return true }) pr.busyConnections.Clear() + pr.scheduler.Stop() pr.scheduler.Clear() pr.logger.Debug().Msg("All busy connections have been closed") } @@ -522,7 +586,7 @@ func (pr *Proxy) AvailableConnections() []string { connections := make([]string, 0) pr.availableConnections.ForEach(func(_, value interface{}) bool { if cl, ok := value.(*Client); ok { - connections = append(connections, cl.Conn.LocalAddr().String()) + connections = append(connections, cl.LocalAddr()) } return true }) @@ -536,38 +600,59 @@ func (pr *Proxy) BusyConnections() []string { connections := make([]string, 0) pr.busyConnections.ForEach(func(key, _ interface{}) bool { - if gconn, ok := key.(gnet.Conn); ok { - connections = append(connections, RemoteAddr(gconn)) + if conn, ok := key.(net.Conn); ok { + connections = append(connections, RemoteAddr(conn)) } return true }) return connections } -// receiveTrafficFromClient is a function that receives data from the client. -func (pr *Proxy) receiveTrafficFromClient(gconn gnet.Conn) ([]byte, error) { +// receiveTrafficFromClient is a function that waits to receive data from the client. +func (pr *Proxy) receiveTrafficFromClient(conn net.Conn) ([]byte, *gerr.GatewayDError) { _, span := otel.Tracer(config.TracerName).Start(pr.ctx, "receiveTrafficFromClient") defer span.End() // request contains the data from the client. - request, err := gconn.Next(-1) - if err != nil { - pr.logger.Error().Err(err).Msg("Error reading from client") - span.RecordError(err) + received := 0 + buffer := bytes.NewBuffer(nil) + for { + chunk := make([]byte, pr.ClientConfig.ReceiveChunkSize) + read, err := conn.Read(chunk) + if read == 0 || err != nil { + pr.logger.Debug().Err(err).Msg("Error reading from client") + span.RecordError(err) + + metrics.BytesReceivedFromClient.Observe(float64(read)) + metrics.TotalTrafficBytes.Observe(float64(read)) + + return chunk[:read], gerr.ErrReadFailed.Wrap(err) + } + + received += read + buffer.Write(chunk[:read]) + + if received == 0 || received < pr.ClientConfig.ReceiveChunkSize { + break + } + + if !pr.isConnectionHealthy(conn) { + break + } } + + length := len(buffer.Bytes()) pr.logger.Debug().Fields( map[string]interface{}{ - "length": len(request), - "local": LocalAddr(gconn), - "remote": RemoteAddr(gconn), + "length": length, + "local": LocalAddr(conn), + "remote": RemoteAddr(conn), }, ).Msg("Received data from client") + metrics.BytesReceivedFromClient.Observe(float64(length)) + metrics.TotalTrafficBytes.Observe(float64(length)) - metrics.BytesReceivedFromClient.Observe(float64(len(request))) - metrics.TotalTrafficBytes.Observe(float64(len(request))) - - //nolint:wrapcheck - return request, err + return buffer.Bytes(), nil } // sendTrafficToServer is a function that sends data to the server. @@ -575,6 +660,11 @@ func (pr *Proxy) sendTrafficToServer(client *Client, request []byte) (int, *gerr _, span := otel.Tracer(config.TracerName).Start(pr.ctx, "sendTrafficToServer") defer span.End() + if len(request) == 0 { + pr.logger.Trace().Msg("Empty request") + return 0, nil + } + // Send the request to the server. sent, err := client.Send(request) if err != nil { @@ -603,14 +693,19 @@ func (pr *Proxy) receiveTrafficFromServer(client *Client) (int, []byte, *gerr.Ga // Receive the response from the server. received, response, err := client.Receive() - pr.logger.Debug().Fields( - map[string]interface{}{ - "function": "proxy.passthrough", - "length": received, - "local": client.LocalAddr(), - "remote": client.RemoteAddr(), - }, - ).Msg("Received data from database") + + fields := map[string]interface{}{ + "function": "proxy.passthrough", + "length": received, + } + if client.LocalAddr() != "" { + fields["local"] = client.LocalAddr() + } + if client.RemoteAddr() != "" { + fields["remote"] = client.RemoteAddr() + } + + pr.logger.Debug().Fields(fields).Msg("Received data from database") metrics.BytesReceivedFromServer.Observe(float64(received)) metrics.TotalTrafficBytes.Observe(float64(received)) @@ -620,30 +715,39 @@ func (pr *Proxy) receiveTrafficFromServer(client *Client) (int, []byte, *gerr.Ga // sendTrafficToClient is a function that sends data to the client. func (pr *Proxy) sendTrafficToClient( - gconn gnet.Conn, response []byte, received int, + conn net.Conn, response []byte, received int, ) *gerr.GatewayDError { _, span := otel.Tracer(config.TracerName).Start(pr.ctx, "sendTrafficToClient") defer span.End() // Send the response to the client async. - origErr := gconn.AsyncWrite(response[:received], func(gconn gnet.Conn, err error) error { - pr.logger.Debug().Fields( - map[string]interface{}{ - "function": "proxy.passthrough", - "length": received, - "local": LocalAddr(gconn), - "remote": RemoteAddr(gconn), - }, - ).Msg("Sent data to client") - span.RecordError(err) - return err - }) - if origErr != nil { - pr.logger.Error().Err(origErr).Msg("Error writing to client") - span.RecordError(origErr) - return gerr.ErrServerSendFailed.Wrap(origErr) + sent := 0 + for { + if sent >= received { + break + } + + written, origErr := conn.Write(response[:received]) + if origErr != nil { + pr.logger.Error().Err(origErr).Msg("Error writing to client") + span.RecordError(origErr) + return gerr.ErrServerSendFailed.Wrap(origErr) + } + + sent += written } + pr.logger.Debug().Fields( + map[string]interface{}{ + "function": "proxy.passthrough", + "length": sent, + "local": LocalAddr(conn), + "remote": RemoteAddr(conn), + }, + ).Msg("Sent data to client") + + span.AddEvent("Sent data to client") + metrics.BytesSentToClient.Observe(float64(received)) metrics.TotalTrafficBytes.Observe(float64(received)) @@ -698,3 +802,17 @@ func (pr *Proxy) getPluginModifiedResponse(result map[string]interface{}) ([]byt return nil, 0 } + +func (pr *Proxy) isConnectionHealthy(conn net.Conn) bool { + if n, err := conn.Read([]byte{}); n == 0 && err != nil { + pr.logger.Debug().Fields( + map[string]interface{}{ + "remote": RemoteAddr(conn), + "local": LocalAddr(conn), + "reason": "read 0 bytes", + }).Msg("Connection to client is closed") + return false + } + + return true +} diff --git a/network/proxy_test.go b/network/proxy_test.go index 6b680193..379b6df0 100644 --- a/network/proxy_test.go +++ b/network/proxy_test.go @@ -23,8 +23,8 @@ func TestNewProxy(t *testing.T) { NoColor: true, }) - // Create a connection pool - pool := pool.NewPool(context.Background(), config.EmptyPoolCapacity) + // Create a connection newPool + newPool := pool.NewPool(context.Background(), config.EmptyPoolCapacity) client := NewClient( context.Background(), @@ -38,13 +38,13 @@ func TestNewProxy(t *testing.T) { TCPKeepAlivePeriod: config.DefaultTCPKeepAlivePeriod, }, logger) - err := pool.Put(client.ID, client) + err := newPool.Put(client.ID, client) assert.Nil(t, err) - // Create a proxy with a fixed buffer pool + // Create a proxy with a fixed buffer newPool proxy := NewProxy( context.Background(), - pool, + newPool, plugin.NewRegistry( context.Background(), config.Loose, @@ -71,7 +71,7 @@ func TestNewProxy(t *testing.T) { assert.Equal(t, false, proxy.Elastic) assert.Equal(t, false, proxy.ReuseElasticClients) assert.Equal(t, false, proxy.IsExhausted()) - c, err := proxy.IsHealty(client) + c, err := proxy.IsHealthy(client) assert.Nil(t, err) assert.Equal(t, client, c) } @@ -86,13 +86,13 @@ func TestNewProxyElastic(t *testing.T) { NoColor: true, }) - // Create a connection pool - pool := pool.NewPool(context.Background(), config.EmptyPoolCapacity) + // Create a connection newPool + newPool := pool.NewPool(context.Background(), config.EmptyPoolCapacity) - // Create a proxy with an elastic buffer pool + // Create a proxy with an elastic buffer newPool proxy := NewProxy( context.Background(), - pool, + newPool, plugin.NewRegistry( context.Background(), config.Loose, @@ -136,14 +136,14 @@ func BenchmarkNewProxy(b *testing.B) { NoColor: true, }) - // Create a connection pool - pool := pool.NewPool(context.Background(), config.EmptyPoolCapacity) + // Create a connection newPool + newPool := pool.NewPool(context.Background(), config.EmptyPoolCapacity) - // Create a proxy with a fixed buffer pool + // Create a proxy with a fixed buffer newPool for i := 0; i < b.N; i++ { proxy := NewProxy( context.Background(), - pool, + newPool, plugin.NewRegistry( context.Background(), config.Loose, @@ -172,14 +172,14 @@ func BenchmarkNewProxyElastic(b *testing.B) { NoColor: true, }) - // Create a connection pool - pool := pool.NewPool(context.Background(), config.EmptyPoolCapacity) + // Create a connection newPool + newPool := pool.NewPool(context.Background(), config.EmptyPoolCapacity) - // Create a proxy with an elastic buffer pool + // Create a proxy with an elastic buffer newPool for i := 0; i < b.N; i++ { proxy := NewProxy( context.Background(), - pool, + newPool, plugin.NewRegistry( context.Background(), config.Loose, @@ -216,8 +216,8 @@ func BenchmarkProxyConnectDisconnect(b *testing.B) { NoColor: true, }) - // Create a connection pool - pool := pool.NewPool(context.Background(), 1) + // Create a connection newPool + newPool := pool.NewPool(context.Background(), 1) clientConfig := config.Client{ Network: "tcp", @@ -229,12 +229,12 @@ func BenchmarkProxyConnectDisconnect(b *testing.B) { TCPKeepAlive: false, TCPKeepAlivePeriod: config.DefaultTCPKeepAlivePeriod, } - pool.Put("client", NewClient(context.Background(), &clientConfig, logger)) //nolint:errcheck + newPool.Put("client", NewClient(context.Background(), &clientConfig, logger)) //nolint:errcheck - // Create a proxy with a fixed buffer pool + // Create a proxy with a fixed buffer newPool proxy := NewProxy( context.Background(), - pool, + newPool, plugin.NewRegistry( context.Background(), config.Loose, @@ -252,12 +252,12 @@ func BenchmarkProxyConnectDisconnect(b *testing.B) { config.DefaultPluginTimeout) defer proxy.Shutdown() - gconn := testGNetConnection{} + conn := testConnection{} // Connect to the proxy for i := 0; i < b.N; i++ { - proxy.Connect(gconn.Conn) //nolint:errcheck - proxy.Disconnect(&gconn) //nolint:errcheck + proxy.Connect(conn.Conn) //nolint:errcheck + proxy.Disconnect(&conn) //nolint:errcheck } } @@ -270,8 +270,8 @@ func BenchmarkProxyPassThrough(b *testing.B) { NoColor: true, }) - // Create a connection pool - pool := pool.NewPool(context.Background(), 1) + // Create a connection newPool + newPool := pool.NewPool(context.Background(), 1) clientConfig := config.Client{ Network: "tcp", @@ -283,12 +283,12 @@ func BenchmarkProxyPassThrough(b *testing.B) { TCPKeepAlive: false, TCPKeepAlivePeriod: config.DefaultTCPKeepAlivePeriod, } - pool.Put("client", NewClient(context.Background(), &clientConfig, logger)) //nolint:errcheck + newPool.Put("client", NewClient(context.Background(), &clientConfig, logger)) //nolint:errcheck - // Create a proxy with a fixed buffer pool + // Create a proxy with a fixed buffer newPool proxy := NewProxy( context.Background(), - pool, + newPool, plugin.NewRegistry( context.Background(), config.Loose, @@ -306,13 +306,16 @@ func BenchmarkProxyPassThrough(b *testing.B) { config.DefaultPluginTimeout) defer proxy.Shutdown() - gconn := testGNetConnection{} - proxy.Connect(gconn.Conn) //nolint:errcheck - defer proxy.Disconnect(&gconn) //nolint:errcheck + conn := testConnection{} + proxy.Connect(conn.Conn) //nolint:errcheck + defer proxy.Disconnect(&conn) //nolint:errcheck + + stack := NewStack() // Connect to the proxy for i := 0; i < b.N; i++ { - proxy.PassThrough(&gconn) //nolint:errcheck + proxy.PassThroughToClient(&conn, stack) //nolint:errcheck + proxy.PassThroughToServer(&conn, stack) //nolint:errcheck } } @@ -325,8 +328,8 @@ func BenchmarkProxyIsHealthyAndIsExhausted(b *testing.B) { NoColor: true, }) - // Create a connection pool - pool := pool.NewPool(context.Background(), 1) + // Create a connection newPool + newPool := pool.NewPool(context.Background(), 1) clientConfig := config.Client{ Network: "tcp", @@ -339,12 +342,12 @@ func BenchmarkProxyIsHealthyAndIsExhausted(b *testing.B) { TCPKeepAlivePeriod: config.DefaultTCPKeepAlivePeriod, } client := NewClient(context.Background(), &clientConfig, logger) - pool.Put("client", client) //nolint:errcheck + newPool.Put("client", client) //nolint:errcheck - // Create a proxy with a fixed buffer pool + // Create a proxy with a fixed buffer newPool proxy := NewProxy( context.Background(), - pool, + newPool, plugin.NewRegistry( context.Background(), config.Loose, @@ -362,13 +365,13 @@ func BenchmarkProxyIsHealthyAndIsExhausted(b *testing.B) { config.DefaultPluginTimeout) defer proxy.Shutdown() - gconn := testGNetConnection{} - proxy.Connect(gconn.Conn) //nolint:errcheck - defer proxy.Disconnect(&gconn) //nolint:errcheck + conn := testConnection{} + proxy.Connect(conn.Conn) //nolint:errcheck + defer proxy.Disconnect(&conn) //nolint:errcheck // Connect to the proxy for i := 0; i < b.N; i++ { - proxy.IsHealty(client) //nolint:errcheck + proxy.IsHealthy(client) //nolint:errcheck proxy.IsExhausted() } } @@ -382,8 +385,8 @@ func BenchmarkProxyAvailableAndBusyConnections(b *testing.B) { NoColor: true, }) - // Create a connection pool - pool := pool.NewPool(context.Background(), 1) + // Create a connection newPool + newPool := pool.NewPool(context.Background(), 1) clientConfig := config.Client{ Network: "tcp", @@ -396,12 +399,12 @@ func BenchmarkProxyAvailableAndBusyConnections(b *testing.B) { TCPKeepAlivePeriod: config.DefaultTCPKeepAlivePeriod, } client := NewClient(context.Background(), &clientConfig, logger) - pool.Put("client", client) //nolint:errcheck + newPool.Put("client", client) //nolint:errcheck - // Create a proxy with a fixed buffer pool + // Create a proxy with a fixed buffer newPool proxy := NewProxy( context.Background(), - pool, + newPool, plugin.NewRegistry( context.Background(), config.Loose, @@ -419,9 +422,9 @@ func BenchmarkProxyAvailableAndBusyConnections(b *testing.B) { config.DefaultPluginTimeout) defer proxy.Shutdown() - gconn := testGNetConnection{} - proxy.Connect(gconn.Conn) //nolint:errcheck - defer proxy.Disconnect(&gconn) //nolint:errcheck + conn := testConnection{} + proxy.Connect(conn.Conn) //nolint:errcheck + defer proxy.Disconnect(&conn) //nolint:errcheck // Connect to the proxy for i := 0; i < b.N; i++ { diff --git a/network/server.go b/network/server.go index cfba1da2..71738b08 100644 --- a/network/server.go +++ b/network/server.go @@ -4,8 +4,10 @@ import ( "context" "errors" "fmt" - "io" + "net" "os" + "strconv" + "sync" "time" v1 "github.com/gatewayd-io/gatewayd-plugin-sdk/plugin/v1" @@ -13,24 +15,35 @@ import ( gerr "github.com/gatewayd-io/gatewayd/errors" "github.com/gatewayd-io/gatewayd/metrics" "github.com/gatewayd-io/gatewayd/plugin" - "github.com/panjf2000/gnet/v2" "github.com/rs/zerolog" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" ) +type Option struct { + EnableTicker bool +} + +type Action int + +const ( + None Action = iota + Close + Shutdown +) + type Server struct { - gnet.BuiltinEventEngine - engine gnet.Engine + engine Engine proxy IProxy logger zerolog.Logger pluginRegistry *plugin.Registry ctx context.Context //nolint:containedctx pluginTimeout time.Duration + mu *sync.RWMutex Network string // tcp/udp/unix Address string - Options []gnet.Option + Options Option Status config.Status TickInterval time.Duration } @@ -38,7 +51,7 @@ type Server struct { // OnBoot is called when the server is booted. It calls the OnBooting and OnBooted hooks. // It also sets the status to running, which is used to determine if the server should be running // or shutdown. -func (s *Server) OnBoot(engine gnet.Engine) gnet.Action { +func (s *Server) OnBoot(engine Engine) Action { _, span := otel.Tracer("gatewayd").Start(s.ctx, "OnBoot") defer span.End() @@ -60,9 +73,14 @@ func (s *Server) OnBoot(engine gnet.Engine) gnet.Action { s.engine = engine // Set the server status to running. + s.mu.Lock() s.Status = config.Running + s.mu.Unlock() // Run the OnBooted hooks. + pluginTimeoutCtx, cancel = context.WithTimeout(context.Background(), s.pluginTimeout) + defer cancel() + _, err = s.pluginRegistry.Run( pluginTimeoutCtx, map[string]interface{}{"status": fmt.Sprint(s.Status)}, @@ -75,16 +93,16 @@ func (s *Server) OnBoot(engine gnet.Engine) gnet.Action { s.logger.Debug().Msg("GatewayD booted") - return gnet.None + return None } // OnOpen is called when a new connection is opened. It calls the OnOpening and OnOpened hooks. // It also checks if the server is at the soft or hard limit and closes the connection if it is. -func (s *Server) OnOpen(gconn gnet.Conn) ([]byte, gnet.Action) { +func (s *Server) OnOpen(conn net.Conn) ([]byte, Action) { _, span := otel.Tracer("gatewayd").Start(s.ctx, "OnOpen") defer span.End() - s.logger.Debug().Str("from", RemoteAddr(gconn)).Msg( + s.logger.Debug().Str("from", RemoteAddr(conn)).Msg( "GatewayD is opening a connection") pluginTimeoutCtx, cancel := context.WithTimeout(context.Background(), s.pluginTimeout) @@ -92,8 +110,8 @@ func (s *Server) OnOpen(gconn gnet.Conn) ([]byte, gnet.Action) { // Run the OnOpening hooks. onOpeningData := map[string]interface{}{ "client": map[string]interface{}{ - "local": LocalAddr(gconn), - "remote": RemoteAddr(gconn), + "local": LocalAddr(conn), + "remote": RemoteAddr(conn), }, } _, err := s.pluginRegistry.Run( @@ -107,24 +125,27 @@ func (s *Server) OnOpen(gconn gnet.Conn) ([]byte, gnet.Action) { // Use the proxy to connect to the backend. Close the connection if the pool is exhausted. // This effectively get a connection from the pool and puts both the incoming and the server // connections in the pool of the busy connections. - if err := s.proxy.Connect(gconn); err != nil { + if err := s.proxy.Connect(conn); err != nil { if errors.Is(err, gerr.ErrPoolExhausted) { span.RecordError(err) - return nil, gnet.Close + return nil, Close } // This should never happen. // TODO: Send error to client or retry connection s.logger.Error().Err(err).Msg("Failed to connect to proxy") span.RecordError(err) - return nil, gnet.None + return nil, None } // Run the OnOpened hooks. + pluginTimeoutCtx, cancel = context.WithTimeout(context.Background(), s.pluginTimeout) + defer cancel() + onOpenedData := map[string]interface{}{ "client": map[string]interface{}{ - "local": LocalAddr(gconn), - "remote": RemoteAddr(gconn), + "local": LocalAddr(conn), + "remote": RemoteAddr(conn), }, } _, err = s.pluginRegistry.Run( @@ -137,26 +158,27 @@ func (s *Server) OnOpen(gconn gnet.Conn) ([]byte, gnet.Action) { metrics.ClientConnections.Inc() - return nil, gnet.None + return nil, None } // OnClose is called when a connection is closed. It calls the OnClosing and OnClosed hooks. // It also recycles the connection back to the available connection pool, unless the pool // is elastic and reuse is disabled. -func (s *Server) OnClose(gconn gnet.Conn, err error) gnet.Action { +func (s *Server) OnClose(conn net.Conn, err error) Action { _, span := otel.Tracer("gatewayd").Start(s.ctx, "OnClose") defer span.End() - s.logger.Debug().Str("from", RemoteAddr(gconn)).Msg( + s.logger.Debug().Str("from", RemoteAddr(conn)).Msg( "GatewayD is closing a connection") + // Run the OnClosing hooks. pluginTimeoutCtx, cancel := context.WithTimeout(context.Background(), s.pluginTimeout) defer cancel() - // Run the OnClosing hooks. + data := map[string]interface{}{ "client": map[string]interface{}{ - "local": LocalAddr(gconn), - "remote": RemoteAddr(gconn), + "local": LocalAddr(conn), + "remote": RemoteAddr(conn), }, "error": "", } @@ -172,26 +194,39 @@ func (s *Server) OnClose(gconn gnet.Conn, err error) gnet.Action { span.AddEvent("Ran the OnClosing hooks") // Shutdown the server if there are no more connections and the server is stopped. - // This is used to shutdown the server gracefully. + // This is used to shut down the server gracefully. + s.mu.Lock() if uint64(s.engine.CountConnections()) == 0 && s.Status == config.Stopped { span.AddEvent("Shutting down the server") - return gnet.Shutdown + s.mu.Unlock() + return Shutdown } + s.mu.Unlock() // Disconnect the connection from the proxy. This effectively removes the mapping between // the incoming and the server connections in the pool of the busy connections and either // recycles or disconnects the connections. - if err := s.proxy.Disconnect(gconn); err != nil { + if err := s.proxy.Disconnect(conn); err != nil { s.logger.Error().Err(err).Msg("Failed to disconnect the server connection") span.RecordError(err) - return gnet.Close + return Close + } + + // Close the incoming connection. + if err := conn.Close(); err != nil { + s.logger.Error().Err(err).Msg("Failed to close the incoming connection") + span.RecordError(err) + return Close } // Run the OnClosed hooks. + pluginTimeoutCtx, cancel = context.WithTimeout(context.Background(), s.pluginTimeout) + defer cancel() + data = map[string]interface{}{ "client": map[string]interface{}{ - "local": LocalAddr(gconn), - "remote": RemoteAddr(gconn), + "local": LocalAddr(conn), + "remote": RemoteAddr(conn), }, "error": "", } @@ -208,22 +243,23 @@ func (s *Server) OnClose(gconn gnet.Conn, err error) gnet.Action { metrics.ClientConnections.Dec() - return gnet.Close + return Close } // OnTraffic is called when data is received from the client. It calls the OnTraffic hooks. // It then passes the traffic to the proxied connection. -func (s *Server) OnTraffic(gconn gnet.Conn) gnet.Action { +func (s *Server) OnTraffic(conn net.Conn, stopConnection chan struct{}) Action { _, span := otel.Tracer("gatewayd").Start(s.ctx, "OnTraffic") defer span.End() + // Run the OnTraffic hooks. pluginTimeoutCtx, cancel := context.WithTimeout(context.Background(), s.pluginTimeout) defer cancel() - // Run the OnTraffic hooks. + onTrafficData := map[string]interface{}{ "client": map[string]interface{}{ - "local": LocalAddr(gconn), - "remote": RemoteAddr(gconn), + "local": LocalAddr(conn), + "remote": RemoteAddr(conn), }, } _, err := s.pluginRegistry.Run( @@ -234,37 +270,48 @@ func (s *Server) OnTraffic(gconn gnet.Conn) gnet.Action { } span.AddEvent("Ran the OnTraffic hooks") - // Pass the traffic from the client to server and vice versa. + stack := NewStack() + + // Pass the traffic from the client to server. // If there is an error, log it and close the connection. - if err := s.proxy.PassThrough(gconn); err != nil { - s.logger.Trace().Err(err).Msg("Failed to pass through traffic") - span.RecordError(err) - switch { - case errors.Is(err, gerr.ErrPoolExhausted), - errors.Is(err, gerr.ErrCastFailed), - errors.Is(err, gerr.ErrClientNotFound), - errors.Is(err, gerr.ErrClientNotConnected), - errors.Is(err, gerr.ErrClientSendFailed), - errors.Is(err, gerr.ErrClientReceiveFailed), - errors.Is(err, gerr.ErrHookTerminatedConnection), - errors.Is(err.Unwrap(), io.EOF): - // TODO: Fix bug in handling connection close - // See: https://github.com/gatewayd-io/gatewayd/issues/219 - return gnet.Close + go func(server *Server, conn net.Conn, stopConnection chan struct{}, stack *Stack) { + for { + server.logger.Trace().Msg("Passing through traffic from client to server") + if err := server.proxy.PassThroughToServer(conn, stack); err != nil { + server.logger.Trace().Err(err).Msg("Failed to pass through traffic") + span.RecordError(err) + stopConnection <- struct{}{} + break + } } - } - // Flush the connection to make sure all data is sent - gconn.Flush() + }(s, conn, stopConnection, stack) + + // Pass the traffic from the server to client. + // If there is an error, log it and close the connection. + go func(server *Server, conn net.Conn, stopConnection chan struct{}, stack *Stack) { + for { + server.logger.Debug().Msg("Passing through traffic from server to client") + if err := server.proxy.PassThroughToClient(conn, stack); err != nil { + server.logger.Trace().Err(err).Msg("Failed to pass through traffic") + span.RecordError(err) + stopConnection <- struct{}{} + break + } + } + }(s, conn, stopConnection, stack) + + <-stopConnection + stack.Clear() - return gnet.None + return Close } // OnShutdown is called when the server is shutting down. It calls the OnShutdown hooks. -func (s *Server) OnShutdown(gnet.Engine) { +func (s *Server) OnShutdown() { _, span := otel.Tracer("gatewayd").Start(s.ctx, "OnShutdown") defer span.End() - s.logger.Debug().Msg("GatewayD is shutting down...") + s.logger.Debug().Msg("GatewayD is shutting down") pluginTimeoutCtx, cancel := context.WithTimeout(context.Background(), s.pluginTimeout) defer cancel() @@ -283,11 +330,13 @@ func (s *Server) OnShutdown(gnet.Engine) { s.proxy.Shutdown() // Set the server status to stopped. This is used to shutdown the server gracefully in OnClose. + s.mu.Lock() s.Status = config.Stopped + s.mu.Unlock() } // OnTick is called every TickInterval. It calls the OnTick hooks. -func (s *Server) OnTick() (time.Duration, gnet.Action) { +func (s *Server) OnTick() (time.Duration, Action) { _, span := otel.Tracer("gatewayd").Start(s.ctx, "OnTick") defer span.End() @@ -314,11 +363,11 @@ func (s *Server) OnTick() (time.Duration, gnet.Action) { // TickInterval is the interval at which the OnTick hooks are called. It can be adjusted // in the configuration file. - return s.TickInterval, gnet.None + return s.TickInterval, None } // Run starts the server and blocks until the server is stopped. It calls the OnRun hooks. -func (s *Server) Run() error { +func (s *Server) Run() *gerr.GatewayDError { _, span := otel.Tracer("gatewayd").Start(s.ctx, "Run") defer span.End() @@ -334,7 +383,7 @@ func (s *Server) Run() error { pluginTimeoutCtx, cancel := context.WithTimeout(context.Background(), s.pluginTimeout) defer cancel() // Run the OnRun hooks. - // Since gnet.Run is blocking, we need to run OnRun before it. + // Since Run is blocking, we need to run OnRun before it. onRunData := map[string]interface{}{"address": addr} if err != nil && err.Unwrap() != nil { onRunData["error"] = err.OriginalError.Error() @@ -357,15 +406,120 @@ func (s *Server) Run() error { } } - // Start the server. - origErr := gnet.Run(s, s.Network+"://"+addr, s.Options...) + if action := s.OnBoot(s.engine); action != None { + return nil + } + + listener, origErr := net.Listen(s.Network, addr) if origErr != nil { - s.logger.Error().Err(origErr).Msg("Failed to start server") - span.RecordError(origErr) - return gerr.ErrFailedToStartServer.Wrap(origErr) + s.logger.Error().Err(origErr).Msg("Server failed to start listening") + return gerr.ErrServerListenFailed.Wrap(origErr) } + s.engine.listener = listener + defer s.engine.listener.Close() - return nil + if s.engine.listener == nil { + s.logger.Error().Msg("Server is not properly initialized") + return nil + } + + var port string + s.engine.host, port, origErr = net.SplitHostPort(s.engine.listener.Addr().String()) + if origErr != nil { + s.logger.Error().Err(origErr).Msg("Failed to split host and port") + return gerr.ErrSplitHostPortFailed.Wrap(origErr) + } + + if s.engine.port, origErr = strconv.Atoi(port); origErr != nil { + s.logger.Error().Err(origErr).Msg("Failed to convert port to integer") + return gerr.ErrCastFailed.Wrap(origErr) + } + + go func(server *Server) { + <-server.engine.stopServer + server.OnShutdown() + server.logger.Debug().Msg("Server stopped") + }(s) + + go func(server *Server) { + if !server.Options.EnableTicker { + return + } + + for { + select { + case <-server.engine.stopServer: + return + default: + interval, action := server.OnTick() + if action == Shutdown { + server.OnShutdown() + return + } + if interval == time.Duration(0) { + return + } + time.Sleep(interval) + } + } + }(s) + + s.engine.running.Store(true) + + for { + select { + case <-s.engine.stopServer: + s.logger.Info().Msg("Server stopped") + return nil + default: + conn, err := s.engine.listener.Accept() + if err != nil { + if !s.engine.running.Load() { + return nil + } + s.logger.Error().Err(err).Msg("Failed to accept connection") + return gerr.ErrAcceptFailed.Wrap(err) + } + + if out, action := s.OnOpen(conn); action != None { + if _, err := conn.Write(out); err != nil { + s.logger.Error().Err(err).Msg("Failed to write to connection") + } + conn.Close() + if action == Shutdown { + s.OnShutdown() + return nil + } + } + s.engine.mu.Lock() + s.engine.connections++ + s.engine.mu.Unlock() + + // For every new connection, a new unbuffered channel is created to help + // stop the proxy, recycle the server connection and close stale connections. + stopConnection := make(chan struct{}) + go func(server *Server, conn net.Conn, stopConnection chan struct{}) { + if action := server.OnTraffic(conn, stopConnection); action == Close { + stopConnection <- struct{}{} + } + }(s, conn, stopConnection) + + go func(server *Server, conn net.Conn, stopConnection chan struct{}) { + for { + select { + case <-stopConnection: + server.engine.mu.Lock() + server.engine.connections-- + server.engine.mu.Unlock() + server.OnClose(conn, err) + return + case <-server.engine.stopServer: + return + } + } + }(s, conn, stopConnection) + } + } } // Shutdown stops the server. @@ -377,7 +531,9 @@ func (s *Server) Shutdown() { s.proxy.Shutdown() // Set the server status to stopped. This is used to shutdown the server gracefully in OnClose. + s.mu.Lock() s.Status = config.Stopped + s.mu.Unlock() // Shutdown the server. if err := s.engine.Stop(context.Background()); err != nil { @@ -392,6 +548,8 @@ func (s *Server) IsRunning() bool { defer span.End() span.SetAttributes(attribute.Bool("status", s.Status == config.Running)) + s.mu.Lock() + defer s.mu.Unlock() return s.Status == config.Running } @@ -400,7 +558,7 @@ func NewServer( ctx context.Context, network, address string, tickInterval time.Duration, - options []gnet.Option, + options Option, proxy IProxy, logger zerolog.Logger, pluginRegistry *plugin.Registry, @@ -421,6 +579,8 @@ func NewServer( logger: logger, pluginRegistry: pluginRegistry, pluginTimeout: pluginTimeout, + mu: &sync.RWMutex{}, + engine: NewEngine(logger), } // Try to resolve the address and log an error if it can't be resolved. diff --git a/network/server_test.go b/network/server_test.go index 2a3028c6..9b0643c2 100644 --- a/network/server_test.go +++ b/network/server_test.go @@ -2,10 +2,12 @@ package network import ( "bufio" + "bytes" "context" "errors" "io" "os" + "sync" "testing" "time" @@ -14,7 +16,7 @@ import ( "github.com/gatewayd-io/gatewayd/logging" "github.com/gatewayd-io/gatewayd/plugin" "github.com/gatewayd-io/gatewayd/pool" - "github.com/panjf2000/gnet/v2" + "github.com/prometheus/client_golang/prometheus" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" "google.golang.org/grpc" @@ -24,14 +26,16 @@ import ( func TestRunServer(t *testing.T) { errs := make(chan error) + // Reset prometheus metrics. + prometheus.DefaultRegisterer = prometheus.NewRegistry() + logger := logging.NewLogger(context.Background(), logging.LoggerConfig{ Output: []config.LogOutput{ - config.Console, config.File, }, TimeFormat: zerolog.TimeFormatUnix, ConsoleTimeFormat: time.RFC3339, - Level: zerolog.WarnLevel, + Level: zerolog.DebugLevel, NoColor: true, FileName: "server_test.log", }) @@ -56,9 +60,10 @@ func TestRunServer(t *testing.T) { errs <- errors.New("request is nil") //nolint:goerr113 } - logger.Info().Msg("Ingress traffic") if req, ok := paramsMap["request"].([]byte); ok { - assert.Equal(t, CreatePgStartupPacket(), req) + if !bytes.Equal(req, CreatePgStartupPacket()) { + errs <- errors.New("request does not match") //nolint:goerr113 + } } else { errs <- errors.New("request is not a []byte") //nolint:goerr113 } @@ -80,7 +85,9 @@ func TestRunServer(t *testing.T) { logger.Info().Msg("Ingress traffic") if req, ok := paramsMap["request"].([]byte); ok { - assert.Equal(t, CreatePgStartupPacket(), req) + if !bytes.Equal(req, CreatePgStartupPacket()) { + errs <- errors.New("request does not match") //nolint:goerr113 + } } else { errs <- errors.New("request is not a []byte") //nolint:goerr113 } @@ -143,22 +150,22 @@ func TestRunServer(t *testing.T) { TCPKeepAlivePeriod: config.DefaultTCPKeepAlivePeriod, } - // Create a connection pool. - pool := pool.NewPool(context.Background(), 3) + // Create a connection newPool. + newPool := pool.NewPool(context.Background(), 3) client1 := NewClient(context.Background(), &clientConfig, logger) - err := pool.Put(client1.ID, client1) + err := newPool.Put(client1.ID, client1) assert.Nil(t, err) client2 := NewClient(context.Background(), &clientConfig, logger) - err = pool.Put(client2.ID, client2) + err = newPool.Put(client2.ID, client2) assert.Nil(t, err) client3 := NewClient(context.Background(), &clientConfig, logger) - err = pool.Put(client3.ID, client3) + err = newPool.Put(client3.ID, client3) assert.Nil(t, err) - // Create a proxy with a fixed buffer pool. + // Create a proxy with a fixed buffer newPool. proxy := NewProxy( context.Background(), - pool, + newPool, pluginRegistry, false, false, @@ -173,11 +180,8 @@ func TestRunServer(t *testing.T) { "tcp", "127.0.0.1:15432", config.DefaultTickInterval, - []gnet.Option{ - gnet.WithMulticore(false), - gnet.WithReuseAddr(true), - gnet.WithReusePort(true), - gnet.WithTicker(true), // Enable ticker. + Option{ + EnableTicker: true, }, proxy, logger, @@ -186,43 +190,74 @@ func TestRunServer(t *testing.T) { ) assert.NotNil(t, server) - go func(server *Server, errs chan error) { - if err := server.Run(); err != nil { - errs <- err - } - close(errs) + stop := make(chan struct{}) + + var waitGroup sync.WaitGroup + + waitGroup.Add(1) + go func(t *testing.T, server *Server, pluginRegistry *plugin.Registry, stop chan struct{}, waitGroup *sync.WaitGroup) { + t.Helper() + for { + select { + case <-stop: + server.Shutdown() + pluginRegistry.Shutdown() + + // Wait for the server to stop. + time.Sleep(100 * time.Millisecond) - // Read the log file and check if the log file contains the expected log messages. - if _, err := os.Stat("server_test.log"); err == nil { - logFile, err := os.Open("server_test.log") - assert.NoError(t, err) - defer logFile.Close() + // Read the log file and check if the log file contains the expected log messages. + if _, err := os.Stat("server_test.log"); err == nil { + logFile, err := os.Open("server_test.log") + assert.NoError(t, err) - reader := bufio.NewReader(logFile) - assert.NotNil(t, reader) + reader := bufio.NewReader(logFile) + assert.NotNil(t, reader) - buffer, err := io.ReadAll(reader) - assert.NoError(t, err) - assert.Greater(t, len(buffer), 0) // The log file should not be empty. + buffer, err := io.ReadAll(reader) + assert.NoError(t, err) + assert.Greater(t, len(buffer), 0) // The log file should not be empty. + assert.NoError(t, logFile.Close()) - logLines := string(buffer) - assert.Contains(t, logLines, "GatewayD is running", "GatewayD should be running") - assert.Contains(t, logLines, "GatewayD is ticking...", "GatewayD should be ticking") - assert.Contains(t, logLines, "Ingress traffic", "Ingress traffic should be logged") - assert.Contains(t, logLines, "Egress traffic", "Egress traffic should be logged") - assert.Contains(t, logLines, "GatewayD is shutting down...", "GatewayD should be shutting down") + logLines := string(buffer) + assert.Contains(t, logLines, "GatewayD is running", "GatewayD should be running") + assert.Contains(t, logLines, "GatewayD is ticking...", "GatewayD should be ticking") + assert.Contains(t, logLines, "Ingress traffic", "Ingress traffic should be logged") + assert.Contains(t, logLines, "Egress traffic", "Egress traffic should be logged") + assert.Contains(t, logLines, "GatewayD is shutting down", "GatewayD should be shutting down") - assert.NoError(t, os.Remove("server_test.log")) + assert.NoError(t, os.Remove("server_test.log")) + } + waitGroup.Done() + return + case <-errs: + server.Shutdown() + pluginRegistry.Shutdown() + waitGroup.Done() + return + default: //nolint:staticcheck + } + } + }(t, server, pluginRegistry, stop, &waitGroup) + + waitGroup.Add(1) + go func(t *testing.T, server *Server, errs chan error, waitGroup *sync.WaitGroup) { + t.Helper() + if err := server.Run(); err != nil { + errs <- err + t.Fail() } - }(server, errs) + waitGroup.Done() + }(t, server, errs, &waitGroup) + + waitGroup.Add(1) + go func(t *testing.T, server *Server, proxy *Proxy, stop chan struct{}, waitGroup *sync.WaitGroup) { + t.Helper() + // Pause for a while to allow the server to start. + time.Sleep(500 * time.Millisecond) - //nolint:thelper - go func(t *testing.T, server *Server, proxy *Proxy) { for { if server.IsRunning() { - // Pause for a while to allow the server to start. - time.Sleep(500 * time.Millisecond) - client := NewClient( context.Background(), &config.Client{ @@ -264,19 +299,15 @@ func TestRunServer(t *testing.T) { // Test Prometheus metrics. CollectAndComparePrometheusMetrics(t) - // Clean up. client.Close() - // Pause for a while to allow the server to disconnect and shutdown. - time.Sleep(500 * time.Millisecond) - server.Shutdown() break } + time.Sleep(100 * time.Millisecond) } - }(t, server, proxy) + stop <- struct{}{} + close(stop) + waitGroup.Done() + }(t, server, proxy, stop, &waitGroup) - for err := range errs { - if err != nil { - t.Fatal(err) - } - } + waitGroup.Wait() } diff --git a/network/stack.go b/network/stack.go new file mode 100644 index 00000000..a3dd6cfb --- /dev/null +++ b/network/stack.go @@ -0,0 +1,82 @@ +package network + +import "sync" + +type Request struct { + Data []byte +} + +type Stack struct { + items []*Request + mu sync.RWMutex +} + +func (s *Stack) Push(req *Request) { + s.mu.Lock() + defer s.mu.Unlock() + + s.items = append(s.items, req) +} + +func (s *Stack) GetLastRequest() *Request { + s.mu.RLock() + defer s.mu.RUnlock() + + if len(s.items) == 0 { + return nil + } + + //nolint:staticcheck + for i := len(s.items) - 1; i >= 0; i-- { + return s.items[i] + } + + return nil +} + +func (s *Stack) PopLastRequest() *Request { + s.mu.Lock() + defer s.mu.Unlock() + + if len(s.items) == 0 { + return nil + } + + //nolint:staticcheck + for i := len(s.items) - 1; i >= 0; i-- { + req := s.items[i] + s.items = append(s.items[:i], s.items[i+1:]...) + return req + } + + return nil +} + +func (s *Stack) UpdateLastRequest(req *Request) { + s.mu.Lock() + defer s.mu.Unlock() + + if len(s.items) == 0 { + return + } + + //nolint:staticcheck + for i := len(s.items) - 1; i >= 0; i-- { + s.items[i] = req + return + } +} + +func (s *Stack) Clear() { + s.mu.Lock() + defer s.mu.Unlock() + + s.items = make([]*Request, 0) +} + +func NewStack() *Stack { + return &Stack{ + items: make([]*Request, 0), + mu: sync.RWMutex{}, + } +} diff --git a/network/utils.go b/network/utils.go index ff85f337..97fbf559 100644 --- a/network/utils.go +++ b/network/utils.go @@ -9,7 +9,6 @@ import ( "net" gerr "github.com/gatewayd-io/gatewayd/errors" - "github.com/panjf2000/gnet/v2" "github.com/rs/zerolog" ) @@ -52,19 +51,19 @@ func Resolve(network, address string, logger zerolog.Logger) (string, *gerr.Gate // trafficData creates the ingress/egress map for the traffic hooks. func trafficData( - gconn gnet.Conn, + conn net.Conn, client *Client, fields []Field, err interface{}, ) map[string]interface{} { - if gconn == nil || client == nil { + if conn == nil || client == nil { return nil } data := map[string]interface{}{ "client": map[string]interface{}{ - "local": LocalAddr(gconn), - "remote": RemoteAddr(gconn), + "local": LocalAddr(conn), + "remote": RemoteAddr(conn), }, "server": map[string]interface{}{ "local": client.LocalAddr(), @@ -126,17 +125,17 @@ func IsConnClosed(received int, err *gerr.GatewayDError) bool { } // LocalAddr returns the local address of the connection. -func LocalAddr(gconn gnet.Conn) string { - if gconn != nil && gconn.LocalAddr() != nil { - return gconn.LocalAddr().String() +func LocalAddr(conn net.Conn) string { + if conn != nil && conn.LocalAddr() != nil { + return conn.LocalAddr().String() } return "" } // RemoteAddr returns the remote address of the connection. -func RemoteAddr(gconn gnet.Conn) string { - if gconn != nil && gconn.RemoteAddr() != nil { - return gconn.RemoteAddr().String() +func RemoteAddr(conn net.Conn) string { + if conn != nil && conn.RemoteAddr() != nil { + return conn.RemoteAddr().String() } return "" } diff --git a/network/utils_test.go b/network/utils_test.go index 2ab4f0d4..8d6e1d77 100644 --- a/network/utils_test.go +++ b/network/utils_test.go @@ -10,7 +10,6 @@ import ( "github.com/gatewayd-io/gatewayd/config" "github.com/gatewayd-io/gatewayd/logging" - "github.com/panjf2000/gnet/v2" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" ) @@ -112,18 +111,18 @@ func BenchmarkResolveUnix(b *testing.B) { } } -type testGNetConnection struct { - gnet.Conn +type testConnection struct { + net.Conn } -func (c *testGNetConnection) LocalAddr() net.Addr { +func (c *testConnection) LocalAddr() net.Addr { return &net.TCPAddr{ IP: net.ParseIP("127.0.0.1"), Port: 0, } } -func (c *testGNetConnection) RemoteAddr() net.Addr { +func (c *testConnection) RemoteAddr() net.Addr { return &net.TCPAddr{ IP: net.ParseIP("127.0.0.1"), Port: 0, @@ -139,7 +138,7 @@ func BenchmarkTrafficData(b *testing.B) { NoColor: true, }) - gconn := &testGNetConnection{} + conn := &testConnection{} client := NewClient(context.Background(), &config.Client{ Network: "tcp", Address: "localhost:5432", @@ -159,7 +158,7 @@ func BenchmarkTrafficData(b *testing.B) { } err := "test error" for i := 0; i < b.N; i++ { - trafficData(gconn, client, fields, err) + trafficData(conn, client, fields, err) } } diff --git a/plugin/plugin_registry.go b/plugin/plugin_registry.go index b975d86e..1a72116f 100644 --- a/plugin/plugin_registry.go +++ b/plugin/plugin_registry.go @@ -6,7 +6,7 @@ import ( "encoding/hex" "sort" - semver "github.com/Masterminds/semver/v3" + "github.com/Masterminds/semver/v3" sdkPlugin "github.com/gatewayd-io/gatewayd-plugin-sdk/plugin" v1 "github.com/gatewayd-io/gatewayd-plugin-sdk/plugin/v1" "github.com/gatewayd-io/gatewayd/config" @@ -271,6 +271,8 @@ func (reg *Registry) Run( _, span := otel.Tracer(config.TracerName).Start(reg.ctx, "Run") defer span.End() + metrics.PluginHooksExecuted.Inc() + if ctx == nil { return nil, gerr.ErrNilContext } @@ -377,8 +379,6 @@ func (reg *Registry) Run( delete(reg.hooks[hookName], priority) } - metrics.PluginHooksExecuted.Inc() - return returnVal.AsMap(), nil } diff --git a/pool/pool.go b/pool/pool.go index 88eab31a..e902bde6 100644 --- a/pool/pool.go +++ b/pool/pool.go @@ -57,6 +57,12 @@ func (p *Pool) Put(key, value interface{}) *gerr.GatewayDError { span.RecordError(gerr.ErrPoolExhausted) return gerr.ErrPoolExhausted } + + if value == nil { + span.RecordError(gerr.ErrNilPointer) + return gerr.ErrNilPointer + } + p.pool.Store(key, value) return nil } @@ -80,6 +86,12 @@ func (p *Pool) GetOrPut(key, value interface{}) (interface{}, bool, *gerr.Gatewa span.RecordError(gerr.ErrPoolExhausted) return nil, false, gerr.ErrPoolExhausted } + + if value == nil { + span.RecordError(gerr.ErrNilPointer) + return nil, false, gerr.ErrNilPointer + } + val, loaded := p.pool.LoadOrStore(key, value) return val, loaded, nil }