From d094e0c1a1347da02bcaf71564a8189be009daac Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Sun, 8 Oct 2023 00:29:57 +0200 Subject: [PATCH 01/35] Replace gnet/v2 engine API with one that uses net stdlib --- network/engine.go | 150 ++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 150 insertions(+) create mode 100644 network/engine.go diff --git a/network/engine.go b/network/engine.go new file mode 100644 index 00000000..89144ed2 --- /dev/null +++ b/network/engine.go @@ -0,0 +1,150 @@ +package network + +import ( + "context" + "net" + "strconv" + "time" +) + +type Option struct { + EnableTicker bool +} + +type Action int + +const ( + None Action = iota + Close + Shutdown +) + +type TCPSocketOpt int + +const ( + TCPNoDelay TCPSocketOpt = iota + TCPDelay +) + +type Engine struct { + listener net.Listener + host string + port int + connections uint32 + stopServer chan struct{} +} + +func (engine *Engine) CountConnections() int { + return int(engine.connections) +} + +func (engine *Engine) Stop(ctx context.Context) error { + ctx, cancel := context.WithDeadline(ctx, time.Now().Add(2*time.Second)) + defer cancel() + + engine.stopServer <- struct{}{} + return nil +} + +// Run starts a server and connects all the handlers. +func Run(network, address string, server *Server, opts Option) error { + server.engine = Engine{ + connections: 0, + stopServer: make(chan struct{}), + } + + if action := server.OnBoot(server.engine); action != None { + return nil + } + + if ln, err := net.Listen(network, address); err != nil { + server.logger.Error().Err(err).Msg("Server failed to start listening") + return err + } else { + server.engine.listener = ln + } + defer server.engine.listener.Close() + + if server.engine.listener == nil { + server.logger.Error().Msg("Server is not properly initialized") + return nil + } + + if host, port, err := net.SplitHostPort(server.engine.listener.Addr().String()); err != nil { + server.logger.Error().Err(err).Msg("Failed to split host and port") + return err + } else { + server.engine.host = host + if server.engine.port, err = strconv.Atoi(port); err != nil { + server.logger.Error().Err(err).Msg("Failed to convert port to integer") + return err + } + } + + go func(server *Server) { + for { + select { + case <-server.engine.stopServer: + server.OnShutdown(server.engine) + server.logger.Debug().Msg("Server stopped") + } + } + }(server) + + go func(server *Server) { + if !server.Options.EnableTicker { + return + } + + for { + interval, action := server.OnTick() + if action == Shutdown { + server.OnShutdown(server.engine) + return + } + if interval == time.Duration(0) { + return + } + time.Sleep(interval) + } + }(server) + + for { + conn, err := server.engine.listener.Accept() + if err != nil { + server.logger.Error().Err(err).Msg("Failed to accept connection") + return err + } + + if out, action := server.OnOpen(conn); action != None { + conn.Write(out) + conn.Close() + if action == Shutdown { + server.OnShutdown(server.engine) + return nil + } + } + server.engine.connections++ + + // For every new connection, a new unbuffered channel is created to help + // stop the proxy, recycle the server connection and close stale connections. + stopConnection := make(chan struct{}) + go func(server *Server, conn net.Conn, stopConnection chan struct{}) { + if action := server.OnTraffic(conn, stopConnection); action == Close { + return + } + }(server, conn, stopConnection) + + go func(server *Server, conn net.Conn, stopConnection chan struct{}) { + for { + select { + case <-stopConnection: + server.engine.connections-- + if action := server.OnClose(conn, err); action == Close { + return + } + } + } + }(server, conn, stopConnection) + } +} From 4fc692a5cf88ec1b710eeb0b6d2a70f5dc60fbef Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Sun, 8 Oct 2023 00:36:05 +0200 Subject: [PATCH 02/35] Refactor server, proxy and client to adapt to the new API --- network/client.go | 21 +++- network/proxy.go | 249 ++++++++++++++++++++++++++++------------------ network/server.go | 129 +++++++++++++----------- network/utils.go | 21 ++-- 4 files changed, 251 insertions(+), 169 deletions(-) diff --git a/network/client.go b/network/client.go index 7526c0d1..3074b778 100644 --- a/network/client.go +++ b/network/client.go @@ -146,12 +146,23 @@ func (c *Client) Send(data []byte) (int, *gerr.GatewayDError) { _, span := otel.Tracer(config.TracerName).Start(c.ctx, "Send") defer span.End() - sent, err := c.Conn.Write(data) - if err != nil { - c.logger.Error().Err(err).Msg("Couldn't send data to the server") - span.RecordError(err) - return 0, gerr.ErrClientSendFailed.Wrap(err) + sent := 0 + received := len(data) + for { + if sent >= received { + break + } + + n, err := c.Conn.Write(data) + if err != nil { + c.logger.Error().Err(err).Msg("Couldn't send data to the server") + span.RecordError(err) + return 0, gerr.ErrClientSendFailed.Wrap(err) + } + + sent += n } + c.logger.Debug().Fields( map[string]interface{}{ "length": sent, diff --git a/network/proxy.go b/network/proxy.go index fbd04e8d..943113df 100644 --- a/network/proxy.go +++ b/network/proxy.go @@ -1,7 +1,11 @@ package network import ( + "bytes" "context" + "errors" + "io" + "net" "time" v1 "github.com/gatewayd-io/gatewayd-plugin-sdk/plugin/v1" @@ -12,15 +16,15 @@ import ( "github.com/gatewayd-io/gatewayd/pool" "github.com/getsentry/sentry-go" "github.com/go-co-op/gocron" - "github.com/panjf2000/gnet/v2" "github.com/rs/zerolog" "go.opentelemetry.io/otel" ) type IProxy interface { - Connect(gconn gnet.Conn) *gerr.GatewayDError - Disconnect(gconn gnet.Conn) *gerr.GatewayDError - PassThrough(gconn gnet.Conn) *gerr.GatewayDError + Connect(conn net.Conn) *gerr.GatewayDError + Disconnect(conn net.Conn) *gerr.GatewayDError + PassThroughToServer(conn net.Conn) *gerr.GatewayDError + PassThroughToClient(conn net.Conn) *gerr.GatewayDError IsHealty(cl *Client) (*Client, *gerr.GatewayDError) IsExhausted() bool Shutdown() @@ -123,7 +127,7 @@ func NewProxy( // Connect maps a server connection from the available connection pool to a incoming connection. // It returns an error if the pool is exhausted. If the pool is elastic, it creates a new client // and maps it to the incoming connection. -func (pr *Proxy) Connect(gconn gnet.Conn) *gerr.GatewayDError { +func (pr *Proxy) Connect(conn net.Conn) *gerr.GatewayDError { _, span := otel.Tracer(config.TracerName).Start(pr.ctx, "Connect") defer span.End() @@ -162,7 +166,7 @@ func (pr *Proxy) Connect(gconn gnet.Conn) *gerr.GatewayDError { span.RecordError(err) } - if err := pr.busyConnections.Put(gconn, client); err != nil { + if err := pr.busyConnections.Put(conn, client); err != nil { // This should never happen. span.RecordError(err) return err @@ -173,7 +177,7 @@ func (pr *Proxy) Connect(gconn gnet.Conn) *gerr.GatewayDError { fields := map[string]interface{}{ "function": "proxy.connect", "client": "unknown", - "server": RemoteAddr(gconn), + "server": RemoteAddr(conn), } if client.ID != "" { fields["client"] = client.ID[:7] @@ -198,11 +202,11 @@ func (pr *Proxy) Connect(gconn gnet.Conn) *gerr.GatewayDError { // Disconnect removes the client from the busy connection pool and tries to recycle // the server connection. -func (pr *Proxy) Disconnect(gconn gnet.Conn) *gerr.GatewayDError { +func (pr *Proxy) Disconnect(conn net.Conn) *gerr.GatewayDError { _, span := otel.Tracer(config.TracerName).Start(pr.ctx, "Disconnect") defer span.End() - client := pr.busyConnections.Pop(gconn) + client := pr.busyConnections.Pop(conn) //nolint:nestif if client != nil { if client, ok := client.(*Client); ok { @@ -251,24 +255,20 @@ func (pr *Proxy) Disconnect(gconn gnet.Conn) *gerr.GatewayDError { return nil } -// PassThrough sends the data from the client to the server and vice versa. -func (pr *Proxy) PassThrough(gconn gnet.Conn) *gerr.GatewayDError { +// PassThrough sends the data from the client to the server. +func (pr *Proxy) PassThroughToServer(conn net.Conn) *gerr.GatewayDError { _, span := otel.Tracer(config.TracerName).Start(pr.ctx, "PassThrough") defer span.End() - // TODO: Handle bi-directional traffic - // Currently the passthrough is a one-way street from the client to the server, that is, - // the client can send data to the server and receive the response back, but the server - // cannot take initiative and send data to the client. So, there should be another event-loop - // that listens for data from the server and sends it to the client. + // Check if the proxy has a egress client for the incoming connection. var client *Client - if pr.busyConnections.Get(gconn) == nil { + if pr.busyConnections.Get(conn) == nil { span.RecordError(gerr.ErrClientNotFound) return gerr.ErrClientNotFound } // Get the client from the busy connection pool. - if cl, ok := pr.busyConnections.Get(gconn).(*Client); ok { + if cl, ok := pr.busyConnections.Get(conn).(*Client); ok { client = cl } else { span.RecordError(gerr.ErrCastFailed) @@ -276,17 +276,22 @@ func (pr *Proxy) PassThrough(gconn gnet.Conn) *gerr.GatewayDError { } span.AddEvent("Got the client from the busy connection pool") + if !client.IsConnected() || !pr.isConnectionHealthy(conn) { + return gerr.ErrClientNotConnected + } + // Receive the request from the client. - request, origErr := pr.receiveTrafficFromClient(gconn) + request, origErr := pr.receiveTrafficFromClient(conn) span.AddEvent("Received traffic from client") + // Run the OnTrafficFromClient hooks. pluginTimeoutCtx, cancel := context.WithTimeout(context.Background(), pr.pluginTimeout) defer cancel() - // Run the OnTrafficFromClient hooks. + result, err := pr.pluginRegistry.Run( pluginTimeoutCtx, trafficData( - gconn, + conn, client, []Field{ { @@ -302,6 +307,12 @@ func (pr *Proxy) PassThrough(gconn gnet.Conn) *gerr.GatewayDError { } span.AddEvent("Ran the OnTrafficFromClient hooks") + if errors.Is(origErr, io.EOF) { + // Client closed the connection. + span.AddEvent("Client closed the connection") + return gerr.ErrClientNotConnected + } + // If the hook wants to terminate the connection, do it. if pr.shouldTerminate(result) { if modResponse, modReceived := pr.getPluginModifiedResponse(result); modResponse != nil { @@ -311,7 +322,7 @@ func (pr *Proxy) PassThrough(gconn gnet.Conn) *gerr.GatewayDError { metrics.TotalTrafficBytes.Observe(float64(modReceived)) span.AddEvent("Terminating connection") - return pr.sendTrafficToClient(gconn, modResponse, modReceived) + return pr.sendTrafficToClient(conn, modResponse, modReceived) } span.RecordError(gerr.ErrHookTerminatedConnection) return gerr.ErrHookTerminatedConnection @@ -330,7 +341,7 @@ func (pr *Proxy) PassThrough(gconn gnet.Conn) *gerr.GatewayDError { _, err = pr.pluginRegistry.Run( pluginTimeoutCtx, trafficData( - gconn, + conn, client, []Field{ { @@ -346,36 +357,39 @@ func (pr *Proxy) PassThrough(gconn gnet.Conn) *gerr.GatewayDError { } span.AddEvent("Ran the OnTrafficToServer hooks") - // Receive the response from the server. - received, response, err := pr.receiveTrafficFromServer(client) - span.AddEvent("Received traffic from server") + return nil +} - // The connection to the server is closed, so we MUST reconnect, - // otherwise the client will be stuck. - // TODO: Fix bug in handling connection close - // See: https://github.com/gatewayd-io/gatewayd/issues/219 - if IsConnClosed(received, err) || IsConnTimedOut(err) { - pr.logger.Debug().Fields( - map[string]interface{}{ - "function": "proxy.passthrough", - "local": client.LocalAddr(), - "remote": client.RemoteAddr(), - }).Msg("Client disconnected") +// PassThroughToClient sends the data from the server to the client. +func (pr *Proxy) PassThroughToClient(conn net.Conn) *gerr.GatewayDError { + _, span := otel.Tracer(config.TracerName).Start(pr.ctx, "PassThrough") + defer span.End() - client.Close() - client = NewClient(pr.ctx, pr.ClientConfig, pr.logger) - pr.busyConnections.Remove(gconn) - if err := pr.busyConnections.Put(gconn, client); err != nil { - span.RecordError(err) - // This should never happen - return err - } + var client *Client + if pr.busyConnections.Get(conn) == nil { + span.RecordError(gerr.ErrClientNotFound) + return gerr.ErrClientNotFound + } + + // Get the client from the busy connection pool. + if cl, ok := pr.busyConnections.Get(conn).(*Client); ok { + client = cl + } else { + span.RecordError(gerr.ErrCastFailed) + return gerr.ErrCastFailed + } + span.AddEvent("Got the client from the busy connection pool") + + if !client.IsConnected() || !pr.isConnectionHealthy(conn) { + return gerr.ErrClientNotConnected } + // Receive the response from the server. + received, response, err := pr.receiveTrafficFromServer(client) + span.AddEvent("Received traffic from server") + // If the response is empty, don't send anything, instead just close the ingress connection. - // TODO: Fix bug in handling connection close - // See: https://github.com/gatewayd-io/gatewayd/issues/219 - if received == 0 { + if received == 0 || err != nil { pr.logger.Debug().Fields( map[string]interface{}{ "function": "proxy.passthrough", @@ -387,17 +401,16 @@ func (pr *Proxy) PassThrough(gconn gnet.Conn) *gerr.GatewayDError { return err } + pluginTimeoutCtx, cancel := context.WithTimeout(context.Background(), pr.pluginTimeout) + defer cancel() + // Run the OnTrafficFromServer hooks. - result, err = pr.pluginRegistry.Run( + result, err := pr.pluginRegistry.Run( pluginTimeoutCtx, trafficData( - gconn, + conn, client, []Field{ - { - Name: "request", - Value: request, - }, { Name: "response", Value: response[:received], @@ -419,26 +432,22 @@ func (pr *Proxy) PassThrough(gconn gnet.Conn) *gerr.GatewayDError { } // Send the response to the client. - errVerdict := pr.sendTrafficToClient(gconn, response, received) + errVerdict := pr.sendTrafficToClient(conn, response, received) span.AddEvent("Sent traffic to client") // Run the OnTrafficToClient hooks. _, err = pr.pluginRegistry.Run( pluginTimeoutCtx, trafficData( - gconn, + conn, client, []Field{ - { - Name: "request", - Value: request, - }, { Name: "response", Value: response[:received], }, }, - err, + nil, ), v1.HookName_HOOK_NAME_ON_TRAFFIC_TO_CLIENT) if err != nil { @@ -501,8 +510,8 @@ func (pr *Proxy) Shutdown() { pr.logger.Debug().Msg("All available connections have been closed") pr.busyConnections.ForEach(func(key, value interface{}) bool { - if gconn, ok := key.(gnet.Conn); ok { - gconn.Close() + if conn, ok := key.(net.Conn); ok { + conn.Close() } if cl, ok := value.(*Client); ok { cl.Close() @@ -536,38 +545,60 @@ func (pr *Proxy) BusyConnections() []string { connections := make([]string, 0) pr.busyConnections.ForEach(func(key, _ interface{}) bool { - if gconn, ok := key.(gnet.Conn); ok { - connections = append(connections, RemoteAddr(gconn)) + if conn, ok := key.(net.Conn); ok { + connections = append(connections, RemoteAddr(conn)) } return true }) return connections } -// receiveTrafficFromClient is a function that receives data from the client. -func (pr *Proxy) receiveTrafficFromClient(gconn gnet.Conn) ([]byte, error) { +// receiveTrafficFromClient is a function that waits to receive data from the client. +func (pr *Proxy) receiveTrafficFromClient(conn net.Conn) ([]byte, error) { _, span := otel.Tracer(config.TracerName).Start(pr.ctx, "receiveTrafficFromClient") defer span.End() // request contains the data from the client. - request, err := gconn.Next(-1) - if err != nil { - pr.logger.Error().Err(err).Msg("Error reading from client") - span.RecordError(err) + received := 0 + buffer := bytes.NewBuffer(nil) + for { + chunk := make([]byte, pr.ClientConfig.ReceiveChunkSize) + read, err := conn.Read(chunk) + if read == 0 || err != nil { + pr.logger.Debug().Err(err).Msg("Error reading from client") + span.RecordError(err) + + metrics.BytesReceivedFromClient.Observe(float64(read)) + metrics.TotalTrafficBytes.Observe(float64(read)) + + return chunk[:read], err + } + + received += read + buffer.Write(chunk[:read]) + + if received == 0 || received < pr.ClientConfig.ReceiveChunkSize { + break + } + + if !pr.isConnectionHealthy(conn) { + break + } } + + length := len(buffer.Bytes()) pr.logger.Debug().Fields( map[string]interface{}{ - "length": len(request), - "local": LocalAddr(gconn), - "remote": RemoteAddr(gconn), + "length": length, + "local": LocalAddr(conn), + "remote": RemoteAddr(conn), }, ).Msg("Received data from client") - - metrics.BytesReceivedFromClient.Observe(float64(len(request))) - metrics.TotalTrafficBytes.Observe(float64(len(request))) + metrics.BytesReceivedFromClient.Observe(float64(length)) + metrics.TotalTrafficBytes.Observe(float64(length)) //nolint:wrapcheck - return request, err + return buffer.Bytes(), nil } // sendTrafficToServer is a function that sends data to the server. @@ -575,6 +606,11 @@ func (pr *Proxy) sendTrafficToServer(client *Client, request []byte) (int, *gerr _, span := otel.Tracer(config.TracerName).Start(pr.ctx, "sendTrafficToServer") defer span.End() + if len(request) == 0 { + pr.logger.Trace().Msg("Empty request") + return 0, nil + } + // Send the request to the server. sent, err := client.Send(request) if err != nil { @@ -620,30 +656,39 @@ func (pr *Proxy) receiveTrafficFromServer(client *Client) (int, []byte, *gerr.Ga // sendTrafficToClient is a function that sends data to the client. func (pr *Proxy) sendTrafficToClient( - gconn gnet.Conn, response []byte, received int, + conn net.Conn, response []byte, received int, ) *gerr.GatewayDError { _, span := otel.Tracer(config.TracerName).Start(pr.ctx, "sendTrafficToClient") defer span.End() // Send the response to the client async. - origErr := gconn.AsyncWrite(response[:received], func(gconn gnet.Conn, err error) error { - pr.logger.Debug().Fields( - map[string]interface{}{ - "function": "proxy.passthrough", - "length": received, - "local": LocalAddr(gconn), - "remote": RemoteAddr(gconn), - }, - ).Msg("Sent data to client") - span.RecordError(err) - return err - }) - if origErr != nil { - pr.logger.Error().Err(origErr).Msg("Error writing to client") - span.RecordError(origErr) - return gerr.ErrServerSendFailed.Wrap(origErr) + sent := 0 + for { + if sent >= received { + break + } + + n, origErr := conn.Write(response[:received]) + if origErr != nil { + pr.logger.Error().Err(origErr).Msg("Error writing to client") + span.RecordError(origErr) + return gerr.ErrServerSendFailed.Wrap(origErr) + } + + sent += n } + pr.logger.Debug().Fields( + map[string]interface{}{ + "function": "proxy.passthrough", + "length": sent, + "local": LocalAddr(conn), + "remote": RemoteAddr(conn), + }, + ).Msg("Sent data to client") + + span.AddEvent("Sent data to client") + metrics.BytesSentToClient.Observe(float64(received)) metrics.TotalTrafficBytes.Observe(float64(received)) @@ -698,3 +743,17 @@ func (pr *Proxy) getPluginModifiedResponse(result map[string]interface{}) ([]byt return nil, 0 } + +func (pr *Proxy) isConnectionHealthy(conn net.Conn) bool { + if n, err := conn.Read([]byte{}); n == 0 && err != nil { + pr.logger.Debug().Fields( + map[string]interface{}{ + "remote": RemoteAddr(conn), + "local": LocalAddr(conn), + "reason": "read 0 bytes", + }).Msg("Connection to client is closed") + return false + } + + return true +} diff --git a/network/server.go b/network/server.go index cfba1da2..4ba918e9 100644 --- a/network/server.go +++ b/network/server.go @@ -4,7 +4,7 @@ import ( "context" "errors" "fmt" - "io" + "net" "os" "time" @@ -13,15 +13,13 @@ import ( gerr "github.com/gatewayd-io/gatewayd/errors" "github.com/gatewayd-io/gatewayd/metrics" "github.com/gatewayd-io/gatewayd/plugin" - "github.com/panjf2000/gnet/v2" "github.com/rs/zerolog" "go.opentelemetry.io/otel" "go.opentelemetry.io/otel/attribute" ) type Server struct { - gnet.BuiltinEventEngine - engine gnet.Engine + engine Engine proxy IProxy logger zerolog.Logger pluginRegistry *plugin.Registry @@ -30,7 +28,7 @@ type Server struct { Network string // tcp/udp/unix Address string - Options []gnet.Option + Options Option Status config.Status TickInterval time.Duration } @@ -38,7 +36,7 @@ type Server struct { // OnBoot is called when the server is booted. It calls the OnBooting and OnBooted hooks. // It also sets the status to running, which is used to determine if the server should be running // or shutdown. -func (s *Server) OnBoot(engine gnet.Engine) gnet.Action { +func (s *Server) OnBoot(engine Engine) Action { _, span := otel.Tracer("gatewayd").Start(s.ctx, "OnBoot") defer span.End() @@ -75,16 +73,16 @@ func (s *Server) OnBoot(engine gnet.Engine) gnet.Action { s.logger.Debug().Msg("GatewayD booted") - return gnet.None + return None } // OnOpen is called when a new connection is opened. It calls the OnOpening and OnOpened hooks. // It also checks if the server is at the soft or hard limit and closes the connection if it is. -func (s *Server) OnOpen(gconn gnet.Conn) ([]byte, gnet.Action) { +func (s *Server) OnOpen(conn net.Conn) ([]byte, Action) { _, span := otel.Tracer("gatewayd").Start(s.ctx, "OnOpen") defer span.End() - s.logger.Debug().Str("from", RemoteAddr(gconn)).Msg( + s.logger.Debug().Str("from", RemoteAddr(conn)).Msg( "GatewayD is opening a connection") pluginTimeoutCtx, cancel := context.WithTimeout(context.Background(), s.pluginTimeout) @@ -92,8 +90,8 @@ func (s *Server) OnOpen(gconn gnet.Conn) ([]byte, gnet.Action) { // Run the OnOpening hooks. onOpeningData := map[string]interface{}{ "client": map[string]interface{}{ - "local": LocalAddr(gconn), - "remote": RemoteAddr(gconn), + "local": LocalAddr(conn), + "remote": RemoteAddr(conn), }, } _, err := s.pluginRegistry.Run( @@ -107,24 +105,24 @@ func (s *Server) OnOpen(gconn gnet.Conn) ([]byte, gnet.Action) { // Use the proxy to connect to the backend. Close the connection if the pool is exhausted. // This effectively get a connection from the pool and puts both the incoming and the server // connections in the pool of the busy connections. - if err := s.proxy.Connect(gconn); err != nil { + if err := s.proxy.Connect(conn); err != nil { if errors.Is(err, gerr.ErrPoolExhausted) { span.RecordError(err) - return nil, gnet.Close + return nil, Close } // This should never happen. // TODO: Send error to client or retry connection s.logger.Error().Err(err).Msg("Failed to connect to proxy") span.RecordError(err) - return nil, gnet.None + return nil, None } // Run the OnOpened hooks. onOpenedData := map[string]interface{}{ "client": map[string]interface{}{ - "local": LocalAddr(gconn), - "remote": RemoteAddr(gconn), + "local": LocalAddr(conn), + "remote": RemoteAddr(conn), }, } _, err = s.pluginRegistry.Run( @@ -137,26 +135,27 @@ func (s *Server) OnOpen(gconn gnet.Conn) ([]byte, gnet.Action) { metrics.ClientConnections.Inc() - return nil, gnet.None + return nil, None } // OnClose is called when a connection is closed. It calls the OnClosing and OnClosed hooks. // It also recycles the connection back to the available connection pool, unless the pool // is elastic and reuse is disabled. -func (s *Server) OnClose(gconn gnet.Conn, err error) gnet.Action { +func (s *Server) OnClose(conn net.Conn, err error) Action { _, span := otel.Tracer("gatewayd").Start(s.ctx, "OnClose") defer span.End() - s.logger.Debug().Str("from", RemoteAddr(gconn)).Msg( + s.logger.Debug().Str("from", RemoteAddr(conn)).Msg( "GatewayD is closing a connection") + // Run the OnClosing hooks. pluginTimeoutCtx, cancel := context.WithTimeout(context.Background(), s.pluginTimeout) defer cancel() - // Run the OnClosing hooks. + data := map[string]interface{}{ "client": map[string]interface{}{ - "local": LocalAddr(gconn), - "remote": RemoteAddr(gconn), + "local": LocalAddr(conn), + "remote": RemoteAddr(conn), }, "error": "", } @@ -175,23 +174,30 @@ func (s *Server) OnClose(gconn gnet.Conn, err error) gnet.Action { // This is used to shutdown the server gracefully. if uint64(s.engine.CountConnections()) == 0 && s.Status == config.Stopped { span.AddEvent("Shutting down the server") - return gnet.Shutdown + return Shutdown } // Disconnect the connection from the proxy. This effectively removes the mapping between // the incoming and the server connections in the pool of the busy connections and either // recycles or disconnects the connections. - if err := s.proxy.Disconnect(gconn); err != nil { + if err := s.proxy.Disconnect(conn); err != nil { s.logger.Error().Err(err).Msg("Failed to disconnect the server connection") span.RecordError(err) - return gnet.Close + return Close + } + + // Close the incoming connection. + if err := conn.Close(); err != nil { + s.logger.Error().Err(err).Msg("Failed to close the incoming connection") + span.RecordError(err) + return Close } // Run the OnClosed hooks. data = map[string]interface{}{ "client": map[string]interface{}{ - "local": LocalAddr(gconn), - "remote": RemoteAddr(gconn), + "local": LocalAddr(conn), + "remote": RemoteAddr(conn), }, "error": "", } @@ -208,22 +214,23 @@ func (s *Server) OnClose(gconn gnet.Conn, err error) gnet.Action { metrics.ClientConnections.Dec() - return gnet.Close + return Close } // OnTraffic is called when data is received from the client. It calls the OnTraffic hooks. // It then passes the traffic to the proxied connection. -func (s *Server) OnTraffic(gconn gnet.Conn) gnet.Action { +func (s *Server) OnTraffic(conn net.Conn, stopConnection chan struct{}) Action { _, span := otel.Tracer("gatewayd").Start(s.ctx, "OnTraffic") defer span.End() + // Run the OnTraffic hooks. pluginTimeoutCtx, cancel := context.WithTimeout(context.Background(), s.pluginTimeout) defer cancel() - // Run the OnTraffic hooks. + onTrafficData := map[string]interface{}{ "client": map[string]interface{}{ - "local": LocalAddr(gconn), - "remote": RemoteAddr(gconn), + "local": LocalAddr(conn), + "remote": RemoteAddr(conn), }, } _, err := s.pluginRegistry.Run( @@ -234,33 +241,39 @@ func (s *Server) OnTraffic(gconn gnet.Conn) gnet.Action { } span.AddEvent("Ran the OnTraffic hooks") - // Pass the traffic from the client to server and vice versa. + // Pass the traffic from the client to server. // If there is an error, log it and close the connection. - if err := s.proxy.PassThrough(gconn); err != nil { - s.logger.Trace().Err(err).Msg("Failed to pass through traffic") - span.RecordError(err) - switch { - case errors.Is(err, gerr.ErrPoolExhausted), - errors.Is(err, gerr.ErrCastFailed), - errors.Is(err, gerr.ErrClientNotFound), - errors.Is(err, gerr.ErrClientNotConnected), - errors.Is(err, gerr.ErrClientSendFailed), - errors.Is(err, gerr.ErrClientReceiveFailed), - errors.Is(err, gerr.ErrHookTerminatedConnection), - errors.Is(err.Unwrap(), io.EOF): - // TODO: Fix bug in handling connection close - // See: https://github.com/gatewayd-io/gatewayd/issues/219 - return gnet.Close + go func(s *Server, conn net.Conn, stopConnection chan struct{}) { + for { + s.logger.Trace().Msg("Passing through traffic from client to server") + if err := s.proxy.PassThroughToServer(conn); err != nil { + s.logger.Trace().Err(err).Msg("Failed to pass through traffic") + span.RecordError(err) + stopConnection <- struct{}{} + break + } } - } - // Flush the connection to make sure all data is sent - gconn.Flush() + }(s, conn, stopConnection) + + // Pass the traffic from the server to client. + // If there is an error, log it and close the connection. + go func(s *Server, conn net.Conn, stopConnection chan struct{}) { + for { + s.logger.Debug().Msg("Passing through traffic from server to client") + if err := s.proxy.PassThroughToClient(conn); err != nil { + s.logger.Trace().Err(err).Msg("Failed to pass through traffic") + span.RecordError(err) + stopConnection <- struct{}{} + break + } + } + }(s, conn, stopConnection) - return gnet.None + return None } // OnShutdown is called when the server is shutting down. It calls the OnShutdown hooks. -func (s *Server) OnShutdown(gnet.Engine) { +func (s *Server) OnShutdown(Engine) { _, span := otel.Tracer("gatewayd").Start(s.ctx, "OnShutdown") defer span.End() @@ -287,7 +300,7 @@ func (s *Server) OnShutdown(gnet.Engine) { } // OnTick is called every TickInterval. It calls the OnTick hooks. -func (s *Server) OnTick() (time.Duration, gnet.Action) { +func (s *Server) OnTick() (time.Duration, Action) { _, span := otel.Tracer("gatewayd").Start(s.ctx, "OnTick") defer span.End() @@ -314,7 +327,7 @@ func (s *Server) OnTick() (time.Duration, gnet.Action) { // TickInterval is the interval at which the OnTick hooks are called. It can be adjusted // in the configuration file. - return s.TickInterval, gnet.None + return s.TickInterval, None } // Run starts the server and blocks until the server is stopped. It calls the OnRun hooks. @@ -334,7 +347,7 @@ func (s *Server) Run() error { pluginTimeoutCtx, cancel := context.WithTimeout(context.Background(), s.pluginTimeout) defer cancel() // Run the OnRun hooks. - // Since gnet.Run is blocking, we need to run OnRun before it. + // Since Run is blocking, we need to run OnRun before it. onRunData := map[string]interface{}{"address": addr} if err != nil && err.Unwrap() != nil { onRunData["error"] = err.OriginalError.Error() @@ -358,7 +371,7 @@ func (s *Server) Run() error { } // Start the server. - origErr := gnet.Run(s, s.Network+"://"+addr, s.Options...) + origErr := Run(s.Network, addr, s, s.Options) if origErr != nil { s.logger.Error().Err(origErr).Msg("Failed to start server") span.RecordError(origErr) @@ -400,7 +413,7 @@ func NewServer( ctx context.Context, network, address string, tickInterval time.Duration, - options []gnet.Option, + options Option, proxy IProxy, logger zerolog.Logger, pluginRegistry *plugin.Registry, diff --git a/network/utils.go b/network/utils.go index ff85f337..97fbf559 100644 --- a/network/utils.go +++ b/network/utils.go @@ -9,7 +9,6 @@ import ( "net" gerr "github.com/gatewayd-io/gatewayd/errors" - "github.com/panjf2000/gnet/v2" "github.com/rs/zerolog" ) @@ -52,19 +51,19 @@ func Resolve(network, address string, logger zerolog.Logger) (string, *gerr.Gate // trafficData creates the ingress/egress map for the traffic hooks. func trafficData( - gconn gnet.Conn, + conn net.Conn, client *Client, fields []Field, err interface{}, ) map[string]interface{} { - if gconn == nil || client == nil { + if conn == nil || client == nil { return nil } data := map[string]interface{}{ "client": map[string]interface{}{ - "local": LocalAddr(gconn), - "remote": RemoteAddr(gconn), + "local": LocalAddr(conn), + "remote": RemoteAddr(conn), }, "server": map[string]interface{}{ "local": client.LocalAddr(), @@ -126,17 +125,17 @@ func IsConnClosed(received int, err *gerr.GatewayDError) bool { } // LocalAddr returns the local address of the connection. -func LocalAddr(gconn gnet.Conn) string { - if gconn != nil && gconn.LocalAddr() != nil { - return gconn.LocalAddr().String() +func LocalAddr(conn net.Conn) string { + if conn != nil && conn.LocalAddr() != nil { + return conn.LocalAddr().String() } return "" } // RemoteAddr returns the remote address of the connection. -func RemoteAddr(gconn gnet.Conn) string { - if gconn != nil && gconn.RemoteAddr() != nil { - return gconn.RemoteAddr().String() +func RemoteAddr(conn net.Conn) string { + if conn != nil && conn.RemoteAddr() != nil { + return conn.RemoteAddr().String() } return "" } From ad69ae892a8e726d63f32d017992350414cc6914 Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Sun, 8 Oct 2023 00:37:20 +0200 Subject: [PATCH 03/35] Remove unused Server options --- cmd/run.go | 38 ++------------------------------------ config/config.go | 19 ++++--------------- config/getters.go | 17 ----------------- config/types.go | 29 ++++------------------------- gatewayd.yaml | 11 ----------- 5 files changed, 10 insertions(+), 104 deletions(-) diff --git a/cmd/run.go b/cmd/run.go index 4d7bb407..7aba4a41 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -30,7 +30,6 @@ import ( usage "github.com/gatewayd-io/gatewayd/usagereport/v1" "github.com/getsentry/sentry-go" "github.com/go-co-op/gocron" - "github.com/panjf2000/gnet/v2" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "github.com/rs/zerolog" @@ -633,30 +632,9 @@ var runCmd = &cobra.Command{ cfg.Network, cfg.Address, cfg.GetTickInterval(), - []gnet.Option{ - // Scheduling options - gnet.WithMulticore(cfg.MultiCore), - gnet.WithLockOSThread(cfg.LockOSThread), - // NumEventLoop overrides Multicore option. - // gnet.WithNumEventLoop(1), - + network.Option{ // Can be used to send keepalive messages to the client. - gnet.WithTicker(cfg.EnableTicker), - - // Internal event-loop load balancing options - gnet.WithLoadBalancing(cfg.GetLoadBalancer()), - - // Buffer options - gnet.WithReadBufferCap(cfg.ReadBufferCap), - gnet.WithWriteBufferCap(cfg.WriteBufferCap), - gnet.WithSocketRecvBuffer(cfg.SocketRecvBuffer), - gnet.WithSocketSendBuffer(cfg.SocketSendBuffer), - - // TCP options - gnet.WithReuseAddr(cfg.ReuseAddress), - gnet.WithReusePort(cfg.ReusePort), - gnet.WithTCPKeepAlive(cfg.TCPKeepAlive), - gnet.WithTCPNoDelay(cfg.GetTCPNoDelay()), + EnableTicker: cfg.EnableTicker, }, proxies[name], logger, @@ -669,18 +647,6 @@ var runCmd = &cobra.Command{ attribute.String("network", cfg.Network), attribute.String("address", cfg.Address), attribute.String("tickInterval", cfg.TickInterval.String()), - attribute.Bool("multiCore", cfg.MultiCore), - attribute.Bool("lockOSThread", cfg.LockOSThread), - attribute.Bool("enableTicker", cfg.EnableTicker), - attribute.String("loadBalancer", cfg.LoadBalancer), - attribute.Int("readBufferCap", cfg.ReadBufferCap), - attribute.Int("writeBufferCap", cfg.WriteBufferCap), - attribute.Int("socketRecvBuffer", cfg.SocketRecvBuffer), - attribute.Int("socketSendBuffer", cfg.SocketSendBuffer), - attribute.Bool("reuseAddress", cfg.ReuseAddress), - attribute.Bool("reusePort", cfg.ReusePort), - attribute.String("tcpKeepAlive", cfg.TCPKeepAlive.String()), - attribute.Bool("tcpNoDelay", cfg.TCPNoDelay), attribute.String("pluginTimeout", conf.Plugin.Timeout.String()), )) diff --git a/config/config.go b/config/config.go index e2d38f9b..34b97279 100644 --- a/config/config.go +++ b/config/config.go @@ -129,21 +129,10 @@ func (c *Config) LoadDefaults(ctx context.Context) { } defaultServer := Server{ - Network: DefaultListenNetwork, - Address: DefaultListenAddress, - EnableTicker: false, - TickInterval: DefaultTickInterval, - MultiCore: true, - LockOSThread: false, - ReuseAddress: true, - ReusePort: true, - LoadBalancer: DefaultLoadBalancer, - ReadBufferCap: DefaultBufferSize, - WriteBufferCap: DefaultBufferSize, - SocketRecvBuffer: DefaultBufferSize, - SocketSendBuffer: DefaultBufferSize, - TCPKeepAlive: DefaultTCPKeepAliveDuration, - TCPNoDelay: DefaultTCPNoDelay, + Network: DefaultListenNetwork, + Address: DefaultListenAddress, + EnableTicker: false, + TickInterval: DefaultTickInterval, } c.globalDefaults = GlobalConfig{ diff --git a/config/getters.go b/config/getters.go index 11a8aa3b..da5aff20 100644 --- a/config/getters.go +++ b/config/getters.go @@ -166,23 +166,6 @@ func (s Server) GetTickInterval() time.Duration { return s.TickInterval } -// GetLoadBalancer returns the load balancing algorithm to use. -func (s Server) GetLoadBalancer() gnet.LoadBalancing { - if lb, ok := loadBalancers[s.LoadBalancer]; ok { - return lb - } - return gnet.RoundRobin -} - -// GetTCPNoDelay returns the TCP no delay option from config file. -func (s Server) GetTCPNoDelay() gnet.TCPSocketOpt { - if s.TCPNoDelay { - return gnet.TCPNoDelay - } - - return gnet.TCPDelay -} - // GetSize returns the pool size from config file. func (p Pool) GetSize() int { if p.Size == 0 { diff --git a/config/types.go b/config/types.go index 5924e855..d1a499b3 100644 --- a/config/types.go +++ b/config/types.go @@ -5,16 +5,6 @@ import ( "time" ) -// // getPath returns the path to the referenced config value. -// func getPath(cfg *koanf.Koanf, path string) string { -// ref := cfg.String(path) -// if cfg.Exists(path) && cfg.StringMap(ref) != nil { -// return ref -// } - -// return path -// } - type Plugin struct { Name string `json:"name" jsonschema:"required"` Enabled bool `json:"enabled"` @@ -88,21 +78,10 @@ type Proxy struct { } type Server struct { - EnableTicker bool `json:"enableTicker"` - MultiCore bool `json:"multiCore"` - LockOSThread bool `json:"lockOSThread"` //nolint:tagliatelle - ReuseAddress bool `json:"reuseAddress"` - ReusePort bool `json:"reusePort"` - TCPNoDelay bool `json:"tcpNoDelay"` - ReadBufferCap int `json:"readBufferCap"` - WriteBufferCap int `json:"writeBufferCap"` - SocketRecvBuffer int `json:"socketRecvBuffer"` - SocketSendBuffer int `json:"socketSendBuffer"` - TCPKeepAlive time.Duration `json:"tcpKeepAlive" jsonschema:"oneof_type=string;integer"` - TickInterval time.Duration `json:"tickInterval" jsonschema:"oneof_type=string;integer"` - Network string `json:"network" jsonschema:"enum=tcp,enum=udp,enum=unix"` - Address string `json:"address"` - LoadBalancer string `json:"loadBalancer" jsonschema:"enum=roundrobin,enum=leastconnections,enum=sourceaddrhash"` + EnableTicker bool `json:"enableTicker"` + TickInterval time.Duration `json:"tickInterval" jsonschema:"oneof_type=string;integer"` + Network string `json:"network" jsonschema:"enum=tcp,enum=udp,enum=unix"` + Address string `json:"address"` } type API struct { diff --git a/gatewayd.yaml b/gatewayd.yaml index f0032f2d..d30880dc 100644 --- a/gatewayd.yaml +++ b/gatewayd.yaml @@ -56,17 +56,6 @@ servers: address: 0.0.0.0:15432 enableTicker: False tickInterval: 5s # duration - multiCore: True - lockOSThread: False - loadBalancer: roundrobin - readBufferCap: 134217728 - writeBufferCap: 134217728 - socketRecvBuffer: 134217728 - socketSendBuffer: 134217728 - reuseAddress: True - reusePort: True - tcpKeepAlive: 3s # duration - tcpNoDelay: True api: enabled: True From f89cfde2eacb8515a3df93ba365f6b641687d547 Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Sun, 8 Oct 2023 00:37:46 +0200 Subject: [PATCH 04/35] Fix tests --- config/getters_test.go | 13 ------------- network/proxy_test.go | 27 ++++++++++++++------------- network/server_test.go | 8 ++------ network/utils_test.go | 13 ++++++------- 4 files changed, 22 insertions(+), 39 deletions(-) diff --git a/config/getters_test.go b/config/getters_test.go index 2d485c39..4c3d6b71 100644 --- a/config/getters_test.go +++ b/config/getters_test.go @@ -4,7 +4,6 @@ import ( "testing" "time" - "github.com/panjf2000/gnet/v2" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" ) @@ -75,18 +74,6 @@ func TestGetTickInterval(t *testing.T) { assert.Equal(t, DefaultTickInterval, server.GetTickInterval()) } -// TestGetLoadBalancer tests the GetLoadBalancer function. -func TestGetLoadBalancer(t *testing.T) { - server := Server{} - assert.Equal(t, gnet.RoundRobin, server.GetLoadBalancer()) -} - -// TestGetTCPNoDelay tests the GetTCPNoDelay function. -func TestGetTCPNoDelay(t *testing.T) { - server := Server{} - assert.Equal(t, gnet.TCPDelay, server.GetTCPNoDelay()) -} - // TestGetSize tests the GetSize function. func TestGetSize(t *testing.T) { pool := Pool{} diff --git a/network/proxy_test.go b/network/proxy_test.go index 6b680193..14f7a0ec 100644 --- a/network/proxy_test.go +++ b/network/proxy_test.go @@ -252,12 +252,12 @@ func BenchmarkProxyConnectDisconnect(b *testing.B) { config.DefaultPluginTimeout) defer proxy.Shutdown() - gconn := testGNetConnection{} + conn := testConnection{} // Connect to the proxy for i := 0; i < b.N; i++ { - proxy.Connect(gconn.Conn) //nolint:errcheck - proxy.Disconnect(&gconn) //nolint:errcheck + proxy.Connect(conn.Conn) //nolint:errcheck + proxy.Disconnect(&conn) //nolint:errcheck } } @@ -306,13 +306,14 @@ func BenchmarkProxyPassThrough(b *testing.B) { config.DefaultPluginTimeout) defer proxy.Shutdown() - gconn := testGNetConnection{} - proxy.Connect(gconn.Conn) //nolint:errcheck - defer proxy.Disconnect(&gconn) //nolint:errcheck + conn := testConnection{} + proxy.Connect(conn.Conn) //nolint:errcheck + defer proxy.Disconnect(&conn) //nolint:errcheck // Connect to the proxy for i := 0; i < b.N; i++ { - proxy.PassThrough(&gconn) //nolint:errcheck + proxy.PassThroughToClient(&conn) //nolint:errcheck + proxy.PassThroughToServer(&conn) //nolint:errcheck } } @@ -362,9 +363,9 @@ func BenchmarkProxyIsHealthyAndIsExhausted(b *testing.B) { config.DefaultPluginTimeout) defer proxy.Shutdown() - gconn := testGNetConnection{} - proxy.Connect(gconn.Conn) //nolint:errcheck - defer proxy.Disconnect(&gconn) //nolint:errcheck + conn := testConnection{} + proxy.Connect(conn.Conn) //nolint:errcheck + defer proxy.Disconnect(&conn) //nolint:errcheck // Connect to the proxy for i := 0; i < b.N; i++ { @@ -419,9 +420,9 @@ func BenchmarkProxyAvailableAndBusyConnections(b *testing.B) { config.DefaultPluginTimeout) defer proxy.Shutdown() - gconn := testGNetConnection{} - proxy.Connect(gconn.Conn) //nolint:errcheck - defer proxy.Disconnect(&gconn) //nolint:errcheck + conn := testConnection{} + proxy.Connect(conn.Conn) //nolint:errcheck + defer proxy.Disconnect(&conn) //nolint:errcheck // Connect to the proxy for i := 0; i < b.N; i++ { diff --git a/network/server_test.go b/network/server_test.go index 2a3028c6..0a2ee253 100644 --- a/network/server_test.go +++ b/network/server_test.go @@ -14,7 +14,6 @@ import ( "github.com/gatewayd-io/gatewayd/logging" "github.com/gatewayd-io/gatewayd/plugin" "github.com/gatewayd-io/gatewayd/pool" - "github.com/panjf2000/gnet/v2" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" "google.golang.org/grpc" @@ -173,11 +172,8 @@ func TestRunServer(t *testing.T) { "tcp", "127.0.0.1:15432", config.DefaultTickInterval, - []gnet.Option{ - gnet.WithMulticore(false), - gnet.WithReuseAddr(true), - gnet.WithReusePort(true), - gnet.WithTicker(true), // Enable ticker. + Option{ + EnableTicker: false, }, proxy, logger, diff --git a/network/utils_test.go b/network/utils_test.go index 2ab4f0d4..8d6e1d77 100644 --- a/network/utils_test.go +++ b/network/utils_test.go @@ -10,7 +10,6 @@ import ( "github.com/gatewayd-io/gatewayd/config" "github.com/gatewayd-io/gatewayd/logging" - "github.com/panjf2000/gnet/v2" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" ) @@ -112,18 +111,18 @@ func BenchmarkResolveUnix(b *testing.B) { } } -type testGNetConnection struct { - gnet.Conn +type testConnection struct { + net.Conn } -func (c *testGNetConnection) LocalAddr() net.Addr { +func (c *testConnection) LocalAddr() net.Addr { return &net.TCPAddr{ IP: net.ParseIP("127.0.0.1"), Port: 0, } } -func (c *testGNetConnection) RemoteAddr() net.Addr { +func (c *testConnection) RemoteAddr() net.Addr { return &net.TCPAddr{ IP: net.ParseIP("127.0.0.1"), Port: 0, @@ -139,7 +138,7 @@ func BenchmarkTrafficData(b *testing.B) { NoColor: true, }) - gconn := &testGNetConnection{} + conn := &testConnection{} client := NewClient(context.Background(), &config.Client{ Network: "tcp", Address: "localhost:5432", @@ -159,7 +158,7 @@ func BenchmarkTrafficData(b *testing.B) { } err := "test error" for i := 0; i < b.N; i++ { - trafficData(gconn, client, fields, err) + trafficData(conn, client, fields, err) } } From 5cb26b275032481aa7884c39fded3f439c93a437 Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Sun, 8 Oct 2023 19:04:30 +0200 Subject: [PATCH 05/35] Fix linter errors Remove unused dependencies (gnet/v2 and others) --- config/constants.go | 1 + config/getters.go | 6 ----- errors/errors.go | 13 ++++++++++ go.mod | 5 ---- go.sum | 12 --------- network/client.go | 4 +-- network/engine.go | 60 ++++++++++++++++++++++----------------------- network/proxy.go | 9 +++---- network/server.go | 2 +- 9 files changed, 50 insertions(+), 62 deletions(-) diff --git a/config/constants.go b/config/constants.go index a40175e8..9cbafb4e 100644 --- a/config/constants.go +++ b/config/constants.go @@ -117,6 +117,7 @@ const ( DefaultTCPKeepAliveDuration = 3 * time.Second DefaultLoadBalancer = "roundrobin" DefaultTCPNoDelay = true + DefaultEngineStopTimeout = 5 * time.Second // Utility constants. DefaultSeed = 1000 diff --git a/config/getters.go b/config/getters.go index da5aff20..0d09ee35 100644 --- a/config/getters.go +++ b/config/getters.go @@ -5,7 +5,6 @@ import ( "path/filepath" "time" - "github.com/panjf2000/gnet/v2" "github.com/rs/zerolog" ) @@ -28,11 +27,6 @@ var ( "continue": Continue, "stop": Stop, } - loadBalancers = map[string]gnet.LoadBalancing{ - "roundrobin": gnet.RoundRobin, - "leastconnections": gnet.LeastConnections, - "sourceaddrhash": gnet.SourceAddrHash, - } logOutputs = map[string]LogOutput{ "console": Console, "stdout": Stdout, diff --git a/errors/errors.go b/errors/errors.go index d884d184..f16a6501 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -21,6 +21,10 @@ const ( ErrCodeClientSendFailed ErrCodeServerReceiveFailed ErrCodeServerSendFailed + ErrCodeServerListenFailed + ErrCodeSplitHostPortFailed + ErrCodeAcceptFailed + ErrCodeReadFailed ErrCodePutFailed ErrCodeCastFailed ErrCodeHookVerificationFailed @@ -77,6 +81,15 @@ var ( ErrCodeServerSendFailed, "couldn't send data to the client", nil) ErrServerReceiveFailed = NewGatewayDError( ErrCodeServerReceiveFailed, "couldn't receive data from the client", nil) + ErrServerListenFailed = NewGatewayDError( + ErrCodeServerListenFailed, "couldn't listen on the server", nil) + ErrSplitHostPortFailed = NewGatewayDError( + ErrCodeSplitHostPortFailed, "failed to split host:port", nil) + ErrAcceptFailed = NewGatewayDError( + ErrCodeAcceptFailed, "failed to accept connection", nil) + + ErrReadFailed = NewGatewayDError( + ErrCodeReadFailed, "failed to read from the client", nil) ErrPutFailed = NewGatewayDError( ErrCodePutFailed, "failed to put in pool", nil) diff --git a/go.mod b/go.mod index 1d614dc6..deba5272 100644 --- a/go.mod +++ b/go.mod @@ -18,7 +18,6 @@ require ( github.com/invopop/jsonschema v0.8.0 github.com/knadh/koanf v1.5.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/panjf2000/gnet/v2 v2.3.2 github.com/prometheus/client_golang v1.16.0 github.com/prometheus/client_model v0.4.0 github.com/prometheus/common v0.44.0 @@ -70,16 +69,12 @@ require ( github.com/prometheus/procfs v0.11.1 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/spf13/pflag v1.0.5 // indirect - github.com/valyala/bytebufferpool v1.0.0 // indirect go.opentelemetry.io/otel/metric v1.18.0 // indirect go.opentelemetry.io/proto/otlp v1.0.0 // indirect go.uber.org/atomic v1.11.0 // indirect - go.uber.org/multierr v1.11.0 // indirect - go.uber.org/zap v1.26.0 // indirect golang.org/x/crypto v0.13.0 // indirect golang.org/x/net v0.15.0 // indirect golang.org/x/oauth2 v0.12.0 // indirect - golang.org/x/sync v0.3.0 // indirect golang.org/x/sys v0.12.0 // indirect golang.org/x/text v0.13.0 // indirect google.golang.org/appengine v1.6.8 // indirect diff --git a/go.sum b/go.sum index 83411919..23d47e47 100644 --- a/go.sum +++ b/go.sum @@ -274,10 +274,6 @@ github.com/npillmayer/nestext v0.1.3/go.mod h1:h2lrijH8jpicr25dFY+oAJLyzlya6jhnu github.com/oklog/run v1.0.0/go.mod h1:dlhp/R75TPv97u0XWUtDeV/lRKWPKSdTuV0TZvrmrQA= github.com/oklog/run v1.1.0 h1:GEenZ1cK0+q0+wsJew9qUg/DyD8k3JzYsZAi5gYi2mA= github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DVU= -github.com/panjf2000/ants/v2 v2.8.1 h1:C+n/f++aiW8kHCExKlpX6X+okmxKXP7DWLutxuAPuwQ= -github.com/panjf2000/ants/v2 v2.8.1/go.mod h1:KIBmYG9QQX5U2qzFP/yQJaq/nSb6rahS9iEHkrCMgM8= -github.com/panjf2000/gnet/v2 v2.3.2 h1:cwzq4S2fZbHvBaGriMlZTDtiFL/EzaaVny2V03wOuj0= -github.com/panjf2000/gnet/v2 v2.3.2/go.mod h1:jQ0+i/ZSs4wxUKl06sgjWE0bL/yWI1d5LkNdqulZXFc= github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= @@ -359,8 +355,6 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= -github.com/valyala/bytebufferpool v1.0.0 h1:GqA5TC/0021Y/b9FG4Oi9Mr3q7XYx6KllzawFIhcdPw= -github.com/valyala/bytebufferpool v1.0.0/go.mod h1:6bBcMArwyJ5K/AmCkWv1jt77kVWyCJ6HpOuEn7z0Csc= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= @@ -391,11 +385,7 @@ go.uber.org/atomic v1.11.0/go.mod h1:LUxbIzbOniOlMKjJjyPfpl4v+PKK2cNJn91OQbhoJI0 go.uber.org/goleak v1.2.1 h1:NBol2c7O1ZokfZ0LEU9K6Whx/KnwvepVetCUhtKja4A= go.uber.org/goleak v1.2.1/go.mod h1:qlT2yGI9QafXHhZZLxlSuNsMw3FFLxBr+tBRlmO1xH4= go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU= -go.uber.org/multierr v1.11.0 h1:blXXJkSxSSfBVBlC76pxqeO+LN3aDfLQo+309xJstO0= -go.uber.org/multierr v1.11.0/go.mod h1:20+QtiLqy0Nd6FdQB9TLXag12DsQkrbs3htMFfDN80Y= go.uber.org/zap v1.17.0/go.mod h1:MXVU+bhUf/A7Xi2HNOnopQOrmycQ5Ih87HtOu4q5SSo= -go.uber.org/zap v1.26.0 h1:sI7k6L95XOKS281NhVKOFCUNIvv9e0w4BF8N3u+tCRo= -go.uber.org/zap v1.26.0/go.mod h1:dtElttAiwGvoJ/vj4IwHBS/gXsEu/pZ50mUIRWuG0so= golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4= golang.org/x/crypto v0.0.0-20190308221718-c2843e01d9a2/go.mod h1:djNgcEr1/C05ACkg1iLfiJU5Ep61QUkGW8qpdssI0+w= golang.org/x/crypto v0.0.0-20190923035154-9ee001bba392/go.mod h1:/lpIB1dKB+9EgE3H3cr1v9wB50oz8l4C4h62xy7jSTY= @@ -458,8 +448,6 @@ golang.org/x/sync v0.0.0-20201207232520-09787c993a3a/go.mod h1:RxMgew5VJxzue5/jJ golang.org/x/sync v0.0.0-20210220032951-036812b2e83c/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20220722155255-886fb9371eb4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.1.0/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= -golang.org/x/sync v0.3.0 h1:ftCYgMx6zT/asHUrPw8BLLscYtGznsLAnjq5RH9P66E= -golang.org/x/sync v0.3.0/go.mod h1:FU7BRWz2tNW+3quACPkgCx/L+uEAv1htQ0V83Z9Rj+Y= golang.org/x/sys v0.0.0-20180823144017-11551d06cbcc/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180830151530-49385e6e1522/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20180905080454-ebe1bf3edb33/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/network/client.go b/network/client.go index 3074b778..de2ef2a8 100644 --- a/network/client.go +++ b/network/client.go @@ -153,14 +153,14 @@ func (c *Client) Send(data []byte) (int, *gerr.GatewayDError) { break } - n, err := c.Conn.Write(data) + written, err := c.Conn.Write(data) if err != nil { c.logger.Error().Err(err).Msg("Couldn't send data to the server") span.RecordError(err) return 0, gerr.ErrClientSendFailed.Wrap(err) } - sent += n + sent += written } c.logger.Debug().Fields( diff --git a/network/engine.go b/network/engine.go index 89144ed2..f38ababb 100644 --- a/network/engine.go +++ b/network/engine.go @@ -5,6 +5,9 @@ import ( "net" "strconv" "time" + + "github.com/gatewayd-io/gatewayd/config" + gerr "github.com/gatewayd-io/gatewayd/errors" ) type Option struct { @@ -39,7 +42,7 @@ func (engine *Engine) CountConnections() int { } func (engine *Engine) Stop(ctx context.Context) error { - ctx, cancel := context.WithDeadline(ctx, time.Now().Add(2*time.Second)) + _, cancel := context.WithDeadline(ctx, time.Now().Add(config.DefaultEngineStopTimeout)) defer cancel() engine.stopServer <- struct{}{} @@ -47,7 +50,7 @@ func (engine *Engine) Stop(ctx context.Context) error { } // Run starts a server and connects all the handlers. -func Run(network, address string, server *Server, opts Option) error { +func Run(network, address string, server *Server) *gerr.GatewayDError { server.engine = Engine{ connections: 0, stopServer: make(chan struct{}), @@ -57,11 +60,11 @@ func Run(network, address string, server *Server, opts Option) error { return nil } - if ln, err := net.Listen(network, address); err != nil { + var err error + server.engine.listener, err = net.Listen(network, address) + if err != nil { server.logger.Error().Err(err).Msg("Server failed to start listening") - return err - } else { - server.engine.listener = ln + return gerr.ErrServerListenFailed.Wrap(err) } defer server.engine.listener.Close() @@ -70,25 +73,22 @@ func Run(network, address string, server *Server, opts Option) error { return nil } - if host, port, err := net.SplitHostPort(server.engine.listener.Addr().String()); err != nil { + var port string + server.engine.host, port, err = net.SplitHostPort(server.engine.listener.Addr().String()) + if err != nil { server.logger.Error().Err(err).Msg("Failed to split host and port") - return err - } else { - server.engine.host = host - if server.engine.port, err = strconv.Atoi(port); err != nil { - server.logger.Error().Err(err).Msg("Failed to convert port to integer") - return err - } + return gerr.ErrSplitHostPortFailed.Wrap(err) + } + + if server.engine.port, err = strconv.Atoi(port); err != nil { + server.logger.Error().Err(err).Msg("Failed to convert port to integer") + return gerr.ErrCastFailed.Wrap(err) } go func(server *Server) { - for { - select { - case <-server.engine.stopServer: - server.OnShutdown(server.engine) - server.logger.Debug().Msg("Server stopped") - } - } + <-server.engine.stopServer + server.OnShutdown(server.engine) + server.logger.Debug().Msg("Server stopped") }(server) go func(server *Server) { @@ -113,11 +113,13 @@ func Run(network, address string, server *Server, opts Option) error { conn, err := server.engine.listener.Accept() if err != nil { server.logger.Error().Err(err).Msg("Failed to accept connection") - return err + return gerr.ErrAcceptFailed.Wrap(err) } if out, action := server.OnOpen(conn); action != None { - conn.Write(out) + if _, err := conn.Write(out); err != nil { + server.logger.Error().Err(err).Msg("Failed to write to connection") + } conn.Close() if action == Shutdown { server.OnShutdown(server.engine) @@ -136,14 +138,10 @@ func Run(network, address string, server *Server, opts Option) error { }(server, conn, stopConnection) go func(server *Server, conn net.Conn, stopConnection chan struct{}) { - for { - select { - case <-stopConnection: - server.engine.connections-- - if action := server.OnClose(conn, err); action == Close { - return - } - } + <-stopConnection + server.engine.connections-- + if action := server.OnClose(conn, err); action == Close { + return } }(server, conn, stopConnection) } diff --git a/network/proxy.go b/network/proxy.go index 943113df..04db6dd9 100644 --- a/network/proxy.go +++ b/network/proxy.go @@ -554,7 +554,7 @@ func (pr *Proxy) BusyConnections() []string { } // receiveTrafficFromClient is a function that waits to receive data from the client. -func (pr *Proxy) receiveTrafficFromClient(conn net.Conn) ([]byte, error) { +func (pr *Proxy) receiveTrafficFromClient(conn net.Conn) ([]byte, *gerr.GatewayDError) { _, span := otel.Tracer(config.TracerName).Start(pr.ctx, "receiveTrafficFromClient") defer span.End() @@ -571,7 +571,7 @@ func (pr *Proxy) receiveTrafficFromClient(conn net.Conn) ([]byte, error) { metrics.BytesReceivedFromClient.Observe(float64(read)) metrics.TotalTrafficBytes.Observe(float64(read)) - return chunk[:read], err + return chunk[:read], gerr.ErrReadFailed.Wrap(err) } received += read @@ -597,7 +597,6 @@ func (pr *Proxy) receiveTrafficFromClient(conn net.Conn) ([]byte, error) { metrics.BytesReceivedFromClient.Observe(float64(length)) metrics.TotalTrafficBytes.Observe(float64(length)) - //nolint:wrapcheck return buffer.Bytes(), nil } @@ -668,14 +667,14 @@ func (pr *Proxy) sendTrafficToClient( break } - n, origErr := conn.Write(response[:received]) + written, origErr := conn.Write(response[:received]) if origErr != nil { pr.logger.Error().Err(origErr).Msg("Error writing to client") span.RecordError(origErr) return gerr.ErrServerSendFailed.Wrap(origErr) } - sent += n + sent += written } pr.logger.Debug().Fields( diff --git a/network/server.go b/network/server.go index 4ba918e9..ffa365e8 100644 --- a/network/server.go +++ b/network/server.go @@ -371,7 +371,7 @@ func (s *Server) Run() error { } // Start the server. - origErr := Run(s.Network, addr, s, s.Options) + origErr := Run(s.Network, addr, s) if origErr != nil { s.logger.Error().Err(origErr).Msg("Failed to start server") span.RecordError(origErr) From 57bc655468ffc78415a049037f72fce8cafbbd9c Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Sun, 8 Oct 2023 19:41:54 +0200 Subject: [PATCH 06/35] Fix nil error and wrap the error --- network/proxy.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/network/proxy.go b/network/proxy.go index 04db6dd9..66bf065d 100644 --- a/network/proxy.go +++ b/network/proxy.go @@ -307,10 +307,10 @@ func (pr *Proxy) PassThroughToServer(conn net.Conn) *gerr.GatewayDError { } span.AddEvent("Ran the OnTrafficFromClient hooks") - if errors.Is(origErr, io.EOF) { + if origErr != nil && errors.Is(origErr, io.EOF) { // Client closed the connection. span.AddEvent("Client closed the connection") - return gerr.ErrClientNotConnected + return gerr.ErrClientNotConnected.Wrap(origErr) } // If the hook wants to terminate the connection, do it. From a219b403b0cf6f01ccccf4d3a3b7fa44917066c4 Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Mon, 9 Oct 2023 23:37:42 +0200 Subject: [PATCH 07/35] Prevent race using a RWMutex --- network/engine.go | 9 +++++++++ network/server.go | 6 ++++++ 2 files changed, 15 insertions(+) diff --git a/network/engine.go b/network/engine.go index f38ababb..c9800e1a 100644 --- a/network/engine.go +++ b/network/engine.go @@ -4,6 +4,7 @@ import ( "context" "net" "strconv" + "sync" "time" "github.com/gatewayd-io/gatewayd/config" @@ -35,9 +36,12 @@ type Engine struct { port int connections uint32 stopServer chan struct{} + mu *sync.RWMutex } func (engine *Engine) CountConnections() int { + engine.mu.RLock() + defer engine.mu.RUnlock() return int(engine.connections) } @@ -54,6 +58,7 @@ func Run(network, address string, server *Server) *gerr.GatewayDError { server.engine = Engine{ connections: 0, stopServer: make(chan struct{}), + mu: &sync.RWMutex{}, } if action := server.OnBoot(server.engine); action != None { @@ -126,7 +131,9 @@ func Run(network, address string, server *Server) *gerr.GatewayDError { return nil } } + server.engine.mu.Lock() server.engine.connections++ + server.engine.mu.Unlock() // For every new connection, a new unbuffered channel is created to help // stop the proxy, recycle the server connection and close stale connections. @@ -139,7 +146,9 @@ func Run(network, address string, server *Server) *gerr.GatewayDError { go func(server *Server, conn net.Conn, stopConnection chan struct{}) { <-stopConnection + server.engine.mu.Lock() server.engine.connections-- + server.engine.mu.Unlock() if action := server.OnClose(conn, err); action == Close { return } diff --git a/network/server.go b/network/server.go index ffa365e8..3280fc30 100644 --- a/network/server.go +++ b/network/server.go @@ -58,7 +58,9 @@ func (s *Server) OnBoot(engine Engine) Action { s.engine = engine // Set the server status to running. + s.engine.mu.Lock() s.Status = config.Running + s.engine.mu.Unlock() // Run the OnBooted hooks. _, err = s.pluginRegistry.Run( @@ -296,7 +298,9 @@ func (s *Server) OnShutdown(Engine) { s.proxy.Shutdown() // Set the server status to stopped. This is used to shutdown the server gracefully in OnClose. + s.engine.mu.Lock() s.Status = config.Stopped + s.engine.mu.Unlock() } // OnTick is called every TickInterval. It calls the OnTick hooks. @@ -390,7 +394,9 @@ func (s *Server) Shutdown() { s.proxy.Shutdown() // Set the server status to stopped. This is used to shutdown the server gracefully in OnClose. + s.engine.mu.Lock() s.Status = config.Stopped + s.engine.mu.Unlock() // Shutdown the server. if err := s.engine.Stop(context.Background()); err != nil { From 74856a06777d52c84600dfbc597e62dcab609af6 Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Mon, 9 Oct 2023 23:37:54 +0200 Subject: [PATCH 08/35] Fix server test --- network/server_test.go | 109 +++++++++++++++++++++++++---------------- 1 file changed, 67 insertions(+), 42 deletions(-) diff --git a/network/server_test.go b/network/server_test.go index 0a2ee253..3e646ac6 100644 --- a/network/server_test.go +++ b/network/server_test.go @@ -6,6 +6,7 @@ import ( "errors" "io" "os" + "sync" "testing" "time" @@ -22,6 +23,7 @@ import ( // TestRunServer tests an entire server run with a single client connection and hooks. func TestRunServer(t *testing.T) { errs := make(chan error) + defer close(errs) logger := logging.NewLogger(context.Background(), logging.LoggerConfig{ Output: []config.LogOutput{ @@ -30,7 +32,7 @@ func TestRunServer(t *testing.T) { }, TimeFormat: zerolog.TimeFormatUnix, ConsoleTimeFormat: time.RFC3339, - Level: zerolog.WarnLevel, + Level: zerolog.DebugLevel, NoColor: true, FileName: "server_test.log", }) @@ -182,43 +184,70 @@ func TestRunServer(t *testing.T) { ) assert.NotNil(t, server) - go func(server *Server, errs chan error) { - if err := server.Run(); err != nil { - errs <- err - } - close(errs) - - // Read the log file and check if the log file contains the expected log messages. - if _, err := os.Stat("server_test.log"); err == nil { - logFile, err := os.Open("server_test.log") - assert.NoError(t, err) - defer logFile.Close() - - reader := bufio.NewReader(logFile) - assert.NotNil(t, reader) + stop := make(chan struct{}) + defer close(stop) - buffer, err := io.ReadAll(reader) - assert.NoError(t, err) - assert.Greater(t, len(buffer), 0) // The log file should not be empty. + var waitGroup sync.WaitGroup - logLines := string(buffer) - assert.Contains(t, logLines, "GatewayD is running", "GatewayD should be running") - assert.Contains(t, logLines, "GatewayD is ticking...", "GatewayD should be ticking") - assert.Contains(t, logLines, "Ingress traffic", "Ingress traffic should be logged") - assert.Contains(t, logLines, "Egress traffic", "Egress traffic should be logged") - assert.Contains(t, logLines, "GatewayD is shutting down...", "GatewayD should be shutting down") + waitGroup.Add(1) + go func(t *testing.T, server *Server, stop chan struct{}, waitGroup *sync.WaitGroup) { + t.Helper() + for { + select { + case <-stop: + // Read the log file and check if the log file contains the expected log messages. + if _, err := os.Stat("server_test.log"); err == nil { + logFile, err := os.Open("server_test.log") + assert.NoError(t, err) + defer logFile.Close() + + reader := bufio.NewReader(logFile) + assert.NotNil(t, reader) + + buffer, err := io.ReadAll(reader) + assert.NoError(t, err) + assert.Greater(t, len(buffer), 0) // The log file should not be empty. + + logLines := string(buffer) + assert.Contains(t, logLines, "GatewayD is running", "GatewayD should be running") + assert.Contains(t, logLines, "GatewayD is ticking...", "GatewayD should be ticking") + assert.Contains(t, logLines, "Ingress traffic", "Ingress traffic should be logged") + assert.Contains(t, logLines, "Egress traffic", "Egress traffic should be logged") + assert.Contains(t, logLines, "GatewayD is shutting down...", "GatewayD should be shutting down") + + assert.NoError(t, os.Remove("server_test.log")) + server.Shutdown() + } + return + case err := <-errs: + server.Shutdown() + t.Log(err) + t.Fail() + waitGroup.Done() + return + default: //nolint:staticcheck + } + } + }(t, server, stop, &waitGroup) - assert.NoError(t, os.Remove("server_test.log")) + waitGroup.Add(1) + go func(t *testing.T, server *Server, errs chan error, waitGroup *sync.WaitGroup) { + t.Helper() + if err := server.Run(); err != nil { + errs <- err + t.Fail() } - }(server, errs) + waitGroup.Done() + }(t, server, errs, &waitGroup) + + waitGroup.Add(1) + go func(t *testing.T, server *Server, proxy *Proxy, stop chan struct{}, waitGroup *sync.WaitGroup) { + t.Helper() + // Pause for a while to allow the server to start. + time.Sleep(500 * time.Millisecond) - //nolint:thelper - go func(t *testing.T, server *Server, proxy *Proxy) { for { if server.IsRunning() { - // Pause for a while to allow the server to start. - time.Sleep(500 * time.Millisecond) - client := NewClient( context.Background(), &config.Client{ @@ -260,19 +289,15 @@ func TestRunServer(t *testing.T) { // Test Prometheus metrics. CollectAndComparePrometheusMetrics(t) - // Clean up. client.Close() - // Pause for a while to allow the server to disconnect and shutdown. - time.Sleep(500 * time.Millisecond) - server.Shutdown() - break + time.Sleep(100 * time.Millisecond) + stop <- struct{}{} + waitGroup.Done() + return } + time.Sleep(100 * time.Millisecond) } - }(t, server, proxy) + }(t, server, proxy, stop, &waitGroup) - for err := range errs { - if err != nil { - t.Fatal(err) - } - } + waitGroup.Wait() } From 9defae59847ffe21632aa2343d28bdf2ccd4fa6e Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Fri, 13 Oct 2023 16:59:47 +0200 Subject: [PATCH 09/35] Update dependencies --- go.mod | 57 ++++++++++++++-------------- go.sum | 116 ++++++++++++++++++++++++++++++--------------------------- 2 files changed, 91 insertions(+), 82 deletions(-) diff --git a/go.mod b/go.mod index deba5272..c6be2933 100644 --- a/go.mod +++ b/go.mod @@ -8,40 +8,42 @@ require ( github.com/codingsince1985/checksum v1.3.0 github.com/envoyproxy/protoc-gen-validate v1.0.2 github.com/gatewayd-io/gatewayd-plugin-sdk v0.1.1 - github.com/getsentry/sentry-go v0.24.1 - github.com/go-co-op/gocron v1.34.0 - github.com/google/go-cmp v0.5.9 + github.com/getsentry/sentry-go v0.25.0 + github.com/go-co-op/gocron v1.35.2 + github.com/google/go-cmp v0.6.0 github.com/google/go-github/v53 v53.2.0 github.com/grpc-ecosystem/grpc-gateway/v2 v2.18.0 github.com/hashicorp/go-hclog v1.5.0 - github.com/hashicorp/go-plugin v1.5.1 - github.com/invopop/jsonschema v0.8.0 + github.com/hashicorp/go-plugin v1.5.2 + github.com/invopop/jsonschema v0.12.0 github.com/knadh/koanf v1.5.0 github.com/mitchellh/mapstructure v1.5.0 - github.com/prometheus/client_golang v1.16.0 - github.com/prometheus/client_model v0.4.0 + github.com/prometheus/client_golang v1.17.0 + github.com/prometheus/client_model v0.5.0 github.com/prometheus/common v0.44.0 - github.com/rs/zerolog v1.30.0 + github.com/rs/zerolog v1.31.0 github.com/santhosh-tekuri/jsonschema/v5 v5.3.1 github.com/spf13/cobra v1.7.0 github.com/stretchr/testify v1.8.4 github.com/zenizh/go-capturer v0.0.0-20211219060012-52ea6c8fed04 - go.opentelemetry.io/otel v1.18.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.18.0 - go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.18.0 - go.opentelemetry.io/otel/sdk v1.18.0 - go.opentelemetry.io/otel/trace v1.18.0 - golang.org/x/exp v0.0.0-20230905200255-921286631fa9 - google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb - google.golang.org/grpc v1.58.1 + go.opentelemetry.io/otel v1.19.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 + go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 + go.opentelemetry.io/otel/sdk v1.19.0 + go.opentelemetry.io/otel/trace v1.19.0 + golang.org/x/exp v0.0.0-20231006140011-7918f672742d + google.golang.org/genproto/googleapis/api v0.0.0-20231012201019-e917dd12ba7a + google.golang.org/grpc v1.58.3 google.golang.org/protobuf v1.31.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 gopkg.in/yaml.v3 v3.0.1 ) require ( - github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 // indirect + github.com/ProtonMail/go-crypto v0.0.0-20230923063757-afb1ddc0824c // indirect + github.com/bahlo/generic-list-go v0.2.0 // indirect github.com/beorn7/perks v1.0.1 // indirect + github.com/buger/jsonparser v1.1.1 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect github.com/cloudflare/circl v1.3.3 // indirect @@ -55,8 +57,8 @@ require ( github.com/google/go-querystring v1.1.0 // indirect github.com/google/uuid v1.3.1 // indirect github.com/hashicorp/yamux v0.1.1 // indirect - github.com/iancoleman/orderedmap v0.3.0 // indirect github.com/inconshreveable/mousetrap v1.1.0 // indirect + github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect github.com/mattn/go-isatty v0.0.19 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect @@ -64,20 +66,21 @@ require ( github.com/mitchellh/go-testing-interface v1.14.1 // indirect github.com/mitchellh/reflectwalk v1.0.2 // indirect github.com/oklog/run v1.1.0 // indirect - github.com/pelletier/go-toml v1.9.4 // indirect + github.com/pelletier/go-toml v1.9.5 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/procfs v0.11.1 // indirect + github.com/prometheus/procfs v0.12.0 // indirect github.com/robfig/cron/v3 v3.0.1 // indirect github.com/spf13/pflag v1.0.5 // indirect - go.opentelemetry.io/otel/metric v1.18.0 // indirect + github.com/wk8/go-ordered-map/v2 v2.1.8 // indirect + go.opentelemetry.io/otel/metric v1.19.0 // indirect go.opentelemetry.io/proto/otlp v1.0.0 // indirect go.uber.org/atomic v1.11.0 // indirect - golang.org/x/crypto v0.13.0 // indirect - golang.org/x/net v0.15.0 // indirect - golang.org/x/oauth2 v0.12.0 // indirect - golang.org/x/sys v0.12.0 // indirect + golang.org/x/crypto v0.14.0 // indirect + golang.org/x/net v0.17.0 // indirect + golang.org/x/oauth2 v0.13.0 // indirect + golang.org/x/sys v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20230913181813-007df8e322eb // indirect + google.golang.org/genproto v0.0.0-20231012201019-e917dd12ba7a // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231012201019-e917dd12ba7a // indirect ) diff --git a/go.sum b/go.sum index 23d47e47..34a522b1 100644 --- a/go.sum +++ b/go.sum @@ -5,8 +5,8 @@ github.com/Masterminds/semver/v3 v3.2.1 h1:RN9w6+7QoMeJVGyfmbcgs28Br8cvmnucEXnY0 github.com/Masterminds/semver/v3 v3.2.1/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ= github.com/NYTimes/gziphandler v1.1.1 h1:ZUDjpQae29j0ryrS0u/B8HZfJBtBQHjqw2rQ2cqUQ3I= github.com/NYTimes/gziphandler v1.1.1/go.mod h1:n/CVRwUEOgIxrgPvAQhUUr9oeUtvrhMomdKFjzJNB0c= -github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371 h1:kkhsdkhsCvIsutKu5zLMgWtgh9YxGCNAw8Ad8hjwfYg= -github.com/ProtonMail/go-crypto v0.0.0-20230828082145-3c4c8a2d2371/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= +github.com/ProtonMail/go-crypto v0.0.0-20230923063757-afb1ddc0824c h1:kMFnB0vCcX7IL/m9Y5LO+KQYv+t1CQOiFe6+SV2J7bE= +github.com/ProtonMail/go-crypto v0.0.0-20230923063757-afb1ddc0824c/go.mod h1:EjAoLdwvbIOoOQr3ihjnSoLZRtE8azugULFRteWMNc0= github.com/alecthomas/template v0.0.0-20160405071501-a0175ee3bccc/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuyumcjzFXgccqObfd/Ljyb9UuFJ6TxHnclSeseNhc= github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= @@ -27,6 +27,8 @@ github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.3.2/go.mod h1:72H github.com/aws/aws-sdk-go-v2/service/sso v1.4.2/go.mod h1:NBvT9R1MEF+Ud6ApJKM0G+IkPchKS7p7c2YPKwHmBOk= github.com/aws/aws-sdk-go-v2/service/sts v1.7.2/go.mod h1:8EzeIqfWt2wWT4rJVu3f21TfrhJ8AEMzVybRNSb/b4g= github.com/aws/smithy-go v1.8.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= +github.com/bahlo/generic-list-go v0.2.0 h1:5sz/EEAK+ls5wF+NeqDpk5+iNdMDXrh3z3nPnH1Wvgk= +github.com/bahlo/generic-list-go v0.2.0/go.mod h1:2KvAjgMlE5NNynlg/5iLrrCCZ2+5xWbdbCW3pNTGyYg= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= @@ -34,6 +36,8 @@ github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6r github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/bufbuild/protocompile v0.4.0 h1:LbFKd2XowZvQ/kajzguUp2DC9UEIQhIq77fZZlaQsNA= github.com/bufbuild/protocompile v0.4.0/go.mod h1:3v93+mbWn/v3xzN+31nwkJfrEpAUwp+BagBSZWx+TP8= +github.com/buger/jsonparser v1.1.1 h1:2PnMjfWD7wBILjqQbt530v576A/cAbQvEW9gGIpYMUs= +github.com/buger/jsonparser v1.1.1/go.mod h1:6RYKKt7H4d4+iWqouImQ9R2FZql3VbhNgx27UK13J/0= github.com/bwesterb/go-ristretto v1.2.3/go.mod h1:fUIoIZaG73pV5biE2Blr2xEzDoMj7NFEuV9ekS419A0= github.com/cenkalti/backoff/v4 v4.2.1 h1:y4OZtCnogmCPw98Zjyt5a6+QwPLGkiQsYW5oUqylYbM= github.com/cenkalti/backoff/v4 v4.2.1/go.mod h1:Y3VNntkOUPxTVeUxJ/G5vcM//AlwfmyYozVcomhLiZE= @@ -76,11 +80,11 @@ github.com/fsnotify/fsnotify v1.6.0 h1:n+5WquG0fcWoWp6xPWfHdbskMCQaFnG6PfBrh1Ky4 github.com/fsnotify/fsnotify v1.6.0/go.mod h1:sl3t1tCWJFWoRz9R8WJCbQihKKwmorjAbSClcnxKAGw= github.com/gatewayd-io/gatewayd-plugin-sdk v0.1.1 h1:ujlh6TSDFmG2e9VtY6hZk8BW0p7i0AEQL3BpMMLzxwc= github.com/gatewayd-io/gatewayd-plugin-sdk v0.1.1/go.mod h1:B4oWVHf7NeSCs7szN8nrlIO6tkznV1F3ZMqE9VxDtKY= -github.com/getsentry/sentry-go v0.24.1 h1:W6/0GyTy8J6ge6lVCc94WB6Gx2ZuLrgopnn9w8Hiwuk= -github.com/getsentry/sentry-go v0.24.1/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= +github.com/getsentry/sentry-go v0.25.0 h1:q6Eo+hS+yoJlTO3uu/azhQadsD8V+jQn2D8VvX1eOyI= +github.com/getsentry/sentry-go v0.25.0/go.mod h1:lc76E2QywIyW8WuBnwl8Lc4bkmQH4+w1gwTf25trprY= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= -github.com/go-co-op/gocron v1.34.0 h1:/rcOZjJWUYnGR0ZDKozPXEnJ+wJt220FSLo2/hxZvV0= -github.com/go-co-op/gocron v1.34.0/go.mod h1:NLi+bkm4rRSy1F8U7iacZOz0xPseMoIOnvabGoSe/no= +github.com/go-co-op/gocron v1.35.2 h1:lG3rdA9TqBBC/PtT2ukQqgLm6jEepnAzz3+OQetvPTE= +github.com/go-co-op/gocron v1.35.2/go.mod h1:NLi+bkm4rRSy1F8U7iacZOz0xPseMoIOnvabGoSe/no= github.com/go-errors/errors v1.4.2 h1:J6MZopCL4uSllY1OfXM374weqZFFItUbrImctkmUxIA= github.com/go-errors/errors v1.4.2/go.mod h1:sIVyrIiJhuEF+Pj9Ebtd6P/rEYROXFi3BopGUQ5a5Og= github.com/go-kit/kit v0.8.0/go.mod h1:xBxKIO96dXMWWy0MnWVtmwkA9/13aqxPnvrjFYMA2as= @@ -132,8 +136,8 @@ github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.5/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.7/go.mod h1:n+brtR0CgQNWTVd5ZUFpTBC8YFBDLK/h/bpaJ8/DtOE= -github.com/google/go-cmp v0.5.9 h1:O2Tfq5qg4qc4AmwVlvv0oLiVAGB7enBSJ2x2DqQFi38= -github.com/google/go-cmp v0.5.9/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= +github.com/google/go-cmp v0.6.0 h1:ofyhxvXcZhMsU5ulbFiLKl/XBFqE1GSq7atu8tAmTRI= +github.com/google/go-cmp v0.6.0/go.mod h1:17dUlkBOakJ0+DkrSSNjCkIjxS6bF9zb3elmeNGIjoY= github.com/google/go-github/v53 v53.2.0 h1:wvz3FyF53v4BK+AsnvCmeNhf8AkTaeh2SoYu/XUvTtI= github.com/google/go-github/v53 v53.2.0/go.mod h1:XhFRObz+m/l+UCm9b7KSIC3lT3NWSXGt7mOsAWEloao= github.com/google/go-querystring v1.1.0 h1:AnCroh3fv4ZBgVIf1Iwtovgjaw/GiKJo8M8yD/fhyJ8= @@ -161,8 +165,8 @@ github.com/hashicorp/go-msgpack v0.5.3/go.mod h1:ahLV/dePpqEmjfWmKiqvPkv/twdG7iP github.com/hashicorp/go-multierror v1.0.0/go.mod h1:dHtQlpGsu+cZNNAkkCN/P3hoUDHhCYQXV3UM06sGGrk= github.com/hashicorp/go-multierror v1.1.0/go.mod h1:spPvp8C1qA32ftKqdAHm4hHTbPw+vmowP0z+KUhOZdA= github.com/hashicorp/go-plugin v1.0.1/go.mod h1:++UyYGoz3o5w9ZzAdZxtQKrWWP+iqPBn3cQptSMzBuY= -github.com/hashicorp/go-plugin v1.5.1 h1:oGm7cWBaYIp3lJpx1RUEfLWophprE2EV/KUeqBYo+6k= -github.com/hashicorp/go-plugin v1.5.1/go.mod h1:w1sAEES3g3PuV/RzUrgow20W2uErMly84hhD3um1WL4= +github.com/hashicorp/go-plugin v1.5.2 h1:aWv8eimFqWlsEiMrYZdPYl+FdHaBJSN4AWwGWfT1G2Y= +github.com/hashicorp/go-plugin v1.5.2/go.mod h1:w1sAEES3g3PuV/RzUrgow20W2uErMly84hhD3um1WL4= github.com/hashicorp/go-retryablehttp v0.5.4/go.mod h1:9B5zBasrRhHXnJnui7y6sL7es7NDiJgTc6Er0maI1Xs= github.com/hashicorp/go-rootcerts v1.0.1/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= github.com/hashicorp/go-rootcerts v1.0.2/go.mod h1:pqUvnprVnM5bf7AOirdbb01K4ccR319Vf4pU3K5EGc8= @@ -188,19 +192,17 @@ github.com/hashicorp/yamux v0.1.1 h1:yrQxtgseBDrq9Y652vSRDvsKCJKOUD+GzTS4Y0Y8pvE github.com/hashicorp/yamux v0.1.1/go.mod h1:CtWFDAQgb7dxtzFs4tWbplKIe2jSi3+5vKbgIO0SLnQ= github.com/hjson/hjson-go/v4 v4.0.0 h1:wlm6IYYqHjOdXH1gHev4VoXCaW20HdQAGCxdOEEg2cs= github.com/hjson/hjson-go/v4 v4.0.0/go.mod h1:KaYt3bTw3zhBjYqnXkYywcYctk0A2nxeEFTse3rH13E= -github.com/iancoleman/orderedmap v0.0.0-20190318233801-ac98e3ecb4b0/go.mod h1:N0Wam8K1arqPXNWjMo21EXnBPOPp36vB07FNRdD2geA= -github.com/iancoleman/orderedmap v0.3.0 h1:5cbR2grmZR/DiVt+VJopEhtVs9YGInGIxAoMJn+Ichc= -github.com/iancoleman/orderedmap v0.3.0/go.mod h1:XuLcCUkdL5owUCQeF2Ue9uuw1EptkJDkXXS7VoV7XGE= github.com/inconshreveable/mousetrap v1.1.0 h1:wN+x4NVGpMsO7ErUn/mUI3vEoE6Jt13X2s0bqwp9tc8= github.com/inconshreveable/mousetrap v1.1.0/go.mod h1:vpF70FUmC8bwa3OWnCshd2FqLfsEA9PFc4w1p2J65bw= -github.com/invopop/jsonschema v0.8.0 h1:9Vblm5uNqURXUSaX0QUYcI/Hcu5rrvOz5MbpWgw0VkM= -github.com/invopop/jsonschema v0.8.0/go.mod h1:O9uiLokuu0+MGFlyiaqtWxwqJm41/+8Nj0lD7A36YH0= +github.com/invopop/jsonschema v0.12.0 h1:6ovsNSuvn9wEQVOyc72aycBMVQFKz7cPdMJn10CvzRI= +github.com/invopop/jsonschema v0.12.0/go.mod h1:ffZ5Km5SWWRAIN6wbDXItl95euhFz2uON45H2qjYt+0= github.com/jhump/protoreflect v1.15.1 h1:HUMERORf3I3ZdX05WaQ6MIpd/NJ434hTp5YiKgfCL6c= github.com/jhump/protoreflect v1.15.1/go.mod h1:jD/2GMKKE6OqX8qTjhADU1e6DShO+gavG9e0Q693nKo= github.com/jmespath/go-jmespath v0.4.0/go.mod h1:T8mJZnbsbmF+m6zOOFylbeCJqk5+pHWvzYPziyZiYoo= github.com/jmespath/go-jmespath/internal/testify v1.5.1/go.mod h1:L3OGu8Wl2/fWfCI6z80xFu9LTZmf1ZRjMHUOPmWr69U= github.com/joho/godotenv v1.3.0 h1:Zjp+RcGpHhGlrMbJzXTrZZPrWj+1vfm90La1wgB6Bhc= github.com/joho/godotenv v1.3.0/go.mod h1:7hK45KPybAkOC6peb+G5yklZfMxEjkZhHbwpqxOKXbg= +github.com/josharian/intern v1.0.0/go.mod h1:5DoeVV0s6jJacbCEi61lwdGj/aVlrQvzHFFd8Hwg//Y= github.com/jpillora/backoff v1.0.0/go.mod h1:J/6gKK9jxlEcS3zixgDgUAsiuZ7yrSoa/FX5e0EB2j4= github.com/json-iterator/go v1.1.6/go.mod h1:+SdeFBvtyEkXs7REEP0seUULqWtbJapLOCVDaaPEHmU= github.com/json-iterator/go v1.1.10/go.mod h1:KdQUCv79m/52Kvf8AW2vK1V8akMuk1QjK/uOdHXbAo4= @@ -224,6 +226,8 @@ github.com/kr/pty v1.1.1/go.mod h1:pFQYn66WHrOpPYNljwOMqo10TkYh1fy3cYio2l3bCsQ= github.com/kr/text v0.1.0/go.mod h1:4Jbv+DJW3UT/LiOwJeYQe1efqtUx/iVham/4vfdArNI= github.com/kr/text v0.2.0 h1:5Nx0Ya0ZqY2ygV366QzturHI13Jq95ApcVaJBhpS+AY= github.com/kr/text v0.2.0/go.mod h1:eLer722TekiGuMkidMxC/pM04lWEeraHUUmBw8l2grE= +github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= +github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= github.com/mattn/go-colorable v0.0.9/go.mod h1:9vuHe8Xs5qXnSaW/c/ABM9alt+Vo+STaOChaDxuIBZU= github.com/mattn/go-colorable v0.1.4/go.mod h1:U0ppj6V5qS13XJ6of8GYAs25YV2eR4EVcfRqFIhoBtE= github.com/mattn/go-colorable v0.1.6/go.mod h1:u6P/XSegPjTcexA+o6vUJrdnUu04hMope9wVRipJSqc= @@ -277,8 +281,8 @@ github.com/oklog/run v1.1.0/go.mod h1:sVPdnTZT1zYwAJeCMu2Th4T21pA3FPOQRfWjQlk7DV github.com/pascaldekloe/goe v0.0.0-20180627143212-57f6aae5913c/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pascaldekloe/goe v0.1.0/go.mod h1:lzWF7FIEvWOWxwDKqyGYQf6ZUaNfKdP144TG7ZOy1lc= github.com/pelletier/go-toml v1.7.0/go.mod h1:vwGMzjaWMwyfHwgIBhI2YUM4fB6nL6lVAvS1LBMMhTE= -github.com/pelletier/go-toml v1.9.4 h1:tjENF6MfZAg8e4ZmZTeWaWiT2vXtsoO6+iuOjFhECwM= -github.com/pelletier/go-toml v1.9.4/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= +github.com/pelletier/go-toml v1.9.5 h1:4yBQzkHv+7BHq2PQUZF3Mx0IYxG7LsP222s7Agd3ve8= +github.com/pelletier/go-toml v1.9.5/go.mod h1:u1nR/EPcESfeI/szUZKdtJ0xRNbUoANCkoOuaOx1Y+c= github.com/pierrec/lz4 v2.0.5+incompatible/go.mod h1:pdkljMzZIN41W+lC3N2tnIh5sFi+IEE17M5jbnwPHcY= github.com/pingcap/errors v0.11.4 h1:lFuQV/oaUMGcD2tqt+01ROSmJs75VG1ToEOkZIZ4nE4= github.com/pingcap/errors v0.11.4/go.mod h1:Oi8TUi2kEtXXLMJk9l1cGmz20kV3TaQ0usTwv5KuLY8= @@ -295,14 +299,14 @@ github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXP github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.7.1/go.mod h1:PY5Wy2awLA44sXw4AOSfFBetzPP4j5+D6mVACh+pe2M= github.com/prometheus/client_golang v1.11.1/go.mod h1:Z6t4BnS23TR94PD6BsDNk8yVqroYurpAkEiz0P2BEV0= -github.com/prometheus/client_golang v1.16.0 h1:yk/hx9hDbrGHovbci4BY+pRMfSuuat626eFsHb7tmT8= -github.com/prometheus/client_golang v1.16.0/go.mod h1:Zsulrv/L9oM40tJ7T815tM89lFEugiJ9HzIqaAx4LKc= +github.com/prometheus/client_golang v1.17.0 h1:rl2sfwZMtSthVU752MqfjQozy7blglC+1SOtjMAMh+Q= +github.com/prometheus/client_golang v1.17.0/go.mod h1:VeL+gMmOAxkS2IqfCq0ZmHSL+LjWfWDUmp1mBz9JgUY= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= -github.com/prometheus/client_model v0.4.0 h1:5lQXD3cAg1OXBf4Wq03gTrXHeaV0TQvGfUooCfx1yqY= -github.com/prometheus/client_model v0.4.0/go.mod h1:oMQmHW1/JoDwqLtg57MGgP/Fb1CJEYF2imWWhWtMkYU= +github.com/prometheus/client_model v0.5.0 h1:VQw1hfvPvk3Uv6Qf29VrPF32JB6rtbgI6cYPYQjL0Qw= +github.com/prometheus/client_model v0.5.0/go.mod h1:dTiFglRmd66nLR9Pv9f0mZi7B7fk5Pm3gvsjB5tr+kI= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.10.0/go.mod h1:Tlit/dnDKsSWFlCLTWaA1cyBgKHSMdTB80sz/V91rCo= github.com/prometheus/common v0.26.0/go.mod h1:M7rCNAaPfAosfx8veZJCuw84e35h3Cfd9VFqTh1DIvc= @@ -312,8 +316,8 @@ github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.1.3/go.mod h1:lV6e/gmhEcM9IjHGsFOCxxuZ+z1YqCvr4OA4YeYWdaU= github.com/prometheus/procfs v0.6.0/go.mod h1:cz+aTbrPOrUb4q7XlbU9ygM+/jj0fzG6c1xBZuNvfVA= -github.com/prometheus/procfs v0.11.1 h1:xRC8Iq1yyca5ypa9n1EZnWZkt7dwcoRPQwX/5gwaUuI= -github.com/prometheus/procfs v0.11.1/go.mod h1:eesXgaPo1q7lBpVMoMy0ZOFTth9hBn4W/y0/p/ScXhY= +github.com/prometheus/procfs v0.12.0 h1:jluTpSng7V9hY0O2R9DzzJHYb2xULk9VTR1V1R/k6Bo= +github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3cnaOZAZEfOo= github.com/rhnvrm/simples3 v0.6.1/go.mod h1:Y+3vYm2V7Y4VijFoJHHTrja6OgPrJ2cBti8dPGkC3sA= github.com/robfig/cron/v3 v3.0.1 h1:WdRxkvbJztn8LMz/QEvLN5sBU+xKpSqwwUO1Pjr4qDs= github.com/robfig/cron/v3 v3.0.1/go.mod h1:eQICP3HwyT7UooqI/z+Ov+PtYAWygg1TEWWzGIFLtro= @@ -323,8 +327,8 @@ github.com/rogpeppe/go-internal v1.8.1/go.mod h1:JeRgkft04UBgHMgCIwADu4Pn6Mtm5d4 github.com/rogpeppe/go-internal v1.10.0 h1:TMyTOH3F/DB16zRVcYyreMH6GnZZrwQVAoYjRBZyWFQ= github.com/rogpeppe/go-internal v1.10.0/go.mod h1:UQnix2H7Ngw/k4C5ijL5+65zddjncjaFoBhdsK/akog= github.com/rs/xid v1.5.0/go.mod h1:trrq9SKmegXys3aeAKXMUTdJsYXVwGY3RLcfgqegfbg= -github.com/rs/zerolog v1.30.0 h1:SymVODrcRsaRaSInD9yQtKbtWqwsfoPcRff/oRXLj4c= -github.com/rs/zerolog v1.30.0/go.mod h1:/tk+P47gFdPXq4QYjvCmT5/Gsug2nagsFWBWhAiSi1w= +github.com/rs/zerolog v1.31.0 h1:FcTR3NnLWW+NnTwwhFWiJSZr4ECLpqCm6QsEnyvbV4A= +github.com/rs/zerolog v1.31.0/go.mod h1:/7mN4D5sKwJLZQ2b/znpjC3/GQWY/xaDXUM0kKWRHss= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= github.com/ryanuber/columnize v0.0.0-20160712163229-9b3edd62028f/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= github.com/ryanuber/columnize v2.1.0+incompatible/go.mod h1:sm1tb6uqfes/u+d4ooFouqFdy9/2g9QGwK3SQygK0Ts= @@ -345,7 +349,6 @@ github.com/stretchr/objx v0.4.0/go.mod h1:YvHI0jy2hoMjB+UWwv71VJQ9isScKT/TqJzVSS github.com/stretchr/objx v0.5.0/go.mod h1:Yh+to48EsGEfYuaHDzXPcE3xhTkx73EhmCGUpEOglKo= github.com/stretchr/testify v1.2.2/go.mod h1:a8OnRcib4nhh0OaRAV+Yts87kKdq0PP7pXfy6kDkUVs= github.com/stretchr/testify v1.3.0/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= -github.com/stretchr/testify v1.3.1-0.20190311161405-34c6fa2dc709/go.mod h1:M5WIy9Dh21IEIfnGCwXGc5bZfKNJtfHm1UVUgZn+9EI= github.com/stretchr/testify v1.4.0/go.mod h1:j7eGeouHqKxXV5pUuKE4zz7dFj8WfuZ+81PSLYec5m4= github.com/stretchr/testify v1.5.1/go.mod h1:5W2xD1RspED5o8YsWQXVCued0rvSQ+mT+I5cxcmMvtA= github.com/stretchr/testify v1.7.0/go.mod h1:6Fq8oRcR53rry900zMqJjRRixrwX3KX962/h/Wwjteg= @@ -355,6 +358,8 @@ github.com/stretchr/testify v1.8.0/go.mod h1:yNjHg4UonilssWZ8iaSj1OCr/vHnekPRkoO github.com/stretchr/testify v1.8.2/go.mod h1:w2LPCIKwWwSfY2zedu0+kehJoqGctiVI29o6fzry7u4= github.com/stretchr/testify v1.8.4 h1:CcVxjf3Q8PM0mHUKJCdn+eZZtm5yQwehR5yeSVQQcUk= github.com/stretchr/testify v1.8.4/go.mod h1:sz/lmYIOXD/1dqDmKjjqLyZ2RngseejIcXlSw2iwfAo= +github.com/wk8/go-ordered-map/v2 v2.1.8 h1:5h/BUHu93oj4gIdvHHHGsScSTMijfx5PeYkE/fJgbpc= +github.com/wk8/go-ordered-map/v2 v2.1.8/go.mod h1:5nJHM5DyteebpVlHnWMV0rPz6Zp7+xBAnxjb1X5vnTw= github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74= github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= @@ -364,18 +369,18 @@ github.com/zenizh/go-capturer v0.0.0-20211219060012-52ea6c8fed04/go.mod h1:FiwNQ go.etcd.io/etcd/api/v3 v3.5.4/go.mod h1:5GB2vv4A4AOn3yk7MftYGHkUfGtDHnEraIjym4dYz5A= go.etcd.io/etcd/client/pkg/v3 v3.5.4/go.mod h1:IJHfcCEKxYu1Os13ZdwCwIUTUVGYTSAM3YSwc9/Ac1g= go.etcd.io/etcd/client/v3 v3.5.4/go.mod h1:ZaRkVgBZC+L+dLCjTcF1hRXpgZXQPOvnA/Ak/gq3kiY= -go.opentelemetry.io/otel v1.18.0 h1:TgVozPGZ01nHyDZxK5WGPFB9QexeTMXEH7+tIClWfzs= -go.opentelemetry.io/otel v1.18.0/go.mod h1:9lWqYO0Db579XzVuCKFNPDl4s73Voa+zEck3wHaAYQI= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.18.0 h1:IAtl+7gua134xcV3NieDhJHjjOVeJhXAnYf/0hswjUY= -go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.18.0/go.mod h1:w+pXobnBzh95MNIkeIuAKcHe/Uu/CX2PKIvBP6ipKRA= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.18.0 h1:yE32ay7mJG2leczfREEhoW3VfSZIvHaB+gvVo1o8DQ8= -go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.18.0/go.mod h1:G17FHPDLt74bCI7tJ4CMitEk4BXTYG4FW6XUpkPBXa4= -go.opentelemetry.io/otel/metric v1.18.0 h1:JwVzw94UYmbx3ej++CwLUQZxEODDj/pOuTCvzhtRrSQ= -go.opentelemetry.io/otel/metric v1.18.0/go.mod h1:nNSpsVDjWGfb7chbRLUNW+PBNdcSTHD4Uu5pfFMOI0k= -go.opentelemetry.io/otel/sdk v1.18.0 h1:e3bAB0wB3MljH38sHzpV/qWrOTCFrdZF2ct9F8rBkcY= -go.opentelemetry.io/otel/sdk v1.18.0/go.mod h1:1RCygWV7plY2KmdskZEDDBs4tJeHG92MdHZIluiYs/M= -go.opentelemetry.io/otel/trace v1.18.0 h1:NY+czwbHbmndxojTEKiSMHkG2ClNH2PwmcHrdo0JY10= -go.opentelemetry.io/otel/trace v1.18.0/go.mod h1:T2+SGJGuYZY3bjj5rgh/hN7KIrlpWC5nS8Mjvzckz+0= +go.opentelemetry.io/otel v1.19.0 h1:MuS/TNf4/j4IXsZuJegVzI1cwut7Qc00344rgH7p8bs= +go.opentelemetry.io/otel v1.19.0/go.mod h1:i0QyjOq3UPoTzff0PJB2N66fb4S0+rSbSB15/oyH9fY= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0 h1:Mne5On7VWdx7omSrSSZvM4Kw7cS7NQkOOmLcgscI51U= +go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.19.0/go.mod h1:IPtUMKL4O3tH5y+iXVyAXqpAwMuzC1IrxVS81rummfE= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0 h1:3d+S281UTjM+AbF31XSOYn1qXn3BgIdWl8HNEpx08Jk= +go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.19.0/go.mod h1:0+KuTDyKL4gjKCF75pHOX4wuzYDUZYfAQdSu43o+Z2I= +go.opentelemetry.io/otel/metric v1.19.0 h1:aTzpGtV0ar9wlV4Sna9sdJyII5jTVJEvKETPiOKwvpE= +go.opentelemetry.io/otel/metric v1.19.0/go.mod h1:L5rUsV9kM1IxCj1MmSdS+JQAcVm319EUrDVLrt7jqt8= +go.opentelemetry.io/otel/sdk v1.19.0 h1:6USY6zH+L8uMH8L3t1enZPR3WFEmSTADlqldyHtJi3o= +go.opentelemetry.io/otel/sdk v1.19.0/go.mod h1:NedEbbS4w3C6zElbLdPJKOpJQOrGUJ+GfzpjUvI0v1A= +go.opentelemetry.io/otel/trace v1.19.0 h1:DFVQmlVbfVeOuBRrwdtaehRrWiL1JoVs9CPIQ1Dzxpg= +go.opentelemetry.io/otel/trace v1.19.0/go.mod h1:mfaSyvGyEJEI0nyV2I4qhNQnbBOUUmYZpYojqMnX2vo= go.opentelemetry.io/proto/otlp v1.0.0 h1:T0TX0tmXU8a3CbNXzEKGeU5mIVOdf0oykP+u2lIVU/I= go.opentelemetry.io/proto/otlp v1.0.0/go.mod h1:Sy6pihPLfYHkr3NkUbEhGHFhINUSI/v80hjKIs5JXpM= go.uber.org/atomic v1.7.0/go.mod h1:fEN4uk6kAWBTFdckzkM89CLk9XfWZrxpCo0nPH17wJc= @@ -394,11 +399,11 @@ golang.org/x/crypto v0.0.0-20200622213623-75b288015ac9/go.mod h1:LzIPMQfyMNhhGPh golang.org/x/crypto v0.0.0-20210921155107-089bfa567519/go.mod h1:GvvjBRRGRdwPK5ydBHafDWAxML/pGHZbMvKqRZ5+Abc= golang.org/x/crypto v0.3.1-0.20221117191849-2c476679df9a/go.mod h1:hebNnKkNXi2UzZN1eVRvBB7co0a+JxK6XbPiWVs/3J4= golang.org/x/crypto v0.7.0/go.mod h1:pYwdfH91IfpZVANVyUOhSIPZaFoJGxTFbZhFTx+dXZU= -golang.org/x/crypto v0.13.0 h1:mvySKfSWJ+UKUii46M40LOvyWfN0s2U+46/jDd0e6Ck= -golang.org/x/crypto v0.13.0/go.mod h1:y6Z2r+Rw4iayiXXAIxJIDAJ1zMW4yaTpebo8fPOliYc= +golang.org/x/crypto v0.14.0 h1:wBqGXzWJW6m1XrIKlAH0Hs1JJ7+9KBwnIO8v66Q9cHc= +golang.org/x/crypto v0.14.0/go.mod h1:MVFd36DqK4CsrnJYDkBA3VC4m2GkXAM0PvzMCn4JQf4= golang.org/x/exp v0.0.0-20190121172915-509febef88a4/go.mod h1:CJ0aWSM057203Lf6IL+f9T1iT9GByDxfZKAQTCR3kQA= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9 h1:GoHiUyI/Tp2nVkLI2mCxVkOjsbSXD66ic0XW0js0R9g= -golang.org/x/exp v0.0.0-20230905200255-921286631fa9/go.mod h1:S2oDrQGGwySpoQPVqRShND87VCbxmc6bL1Yd2oYrm6k= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d h1:jtJma62tbqLibJ5sFQz8bKtEM8rJBtfilJ2qTU199MI= +golang.org/x/exp v0.0.0-20231006140011-7918f672742d/go.mod h1:ldy0pHrwJyGW56pPQzzkH36rKxoZW1tw7ZJpeKx+hdo= golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE= golang.org/x/lint v0.0.0-20190227174305-5b3e6a55c961/go.mod h1:wehouNa3lNwaWXcvxsM5YxQ5yQlVC4a0KAMCusXpPoU= golang.org/x/lint v0.0.0-20190313153728-d0100b6bd8b3/go.mod h1:6SW0HCj/g11FgYtHlgUYUwCkIfeOF89ocIRzGO/8vkc= @@ -430,13 +435,13 @@ golang.org/x/net v0.0.0-20220722155237-a158d28d115b/go.mod h1:XRhObCWvk6IyKnWLug golang.org/x/net v0.2.0/go.mod h1:KqCZLdyyvdV855qA2rE3GC2aiw5xGR5TEjj8smXukLY= golang.org/x/net v0.6.0/go.mod h1:2Tu9+aMcznHK/AK1HMvgo6xiTLG5rD5rZLDS+rp2Bjs= golang.org/x/net v0.8.0/go.mod h1:QVkue5JL9kW//ek3r6jTKnTFis1tRmNAW2P1shuFdJc= -golang.org/x/net v0.15.0 h1:ugBLEUaxABaB5AJqW9enI0ACdci2RUd4eP51NTBvuJ8= -golang.org/x/net v0.15.0/go.mod h1:idbUs1IY1+zTqbi8yxTbhexhEEk5ur9LInksu6HrEpk= +golang.org/x/net v0.17.0 h1:pVaXccu2ozPjCXewfr1S7xza/zcXTity9cCdXQYSjIM= +golang.org/x/net v0.17.0/go.mod h1:NxSsAGuq816PNPmqtQdLE42eU2Fs7NoRIZrHJAlaCOE= golang.org/x/oauth2 v0.0.0-20180821212333-d2e6202438be/go.mod h1:N/0e6XlmueqKjAGxoOufVs8QHGRruUQn6yWY3a++T0U= golang.org/x/oauth2 v0.0.0-20190226205417-e64efc72b421/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= golang.org/x/oauth2 v0.0.0-20200107190931-bf48bf16ab8d/go.mod h1:gOpvHmFTYa4IltrdGE7lF6nIHvwfUNPOp7c8zoXwtLw= -golang.org/x/oauth2 v0.12.0 h1:smVPGxink+n1ZI5pkQa8y6fZT0RW0MgCO5bFpepy4B4= -golang.org/x/oauth2 v0.12.0/go.mod h1:A74bZ3aGXgCY0qaIC9Ahg6Lglin4AMAco8cIv9baba4= +golang.org/x/oauth2 v0.13.0 h1:jDDenyj+WgFtmV3zYVoi8aE2BwtXFLWOA67ZfNWftiY= +golang.org/x/oauth2 v0.13.0/go.mod h1:/JMhi4ZRXAf4HG9LiNmxvk+45+96RUlVThiH8FzNBn0= golang.org/x/sync v0.0.0-20180314180146-1d60e4601c6f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181108010431-42b317875d0f/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= golang.org/x/sync v0.0.0-20181221193216-37e7f081c4d4/go.mod h1:RxMgew5VJxzue5/jJTE5uejpjVlOe/izrB70Jof72aM= @@ -490,8 +495,9 @@ golang.org/x/sys v0.2.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.3.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.5.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.6.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= -golang.org/x/sys v0.12.0 h1:CM0HF96J0hcLAwsHPJZjfdNzs0gftsLfgKt57wWHJ0o= golang.org/x/sys v0.12.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.13.0 h1:Af8nKPmuFypiUBjVoU9V20FiaFXOcuZI21p0ycVYYGE= +golang.org/x/sys v0.13.0/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210927222741-03fcf44c2211/go.mod h1:jbD1KX2456YbFQfuXm/mYQcufACuNUgVhRMnK/tPxf8= golang.org/x/term v0.2.0/go.mod h1:TVmDHMZPmdnySmBfhjOoOdhjzdE1h4u1VwSiw2l1Nuc= @@ -538,12 +544,12 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb h1:XFBgcDwm7irdHTbz4Zk2h7Mh+eis4nfJEFQFYzJzuIA= -google.golang.org/genproto v0.0.0-20230913181813-007df8e322eb/go.mod h1:yZTlhN0tQnXo3h00fuXNCxJdLdIdnVFVBaRJ5LWBbw4= -google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb h1:lK0oleSc7IQsUxO3U5TjL9DWlsxpEBemh+zpB7IqhWI= -google.golang.org/genproto/googleapis/api v0.0.0-20230913181813-007df8e322eb/go.mod h1:KjSP20unUpOx5kyQUFa7k4OJg0qeJ7DEZflGDu2p6Bk= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230913181813-007df8e322eb h1:Isk1sSH7bovx8Rti2wZK0UZF6oraBDK74uoyLEEVFN0= -google.golang.org/genproto/googleapis/rpc v0.0.0-20230913181813-007df8e322eb/go.mod h1:+Bk1OCOj40wS2hwAMA+aCW9ypzm63QTBBHp6lQ3p+9M= +google.golang.org/genproto v0.0.0-20231012201019-e917dd12ba7a h1:fwgW9j3vHirt4ObdHoYNwuO24BEZjSzbh+zPaNWoiY8= +google.golang.org/genproto v0.0.0-20231012201019-e917dd12ba7a/go.mod h1:EMfReVxb80Dq1hhioy0sOsY9jCE46YDgHlJ7fWVUWRE= +google.golang.org/genproto/googleapis/api v0.0.0-20231012201019-e917dd12ba7a h1:myvhA4is3vrit1a6NZCWBIwN0kNEnX21DJOJX/NvIfI= +google.golang.org/genproto/googleapis/api v0.0.0-20231012201019-e917dd12ba7a/go.mod h1:SUBoKXbI1Efip18FClrQVGjWcyd0QZd8KkvdP34t7ww= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231012201019-e917dd12ba7a h1:a2MQQVoTo96JC9PMGtGBymLp7+/RzpFc2yX/9WfFg1c= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231012201019-e917dd12ba7a/go.mod h1:4cYg8o5yUbm77w8ZX00LhMVNl/YVBFJRYWDc0uYWMs0= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg= @@ -552,8 +558,8 @@ google.golang.org/grpc v1.25.1/go.mod h1:c3i+UQWmh7LiEpx4sFZnkU36qjEYZ0imhYfXVyQ google.golang.org/grpc v1.27.0/go.mod h1:qbnxyOmOxrQa7FizSgH+ReBfzJrCY1pSN7KXBS8abTk= google.golang.org/grpc v1.33.1/go.mod h1:fr5YgcSWrqhRRxogOsw7RzIpsmvOZ6IcH4kBYTpR3n0= google.golang.org/grpc v1.38.0/go.mod h1:NREThFqKR1f3iQ6oBuvc5LadQuXVGo9rkm5ZGrQdJfM= -google.golang.org/grpc v1.58.1 h1:OL+Vz23DTtrrldqHK49FUOPHyY75rvFqJfXC84NYW58= -google.golang.org/grpc v1.58.1/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= +google.golang.org/grpc v1.58.3 h1:BjnpXut1btbtgN/6sp+brB2Kbm2LjNXnidYujAVbSoQ= +google.golang.org/grpc v1.58.3/go.mod h1:tgX3ZQDlNJGU96V6yHh1T/JeoBQ2TXdr43YbYSsCJk0= google.golang.org/protobuf v0.0.0-20200109180630-ec00e32a8dfd/go.mod h1:DFci5gLYBciE7Vtevhsrf46CRTquxDuWsQurQQe4oz8= google.golang.org/protobuf v0.0.0-20200221191635-4d8936d0db64/go.mod h1:kwYJMbMJ01Woi6D6+Kah6886xMZcty6N08ah7+eCXa0= google.golang.org/protobuf v0.0.0-20200228230310-ab0ca4ff8a60/go.mod h1:cfTl7dwQJ+fmap5saPgwCLgHXTUD7jkjRqWcaiX5VyM= From 3909b20aaff501dbe4d346473a98b43be3fd7b2d Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Fri, 13 Oct 2023 17:23:51 +0200 Subject: [PATCH 10/35] Use an atomic boolean as a simple state machine for controlling Engine start/stop Close channel after send Close listener gracefully and raise error if something happened Use for/select for stopping long-running goroutines gracefully --- errors/errors.go | 3 ++ network/engine.go | 115 +++++++++++++++++++++++++++++----------------- 2 files changed, 76 insertions(+), 42 deletions(-) diff --git a/errors/errors.go b/errors/errors.go index f16a6501..a8799bbc 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -23,6 +23,7 @@ const ( ErrCodeServerSendFailed ErrCodeServerListenFailed ErrCodeSplitHostPortFailed + ErrCodeCloseListenerFailed ErrCodeAcceptFailed ErrCodeReadFailed ErrCodePutFailed @@ -85,6 +86,8 @@ var ( ErrCodeServerListenFailed, "couldn't listen on the server", nil) ErrSplitHostPortFailed = NewGatewayDError( ErrCodeSplitHostPortFailed, "failed to split host:port", nil) + ErrCloseListenerFailed = NewGatewayDError( + ErrCodeCloseListenerFailed, "failed to close listener", nil) ErrAcceptFailed = NewGatewayDError( ErrCodeAcceptFailed, "failed to accept connection", nil) diff --git a/network/engine.go b/network/engine.go index c9800e1a..ec026d02 100644 --- a/network/engine.go +++ b/network/engine.go @@ -5,6 +5,7 @@ import ( "net" "strconv" "sync" + "sync/atomic" "time" "github.com/gatewayd-io/gatewayd/config" @@ -35,6 +36,7 @@ type Engine struct { host string port int connections uint32 + running *atomic.Bool stopServer chan struct{} mu *sync.RWMutex } @@ -49,7 +51,13 @@ func (engine *Engine) Stop(ctx context.Context) error { _, cancel := context.WithDeadline(ctx, time.Now().Add(config.DefaultEngineStopTimeout)) defer cancel() + engine.running.Store(false) + if err := engine.listener.Close(); err != nil { + engine.stopServer <- struct{}{} + return gerr.ErrCloseListenerFailed.Wrap(err) + } engine.stopServer <- struct{}{} + close(engine.stopServer) return nil } @@ -59,6 +67,7 @@ func Run(network, address string, server *Server) *gerr.GatewayDError { connections: 0, stopServer: make(chan struct{}), mu: &sync.RWMutex{}, + running: &atomic.Bool{}, } if action := server.OnBoot(server.engine); action != None { @@ -71,7 +80,6 @@ func Run(network, address string, server *Server) *gerr.GatewayDError { server.logger.Error().Err(err).Msg("Server failed to start listening") return gerr.ErrServerListenFailed.Wrap(err) } - defer server.engine.listener.Close() if server.engine.listener == nil { server.logger.Error().Msg("Server is not properly initialized") @@ -102,56 +110,79 @@ func Run(network, address string, server *Server) *gerr.GatewayDError { } for { - interval, action := server.OnTick() - if action == Shutdown { - server.OnShutdown(server.engine) - return - } - if interval == time.Duration(0) { + select { + case <-server.engine.stopServer: return + default: + interval, action := server.OnTick() + if action == Shutdown { + server.OnShutdown(server.engine) + return + } + if interval == time.Duration(0) { + return + } + time.Sleep(interval) } - time.Sleep(interval) } }(server) - for { - conn, err := server.engine.listener.Accept() - if err != nil { - server.logger.Error().Err(err).Msg("Failed to accept connection") - return gerr.ErrAcceptFailed.Wrap(err) - } + server.engine.running.Store(true) - if out, action := server.OnOpen(conn); action != None { - if _, err := conn.Write(out); err != nil { - server.logger.Error().Err(err).Msg("Failed to write to connection") - } - conn.Close() - if action == Shutdown { - server.OnShutdown(server.engine) - return nil - } - } - server.engine.mu.Lock() - server.engine.connections++ - server.engine.mu.Unlock() - - // For every new connection, a new unbuffered channel is created to help - // stop the proxy, recycle the server connection and close stale connections. - stopConnection := make(chan struct{}) - go func(server *Server, conn net.Conn, stopConnection chan struct{}) { - if action := server.OnTraffic(conn, stopConnection); action == Close { - return + for { + select { + case <-server.engine.stopServer: + server.logger.Info().Msg("Server stopped") + return nil + default: + conn, err := server.engine.listener.Accept() + if err != nil { + if !server.engine.running.Load() { + return nil + } + server.logger.Error().Err(err).Msg("Failed to accept connection") + return gerr.ErrAcceptFailed.Wrap(err) } - }(server, conn, stopConnection) - go func(server *Server, conn net.Conn, stopConnection chan struct{}) { - <-stopConnection + if out, action := server.OnOpen(conn); action != None { + if _, err := conn.Write(out); err != nil { + server.logger.Error().Err(err).Msg("Failed to write to connection") + } + conn.Close() + if action == Shutdown { + server.OnShutdown(server.engine) + return nil + } + } server.engine.mu.Lock() - server.engine.connections-- + server.engine.connections++ server.engine.mu.Unlock() - if action := server.OnClose(conn, err); action == Close { - return - } - }(server, conn, stopConnection) + + // For every new connection, a new unbuffered channel is created to help + // stop the proxy, recycle the server connection and close stale connections. + stopConnection := make(chan struct{}) + go func(server *Server, conn net.Conn, stopConnection chan struct{}) { + if action := server.OnTraffic(conn, stopConnection); action == Close { + return + } + }(server, conn, stopConnection) + + go func(server *Server, conn net.Conn, stopConnection chan struct{}) { + for { + select { + case <-stopConnection: + server.engine.mu.Lock() + server.engine.connections-- + server.engine.mu.Unlock() + if action := server.OnClose(conn, err); action == Close { + return + } + return + case <-server.engine.stopServer: + return + } + } + }(server, conn, stopConnection) + } } } From 004608a878419f218b56ddda8b2a3f98ab2335e1 Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Fri, 13 Oct 2023 17:28:05 +0200 Subject: [PATCH 11/35] Use a mutex to read/update server.Status --- network/server.go | 34 ++++++++++++++++++++++------------ 1 file changed, 22 insertions(+), 12 deletions(-) diff --git a/network/server.go b/network/server.go index 3280fc30..1d85c300 100644 --- a/network/server.go +++ b/network/server.go @@ -245,13 +245,17 @@ func (s *Server) OnTraffic(conn net.Conn, stopConnection chan struct{}) Action { // Pass the traffic from the client to server. // If there is an error, log it and close the connection. - go func(s *Server, conn net.Conn, stopConnection chan struct{}) { + go func(server *Server, conn net.Conn, stopConnection chan struct{}) { for { - s.logger.Trace().Msg("Passing through traffic from client to server") - if err := s.proxy.PassThroughToServer(conn); err != nil { - s.logger.Trace().Err(err).Msg("Failed to pass through traffic") + server.logger.Trace().Msg("Passing through traffic from client to server") + if err := server.proxy.PassThroughToServer(conn); err != nil { + server.logger.Trace().Err(err).Msg("Failed to pass through traffic") span.RecordError(err) - stopConnection <- struct{}{} + server.engine.mu.Lock() + if server.Status == config.Stopped { + stopConnection <- struct{}{} + } + server.engine.mu.Unlock() break } } @@ -259,13 +263,17 @@ func (s *Server) OnTraffic(conn net.Conn, stopConnection chan struct{}) Action { // Pass the traffic from the server to client. // If there is an error, log it and close the connection. - go func(s *Server, conn net.Conn, stopConnection chan struct{}) { + go func(server *Server, conn net.Conn, stopConnection chan struct{}) { for { - s.logger.Debug().Msg("Passing through traffic from server to client") - if err := s.proxy.PassThroughToClient(conn); err != nil { - s.logger.Trace().Err(err).Msg("Failed to pass through traffic") + server.logger.Debug().Msg("Passing through traffic from server to client") + if err := server.proxy.PassThroughToClient(conn); err != nil { + server.logger.Trace().Err(err).Msg("Failed to pass through traffic") span.RecordError(err) - stopConnection <- struct{}{} + server.engine.mu.Lock() + if server.Status == config.Stopped { + stopConnection <- struct{}{} + } + server.engine.mu.Unlock() break } } @@ -279,7 +287,7 @@ func (s *Server) OnShutdown(Engine) { _, span := otel.Tracer("gatewayd").Start(s.ctx, "OnShutdown") defer span.End() - s.logger.Debug().Msg("GatewayD is shutting down...") + s.logger.Debug().Msg("GatewayD is shutting down") pluginTimeoutCtx, cancel := context.WithTimeout(context.Background(), s.pluginTimeout) defer cancel() @@ -376,7 +384,7 @@ func (s *Server) Run() error { // Start the server. origErr := Run(s.Network, addr, s) - if origErr != nil { + if origErr != nil && origErr.Unwrap() != nil { s.logger.Error().Err(origErr).Msg("Failed to start server") span.RecordError(origErr) return gerr.ErrFailedToStartServer.Wrap(origErr) @@ -411,6 +419,8 @@ func (s *Server) IsRunning() bool { defer span.End() span.SetAttributes(attribute.Bool("status", s.Status == config.Running)) + s.engine.mu.Lock() + defer s.engine.mu.Unlock() return s.Status == config.Running } From fcebc7bbd89c54d6e7d1b04ce38f161c19f3e3ef Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Fri, 13 Oct 2023 17:29:26 +0200 Subject: [PATCH 12/35] Use separate metrics for proxy passthroughs (to-server, to-client) and plugin registry Check if the client is actually connected before closing it Stop scheduler before clearing the jobs (safer) --- metrics/builtins.go | 11 ++++++++--- network/proxy.go | 35 +++++++++++++++++++++++++++++------ plugin/plugin_registry.go | 4 ++-- 3 files changed, 39 insertions(+), 11 deletions(-) diff --git a/metrics/builtins.go b/metrics/builtins.go index e56b978d..2e588a58 100644 --- a/metrics/builtins.go +++ b/metrics/builtins.go @@ -75,10 +75,15 @@ var ( Name: "proxied_connections", Help: "Number of proxy connects", }) - ProxyPassThroughs = promauto.NewCounter(prometheus.CounterOpts{ + ProxyPassThroughsToClient = promauto.NewCounter(prometheus.CounterOpts{ Namespace: Namespace, - Name: "proxy_passthroughs_total", - Help: "Number of successful proxy passthroughs", + Name: "proxy_passthroughs_to_client_total", + Help: "Number of successful proxy passthroughs from server to client", + }) + ProxyPassThroughsToServer = promauto.NewCounter(prometheus.CounterOpts{ + Namespace: Namespace, + Name: "proxy_passthroughs_to_server_total", + Help: "Number of successful proxy passthroughs from client to server", }) ProxyPassThroughTerminations = promauto.NewCounter(prometheus.CounterOpts{ Namespace: Namespace, diff --git a/network/proxy.go b/network/proxy.go index 66bf065d..02f2605c 100644 --- a/network/proxy.go +++ b/network/proxy.go @@ -316,7 +316,7 @@ func (pr *Proxy) PassThroughToServer(conn net.Conn) *gerr.GatewayDError { // If the hook wants to terminate the connection, do it. if pr.shouldTerminate(result) { if modResponse, modReceived := pr.getPluginModifiedResponse(result); modResponse != nil { - metrics.ProxyPassThroughs.Inc() + metrics.ProxyPassThroughsToClient.Inc() metrics.ProxyPassThroughTerminations.Inc() metrics.BytesSentToClient.Observe(float64(modReceived)) metrics.TotalTrafficBytes.Observe(float64(modReceived)) @@ -357,6 +357,8 @@ func (pr *Proxy) PassThroughToServer(conn net.Conn) *gerr.GatewayDError { } span.AddEvent("Ran the OnTrafficToServer hooks") + metrics.ProxyPassThroughsToServer.Inc() + return nil } @@ -459,7 +461,7 @@ func (pr *Proxy) PassThroughToClient(conn net.Conn) *gerr.GatewayDError { span.RecordError(errVerdict) } - metrics.ProxyPassThroughs.Inc() + metrics.ProxyPassThroughsToClient.Inc() return errVerdict } @@ -501,8 +503,16 @@ func (pr *Proxy) Shutdown() { defer span.End() pr.availableConnections.ForEach(func(key, value interface{}) bool { - if cl, ok := value.(*Client); ok { - cl.Close() + if client, ok := value.(*Client); ok { + if client.IsConnected() { + // This will stop all the Conn.Read() and Conn.Write() calls. + // Ref: https://groups.google.com/g/golang-nuts/c/VPVWFrpIEyo + if err := client.Conn.SetDeadline(time.Now()); err != nil { + pr.logger.Error().Err(err).Msg("Error setting the deadline") + span.RecordError(err) + } + client.Close() + } } return true }) @@ -511,14 +521,27 @@ func (pr *Proxy) Shutdown() { pr.busyConnections.ForEach(func(key, value interface{}) bool { if conn, ok := key.(net.Conn); ok { + // This will stop all the Conn.Read() and Conn.Write() calls. + if err := conn.SetDeadline(time.Now()); err != nil { + pr.logger.Error().Err(err).Msg("Error setting the deadline") + span.RecordError(err) + } conn.Close() } - if cl, ok := value.(*Client); ok { - cl.Close() + if client, ok := value.(*Client); ok { + if client != nil { + // This will stop all the Conn.Read() and Conn.Write() calls. + if err := client.Conn.SetDeadline(time.Now()); err != nil { + pr.logger.Error().Err(err).Msg("Error setting the deadline") + span.RecordError(err) + } + client.Close() + } } return true }) pr.busyConnections.Clear() + pr.scheduler.Stop() pr.scheduler.Clear() pr.logger.Debug().Msg("All busy connections have been closed") } diff --git a/plugin/plugin_registry.go b/plugin/plugin_registry.go index b975d86e..e8d0d993 100644 --- a/plugin/plugin_registry.go +++ b/plugin/plugin_registry.go @@ -271,6 +271,8 @@ func (reg *Registry) Run( _, span := otel.Tracer(config.TracerName).Start(reg.ctx, "Run") defer span.End() + metrics.PluginHooksExecuted.Inc() + if ctx == nil { return nil, gerr.ErrNilContext } @@ -377,8 +379,6 @@ func (reg *Registry) Run( delete(reg.hooks[hookName], priority) } - metrics.PluginHooksExecuted.Inc() - return returnVal.AsMap(), nil } From 38fa88a148044b6d01c4d2f7b537c6b58436bed8 Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Fri, 13 Oct 2023 17:33:31 +0200 Subject: [PATCH 13/35] Use an atomic bool to check if the client is connected or not before taking any action The atomic bool will eliminate the need for empty read Remove unnecessary select Add deadline to kill the client connection immediately --- network/client.go | 75 +++++++++++++++++++++++++++-------------------- 1 file changed, 44 insertions(+), 31 deletions(-) diff --git a/network/client.go b/network/client.go index de2ef2a8..9cff7a9d 100644 --- a/network/client.go +++ b/network/client.go @@ -5,6 +5,7 @@ import ( "context" "fmt" "net" + "sync/atomic" "time" "github.com/gatewayd-io/gatewayd/config" @@ -26,8 +27,9 @@ type IClient interface { type Client struct { net.Conn - logger zerolog.Logger - ctx context.Context //nolint:containedctx + logger zerolog.Logger + ctx context.Context //nolint:containedctx + connected atomic.Bool TCPKeepAlive bool TCPKeepAlivePeriod time.Duration @@ -53,6 +55,7 @@ func NewClient(ctx context.Context, clientConfig *config.Client, logger zerolog. return nil } + client.connected.Store(false) client.logger = logger // Try to resolve the address and log an error if it can't be resolved. @@ -87,6 +90,7 @@ func NewClient(ctx context.Context, clientConfig *config.Client, logger zerolog. } client.Conn = conn + client.connected.Store(true) // Set the TCP keep alive. client.TCPKeepAlive = clientConfig.TCPKeepAlive @@ -146,6 +150,11 @@ func (c *Client) Send(data []byte) (int, *gerr.GatewayDError) { _, span := otel.Tracer(config.TracerName).Start(c.ctx, "Send") defer span.End() + if !c.connected.Load() { + span.RecordError(gerr.ErrClientNotConnected) + return 0, gerr.ErrClientNotConnected + } + sent := 0 received := len(data) for { @@ -170,8 +179,6 @@ func (c *Client) Send(data []byte) (int, *gerr.GatewayDError) { }, ).Msg("Sent data to server") - metrics.BytesSentToServer.Observe(float64(sent)) - return sent, nil } @@ -180,6 +187,11 @@ func (c *Client) Receive() (int, []byte, *gerr.GatewayDError) { _, span := otel.Tracer(config.TracerName).Start(c.ctx, "Receive") defer span.End() + if !c.connected.Load() { + span.RecordError(gerr.ErrClientNotConnected) + return 0, nil, gerr.ErrClientNotConnected + } + var ctx context.Context var cancel context.CancelFunc if c.ReceiveTimeout > 0 { @@ -192,26 +204,21 @@ func (c *Client) Receive() (int, []byte, *gerr.GatewayDError) { var received int buffer := bytes.NewBuffer(nil) // Read the data in chunks. - select { //nolint:gosimple - case <-time.After(time.Millisecond): - for ctx.Err() == nil { - chunk := make([]byte, c.ReceiveChunkSize) - read, err := c.Conn.Read(chunk) - if err != nil { - c.logger.Error().Err(err).Msg("Couldn't receive data from the server") - span.RecordError(err) - metrics.BytesReceivedFromServer.Observe(float64(received)) - return received, buffer.Bytes(), gerr.ErrClientReceiveFailed.Wrap(err) - } - received += read - buffer.Write(chunk[:read]) + for ctx.Err() == nil { + chunk := make([]byte, c.ReceiveChunkSize) + read, err := c.Conn.Read(chunk) + if err != nil { + c.logger.Error().Err(err).Msg("Couldn't receive data from the server") + span.RecordError(err) + return received, buffer.Bytes(), gerr.ErrClientReceiveFailed.Wrap(err) + } + received += read + buffer.Write(chunk[:read]) - if read == 0 || read < c.ReceiveChunkSize { - break - } + if read == 0 || read < c.ReceiveChunkSize { + break } } - metrics.BytesReceivedFromServer.Observe(float64(received)) return received, buffer.Bytes(), nil } @@ -220,6 +227,12 @@ func (c *Client) Close() { _, span := otel.Tracer(config.TracerName).Start(c.ctx, "Close") defer span.End() + // Set the deadline to now so that the connection is closed immediately. + if err := c.Conn.SetDeadline(time.Now()); err != nil { + c.logger.Error().Err(err).Msg("Failed to set deadline") + span.RecordError(err) + } + c.logger.Debug().Str("address", c.Address).Msg("Closing connection to server") if c.Conn != nil { c.Conn.Close() @@ -228,6 +241,7 @@ func (c *Client) Close() { c.Conn = nil c.Address = "" c.Network = "" + c.connected.Store(false) metrics.ServerConnections.Dec() } @@ -257,20 +271,15 @@ func (c *Client) IsConnected() bool { return false } - if n, err := c.Read([]byte{}); n == 0 && err != nil { - c.logger.Debug().Fields( - map[string]interface{}{ - "address": c.Address, - "reason": "read 0 bytes", - }).Msg("Connection to server is closed") - return false - } - - return true + return c.connected.Load() } // RemoteAddr returns the remote address of the client safely. func (c *Client) RemoteAddr() string { + if !c.connected.Load() { + return "" + } + if c.Conn != nil && c.Conn.RemoteAddr() != nil { return c.Conn.RemoteAddr().String() } @@ -280,6 +289,10 @@ func (c *Client) RemoteAddr() string { // LocalAddr returns the local address of the client safely. func (c *Client) LocalAddr() string { + if !c.connected.Load() { + return "" + } + if c.Conn != nil && c.Conn.LocalAddr() != nil { return c.Conn.LocalAddr().String() } From 1548fd2ccc2c7185a5dfb73f47e2efac5f98ad6a Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Fri, 13 Oct 2023 17:36:50 +0200 Subject: [PATCH 14/35] Pass all the variables into the goroutine to prevent data races Initialize context (_) and span inside the goroutine to prevent data races Unify log and trace event message --- .gitignore | 1 + cmd/run.go | 18 ++++++++++++------ 2 files changed, 13 insertions(+), 6 deletions(-) diff --git a/.gitignore b/.gitignore index 1934d8c4..5e633aab 100644 --- a/.gitignore +++ b/.gitignore @@ -22,6 +22,7 @@ dist/ # Editor files .vscode +.idea # Logs *.log diff --git a/cmd/run.go b/cmd/run.go index 7aba4a41..03653c00 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -98,11 +98,12 @@ func StopGracefully( } } - logger.Info().Msg("Stopping GatewayD") - span.AddEvent("Stopping GatewayD", trace.WithAttributes( + logger.Info().Msg("GatewayD is shutting down") + span.AddEvent("GatewayD is shutting down", trace.WithAttributes( attribute.String("signal", signal), )) if healthCheckScheduler != nil { + healthCheckScheduler.Stop() healthCheckScheduler.Clear() logger.Info().Msg("Stopped health check scheduler") span.AddEvent("Stopped health check scheduler") @@ -268,7 +269,7 @@ var runCmd = &cobra.Command{ startDelay := time.Now().Add(conf.Plugin.HealthCheckPeriod) if _, err := healthCheckScheduler.Every( conf.Plugin.HealthCheckPeriod).SingletonMode().StartAt(startDelay).Do(func() { - _, span = otel.Tracer(config.TracerName).Start(ctx, "Run plugin health check") + _, span := otel.Tracer(config.TracerName).Start(ctx, "Run plugin health check") defer span.End() var plugins []string @@ -749,9 +750,13 @@ var runCmd = &cobra.Command{ ) signalsCh := make(chan os.Signal, 1) signal.Notify(signalsCh, signals...) - go func(pluginRegistry *plugin.Registry, + go func(pluginTimeoutCtx context.Context, + pluginRegistry *plugin.Registry, logger zerolog.Logger, servers map[string]*network.Server, + metricsMerger *metrics.Merger, + metricsServer *http.Server, + stopChan chan struct{}, ) { for sig := range signalsCh { for _, s := range signals { @@ -771,13 +776,14 @@ var runCmd = &cobra.Command{ } } } - }(pluginRegistry, logger, servers) + }(pluginTimeoutCtx, pluginRegistry, logger, servers, metricsMerger, metricsServer, stopChan) _, span = otel.Tracer(config.TracerName).Start(runCtx, "Start servers") // Start the server. for name, server := range servers { logger := loggers[name] go func( + span trace.Span, server *network.Server, logger zerolog.Logger, healthCheckScheduler *gocron.Scheduler, @@ -797,7 +803,7 @@ var runCmd = &cobra.Command{ pluginRegistry.Shutdown() os.Exit(gerr.FailedToStartServer) } - }(server, logger, healthCheckScheduler, metricsMerger, pluginRegistry) + }(span, server, logger, healthCheckScheduler, metricsMerger, pluginRegistry) } span.End() From 3cb8719b3d3a7170982ecdb89ddbdf3cf28b2c9c Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Fri, 13 Oct 2023 17:38:27 +0200 Subject: [PATCH 15/35] Update tests to reflect the changes --- network/client_test.go | 1 + network/network_helpers_test.go | 17 ++++---- network/proxy_test.go | 74 ++++++++++++++++----------------- network/server_test.go | 58 ++++++++++++++------------ 4 files changed, 80 insertions(+), 70 deletions(-) diff --git a/network/client_test.go b/network/client_test.go index 91789295..ed431883 100644 --- a/network/client_test.go +++ b/network/client_test.go @@ -45,6 +45,7 @@ func TestNewClient(t *testing.T) { client := CreateNewClient(t) defer client.Close() + assert.NotNil(t, client) assert.Equal(t, "tcp", client.Network) assert.Equal(t, "127.0.0.1:5432", client.Address) assert.NotEmpty(t, client.ID) diff --git a/network/network_helpers_test.go b/network/network_helpers_test.go index 766c0315..12fc7ea6 100644 --- a/network/network_helpers_test.go +++ b/network/network_helpers_test.go @@ -91,8 +91,10 @@ func CollectAndComparePrometheusMetrics(t *testing.T) { # TYPE gatewayd_proxy_health_checks_total counter # HELP gatewayd_proxy_passthrough_terminations_total Number of proxy passthrough terminations by plugins # TYPE gatewayd_proxy_passthrough_terminations_total counter - # HELP gatewayd_proxy_passthroughs_total Number of successful proxy passthroughs - # TYPE gatewayd_proxy_passthroughs_total counter + # HELP gatewayd_proxy_passthroughs_to_client_total Number of successful proxy passthroughs + # TYPE gatewayd_proxy_passthroughs_to_client_total counter + # HELP gatewayd_proxy_passthroughs_to_server_total Number of successful proxy passthroughs + # TYPE gatewayd_proxy_passthroughs_to_server_total counter # HELP gatewayd_server_connections Number of server connections # TYPE gatewayd_server_connections gauge # HELP gatewayd_server_ticks_fired_total Total number of server ticks fired @@ -105,12 +107,12 @@ func CollectAndComparePrometheusMetrics(t *testing.T) { want = metadata + ` gatewayd_bytes_received_from_client_sum 67 gatewayd_bytes_received_from_client_count 1 - gatewayd_bytes_received_from_server_sum 96 - gatewayd_bytes_received_from_server_count 4 + gatewayd_bytes_received_from_server_sum 24 + gatewayd_bytes_received_from_server_count 1 gatewayd_bytes_sent_to_client_sum 24 gatewayd_bytes_sent_to_client_count 1 - gatewayd_bytes_sent_to_server_sum 282 - gatewayd_bytes_sent_to_server_count 5 + gatewayd_bytes_sent_to_server_sum 67 + gatewayd_bytes_sent_to_server_count 1 gatewayd_client_connections 1 gatewayd_plugin_hooks_executed_total 11 gatewayd_plugin_hooks_registered_total 0 @@ -118,7 +120,8 @@ func CollectAndComparePrometheusMetrics(t *testing.T) { gatewayd_proxied_connections 1 gatewayd_proxy_health_checks_total 0 gatewayd_proxy_passthrough_terminations_total 0 - gatewayd_proxy_passthroughs_total 1 + gatewayd_proxy_passthroughs_to_client_total 1 + gatewayd_proxy_passthroughs_to_server_total 1 gatewayd_server_connections 5 gatewayd_traffic_bytes_sum 182 gatewayd_traffic_bytes_count 4 diff --git a/network/proxy_test.go b/network/proxy_test.go index 14f7a0ec..ec072740 100644 --- a/network/proxy_test.go +++ b/network/proxy_test.go @@ -23,8 +23,8 @@ func TestNewProxy(t *testing.T) { NoColor: true, }) - // Create a connection pool - pool := pool.NewPool(context.Background(), config.EmptyPoolCapacity) + // Create a connection newPool + newPool := pool.NewPool(context.Background(), config.EmptyPoolCapacity) client := NewClient( context.Background(), @@ -38,13 +38,13 @@ func TestNewProxy(t *testing.T) { TCPKeepAlivePeriod: config.DefaultTCPKeepAlivePeriod, }, logger) - err := pool.Put(client.ID, client) + err := newPool.Put(client.ID, client) assert.Nil(t, err) - // Create a proxy with a fixed buffer pool + // Create a proxy with a fixed buffer newPool proxy := NewProxy( context.Background(), - pool, + newPool, plugin.NewRegistry( context.Background(), config.Loose, @@ -86,13 +86,13 @@ func TestNewProxyElastic(t *testing.T) { NoColor: true, }) - // Create a connection pool - pool := pool.NewPool(context.Background(), config.EmptyPoolCapacity) + // Create a connection newPool + newPool := pool.NewPool(context.Background(), config.EmptyPoolCapacity) - // Create a proxy with an elastic buffer pool + // Create a proxy with an elastic buffer newPool proxy := NewProxy( context.Background(), - pool, + newPool, plugin.NewRegistry( context.Background(), config.Loose, @@ -136,14 +136,14 @@ func BenchmarkNewProxy(b *testing.B) { NoColor: true, }) - // Create a connection pool - pool := pool.NewPool(context.Background(), config.EmptyPoolCapacity) + // Create a connection newPool + newPool := pool.NewPool(context.Background(), config.EmptyPoolCapacity) - // Create a proxy with a fixed buffer pool + // Create a proxy with a fixed buffer newPool for i := 0; i < b.N; i++ { proxy := NewProxy( context.Background(), - pool, + newPool, plugin.NewRegistry( context.Background(), config.Loose, @@ -172,14 +172,14 @@ func BenchmarkNewProxyElastic(b *testing.B) { NoColor: true, }) - // Create a connection pool - pool := pool.NewPool(context.Background(), config.EmptyPoolCapacity) + // Create a connection newPool + newPool := pool.NewPool(context.Background(), config.EmptyPoolCapacity) - // Create a proxy with an elastic buffer pool + // Create a proxy with an elastic buffer newPool for i := 0; i < b.N; i++ { proxy := NewProxy( context.Background(), - pool, + newPool, plugin.NewRegistry( context.Background(), config.Loose, @@ -216,8 +216,8 @@ func BenchmarkProxyConnectDisconnect(b *testing.B) { NoColor: true, }) - // Create a connection pool - pool := pool.NewPool(context.Background(), 1) + // Create a connection newPool + newPool := pool.NewPool(context.Background(), 1) clientConfig := config.Client{ Network: "tcp", @@ -229,12 +229,12 @@ func BenchmarkProxyConnectDisconnect(b *testing.B) { TCPKeepAlive: false, TCPKeepAlivePeriod: config.DefaultTCPKeepAlivePeriod, } - pool.Put("client", NewClient(context.Background(), &clientConfig, logger)) //nolint:errcheck + newPool.Put("client", NewClient(context.Background(), &clientConfig, logger)) //nolint:errcheck - // Create a proxy with a fixed buffer pool + // Create a proxy with a fixed buffer newPool proxy := NewProxy( context.Background(), - pool, + newPool, plugin.NewRegistry( context.Background(), config.Loose, @@ -270,8 +270,8 @@ func BenchmarkProxyPassThrough(b *testing.B) { NoColor: true, }) - // Create a connection pool - pool := pool.NewPool(context.Background(), 1) + // Create a connection newPool + newPool := pool.NewPool(context.Background(), 1) clientConfig := config.Client{ Network: "tcp", @@ -283,12 +283,12 @@ func BenchmarkProxyPassThrough(b *testing.B) { TCPKeepAlive: false, TCPKeepAlivePeriod: config.DefaultTCPKeepAlivePeriod, } - pool.Put("client", NewClient(context.Background(), &clientConfig, logger)) //nolint:errcheck + newPool.Put("client", NewClient(context.Background(), &clientConfig, logger)) //nolint:errcheck - // Create a proxy with a fixed buffer pool + // Create a proxy with a fixed buffer newPool proxy := NewProxy( context.Background(), - pool, + newPool, plugin.NewRegistry( context.Background(), config.Loose, @@ -326,8 +326,8 @@ func BenchmarkProxyIsHealthyAndIsExhausted(b *testing.B) { NoColor: true, }) - // Create a connection pool - pool := pool.NewPool(context.Background(), 1) + // Create a connection newPool + newPool := pool.NewPool(context.Background(), 1) clientConfig := config.Client{ Network: "tcp", @@ -340,12 +340,12 @@ func BenchmarkProxyIsHealthyAndIsExhausted(b *testing.B) { TCPKeepAlivePeriod: config.DefaultTCPKeepAlivePeriod, } client := NewClient(context.Background(), &clientConfig, logger) - pool.Put("client", client) //nolint:errcheck + newPool.Put("client", client) //nolint:errcheck - // Create a proxy with a fixed buffer pool + // Create a proxy with a fixed buffer newPool proxy := NewProxy( context.Background(), - pool, + newPool, plugin.NewRegistry( context.Background(), config.Loose, @@ -383,8 +383,8 @@ func BenchmarkProxyAvailableAndBusyConnections(b *testing.B) { NoColor: true, }) - // Create a connection pool - pool := pool.NewPool(context.Background(), 1) + // Create a connection newPool + newPool := pool.NewPool(context.Background(), 1) clientConfig := config.Client{ Network: "tcp", @@ -397,12 +397,12 @@ func BenchmarkProxyAvailableAndBusyConnections(b *testing.B) { TCPKeepAlivePeriod: config.DefaultTCPKeepAlivePeriod, } client := NewClient(context.Background(), &clientConfig, logger) - pool.Put("client", client) //nolint:errcheck + newPool.Put("client", client) //nolint:errcheck - // Create a proxy with a fixed buffer pool + // Create a proxy with a fixed buffer newPool proxy := NewProxy( context.Background(), - pool, + newPool, plugin.NewRegistry( context.Background(), config.Loose, diff --git a/network/server_test.go b/network/server_test.go index 3e646ac6..4dbb7d24 100644 --- a/network/server_test.go +++ b/network/server_test.go @@ -2,6 +2,7 @@ package network import ( "bufio" + "bytes" "context" "errors" "io" @@ -23,11 +24,9 @@ import ( // TestRunServer tests an entire server run with a single client connection and hooks. func TestRunServer(t *testing.T) { errs := make(chan error) - defer close(errs) logger := logging.NewLogger(context.Background(), logging.LoggerConfig{ Output: []config.LogOutput{ - config.Console, config.File, }, TimeFormat: zerolog.TimeFormatUnix, @@ -57,9 +56,10 @@ func TestRunServer(t *testing.T) { errs <- errors.New("request is nil") //nolint:goerr113 } - logger.Info().Msg("Ingress traffic") if req, ok := paramsMap["request"].([]byte); ok { - assert.Equal(t, CreatePgStartupPacket(), req) + if !bytes.Equal(req, CreatePgStartupPacket()) { + errs <- errors.New("request does not match") //nolint:goerr113 + } } else { errs <- errors.New("request is not a []byte") //nolint:goerr113 } @@ -81,7 +81,9 @@ func TestRunServer(t *testing.T) { logger.Info().Msg("Ingress traffic") if req, ok := paramsMap["request"].([]byte); ok { - assert.Equal(t, CreatePgStartupPacket(), req) + if !bytes.Equal(req, CreatePgStartupPacket()) { + errs <- errors.New("request does not match") //nolint:goerr113 + } } else { errs <- errors.New("request is not a []byte") //nolint:goerr113 } @@ -144,22 +146,22 @@ func TestRunServer(t *testing.T) { TCPKeepAlivePeriod: config.DefaultTCPKeepAlivePeriod, } - // Create a connection pool. - pool := pool.NewPool(context.Background(), 3) + // Create a connection newPool. + newPool := pool.NewPool(context.Background(), 3) client1 := NewClient(context.Background(), &clientConfig, logger) - err := pool.Put(client1.ID, client1) + err := newPool.Put(client1.ID, client1) assert.Nil(t, err) client2 := NewClient(context.Background(), &clientConfig, logger) - err = pool.Put(client2.ID, client2) + err = newPool.Put(client2.ID, client2) assert.Nil(t, err) client3 := NewClient(context.Background(), &clientConfig, logger) - err = pool.Put(client3.ID, client3) + err = newPool.Put(client3.ID, client3) assert.Nil(t, err) - // Create a proxy with a fixed buffer pool. + // Create a proxy with a fixed buffer newPool. proxy := NewProxy( context.Background(), - pool, + newPool, pluginRegistry, false, false, @@ -175,7 +177,7 @@ func TestRunServer(t *testing.T) { "127.0.0.1:15432", config.DefaultTickInterval, Option{ - EnableTicker: false, + EnableTicker: true, }, proxy, logger, @@ -185,21 +187,25 @@ func TestRunServer(t *testing.T) { assert.NotNil(t, server) stop := make(chan struct{}) - defer close(stop) var waitGroup sync.WaitGroup waitGroup.Add(1) - go func(t *testing.T, server *Server, stop chan struct{}, waitGroup *sync.WaitGroup) { + go func(t *testing.T, server *Server, pluginRegistry *plugin.Registry, stop chan struct{}, waitGroup *sync.WaitGroup) { t.Helper() for { select { case <-stop: + server.Shutdown() + pluginRegistry.Shutdown() + + // Wait for the server to stop. + time.Sleep(100 * time.Millisecond) + // Read the log file and check if the log file contains the expected log messages. if _, err := os.Stat("server_test.log"); err == nil { logFile, err := os.Open("server_test.log") assert.NoError(t, err) - defer logFile.Close() reader := bufio.NewReader(logFile) assert.NotNil(t, reader) @@ -207,28 +213,28 @@ func TestRunServer(t *testing.T) { buffer, err := io.ReadAll(reader) assert.NoError(t, err) assert.Greater(t, len(buffer), 0) // The log file should not be empty. + assert.NoError(t, logFile.Close()) logLines := string(buffer) assert.Contains(t, logLines, "GatewayD is running", "GatewayD should be running") assert.Contains(t, logLines, "GatewayD is ticking...", "GatewayD should be ticking") assert.Contains(t, logLines, "Ingress traffic", "Ingress traffic should be logged") assert.Contains(t, logLines, "Egress traffic", "Egress traffic should be logged") - assert.Contains(t, logLines, "GatewayD is shutting down...", "GatewayD should be shutting down") + assert.Contains(t, logLines, "GatewayD is shutting down", "GatewayD should be shutting down") assert.NoError(t, os.Remove("server_test.log")) - server.Shutdown() } + waitGroup.Done() return - case err := <-errs: + case <-errs: server.Shutdown() - t.Log(err) - t.Fail() + pluginRegistry.Shutdown() waitGroup.Done() return default: //nolint:staticcheck } } - }(t, server, stop, &waitGroup) + }(t, server, pluginRegistry, stop, &waitGroup) waitGroup.Add(1) go func(t *testing.T, server *Server, errs chan error, waitGroup *sync.WaitGroup) { @@ -290,13 +296,13 @@ func TestRunServer(t *testing.T) { CollectAndComparePrometheusMetrics(t) client.Close() - time.Sleep(100 * time.Millisecond) - stop <- struct{}{} - waitGroup.Done() - return + break } time.Sleep(100 * time.Millisecond) } + stop <- struct{}{} + close(stop) + waitGroup.Done() }(t, server, proxy, stop, &waitGroup) waitGroup.Wait() From 020e63f4d0e10fab71d4c9d80c80dcaae869d6c9 Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Fri, 13 Oct 2023 21:46:58 +0200 Subject: [PATCH 16/35] Add Reconnect method to recycle the connection --- network/client.go | 36 ++++++++++++++++++++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/network/client.go b/network/client.go index 9cff7a9d..faf256b5 100644 --- a/network/client.go +++ b/network/client.go @@ -18,6 +18,7 @@ import ( type IClient interface { Send(data []byte) (int, *gerr.GatewayDError) Receive() (int, []byte, *gerr.GatewayDError) + Reconnect() error Close() IsConnected() bool RemoteAddr() string @@ -222,6 +223,41 @@ func (c *Client) Receive() (int, []byte, *gerr.GatewayDError) { return received, buffer.Bytes(), nil } +// Reconnect reconnects to the server. +func (c *Client) Reconnect() error { + _, span := otel.Tracer(config.TracerName).Start(c.ctx, "Reconnect") + defer span.End() + + // Save the current address and network. + address := c.Address + network := c.Network + + if c.Conn != nil { + c.Close() + } + c.connected.Store(false) + + // Restore the address and network. + c.Address = address + c.Network = network + + conn, err := net.Dial(c.Network, c.Address) + if err != nil { + c.logger.Error().Err(err).Msg("Failed to reconnect") + span.RecordError(err) + return gerr.ErrClientConnectionFailed.Wrap(err) + } + + c.Conn = conn + c.ID = GetID( + conn.LocalAddr().Network(), conn.LocalAddr().String(), config.DefaultSeed, c.logger) + c.connected.Store(true) + c.logger.Debug().Str("address", c.Address).Msg("Reconnected to server") + metrics.ServerConnections.Inc() + + return nil +} + // Close closes the connection to the server. func (c *Client) Close() { _, span := otel.Tracer(config.TracerName).Start(c.ctx, "Close") From 22a5d0902814eda1f676272000dc3d37d2cc1121 Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Fri, 13 Oct 2023 21:51:17 +0200 Subject: [PATCH 17/35] Fix connection close issue Use a separate rwmutex for server code --- network/engine.go | 6 ++-- network/proxy.go | 67 ++++++++++++++++++++++++------------------- network/proxy_test.go | 4 +-- network/server.go | 39 ++++++++++++------------- 4 files changed, 60 insertions(+), 56 deletions(-) diff --git a/network/engine.go b/network/engine.go index ec026d02..c7158ab2 100644 --- a/network/engine.go +++ b/network/engine.go @@ -163,7 +163,7 @@ func Run(network, address string, server *Server) *gerr.GatewayDError { stopConnection := make(chan struct{}) go func(server *Server, conn net.Conn, stopConnection chan struct{}) { if action := server.OnTraffic(conn, stopConnection); action == Close { - return + stopConnection <- struct{}{} } }(server, conn, stopConnection) @@ -174,9 +174,7 @@ func Run(network, address string, server *Server) *gerr.GatewayDError { server.engine.mu.Lock() server.engine.connections-- server.engine.mu.Unlock() - if action := server.OnClose(conn, err); action == Close { - return - } + server.OnClose(conn, err) return case <-server.engine.stopServer: return diff --git a/network/proxy.go b/network/proxy.go index 02f2605c..a6ac55b1 100644 --- a/network/proxy.go +++ b/network/proxy.go @@ -25,7 +25,7 @@ type IProxy interface { Disconnect(conn net.Conn) *gerr.GatewayDError PassThroughToServer(conn net.Conn) *gerr.GatewayDError PassThroughToClient(conn net.Conn) *gerr.GatewayDError - IsHealty(cl *Client) (*Client, *gerr.GatewayDError) + IsHealthy(cl *Client) (*Client, *gerr.GatewayDError) IsExhausted() bool Shutdown() AvailableConnections() []string @@ -160,7 +160,7 @@ func (pr *Proxy) Connect(conn net.Conn) *gerr.GatewayDError { } } - client, err := pr.IsHealty(client) + client, err := pr.IsHealthy(client) if err != nil { pr.logger.Error().Err(err).Msg("Failed to connect to the client") span.RecordError(err) @@ -207,34 +207,38 @@ func (pr *Proxy) Disconnect(conn net.Conn) *gerr.GatewayDError { defer span.End() client := pr.busyConnections.Pop(conn) + if client == nil { + // If this ever happens, it means that the client connection + // is pre-empted from the busy connections pool. + pr.logger.Debug().Msg("Client connection is pre-empted from the busy connections pool") + span.RecordError(gerr.ErrClientNotFound) + return gerr.ErrClientNotFound + } + //nolint:nestif - if client != nil { - if client, ok := client.(*Client); ok { - if (pr.Elastic && pr.ReuseElasticClients) || !pr.Elastic { - _, err := pr.IsHealty(client) - if err != nil { - pr.logger.Error().Err(err).Msg("Failed to reconnect to the client") - span.RecordError(err) - } - // If the client is not in the pool, put it back. - err = pr.availableConnections.Put(client.ID, client) - if err != nil { - pr.logger.Error().Err(err).Msg("Failed to put the client back in the pool") - span.RecordError(err) - } - } else { - span.RecordError(gerr.ErrClientNotConnected) - return gerr.ErrClientNotConnected + if client, ok := client.(*Client); ok { + if (pr.Elastic && pr.ReuseElasticClients) || !pr.Elastic { + // Recycle the server connection by reconnecting. + if err := client.Reconnect(); err != nil { + pr.logger.Error().Err(err).Msg("Failed to reconnect to the client") + span.RecordError(err) + } + + // If the client is not in the pool, put it back. + if err := pr.availableConnections.Put(client.ID, client); err != nil { + pr.logger.Error().Err(err).Msg("Failed to put the client back in the pool") + span.RecordError(err) } } else { - // This should never happen, but if it does, - // then there are some serious issues with the pool. - span.RecordError(gerr.ErrCastFailed) - return gerr.ErrCastFailed + span.RecordError(gerr.ErrClientNotConnected) + return gerr.ErrClientNotConnected } } else { - span.RecordError(gerr.ErrClientNotFound) - return gerr.ErrClientNotFound + // This should never happen, but if it does, + // then there are some serious issues with the pool. + pr.logger.Error().Msg("Failed to cast the client to the Client type") + span.RecordError(gerr.ErrCastFailed) + return gerr.ErrCastFailed } metrics.ProxiedConnections.Dec() @@ -255,7 +259,7 @@ func (pr *Proxy) Disconnect(conn net.Conn) *gerr.GatewayDError { return nil } -// PassThrough sends the data from the client to the server. +// PassThroughToServer sends the data from the client to the server. func (pr *Proxy) PassThroughToServer(conn net.Conn) *gerr.GatewayDError { _, span := otel.Tracer(config.TracerName).Start(pr.ctx, "PassThrough") defer span.End() @@ -466,9 +470,9 @@ func (pr *Proxy) PassThroughToClient(conn net.Conn) *gerr.GatewayDError { return errVerdict } -// IsHealty checks if the pool is exhausted or the client is disconnected. -func (pr *Proxy) IsHealty(client *Client) (*Client, *gerr.GatewayDError) { - _, span := otel.Tracer(config.TracerName).Start(pr.ctx, "IsHealty") +// IsHealthy checks if the pool is exhausted or the client is disconnected. +func (pr *Proxy) IsHealthy(client *Client) (*Client, *gerr.GatewayDError) { + _, span := otel.Tracer(config.TracerName).Start(pr.ctx, "IsHealthy") defer span.End() if pr.IsExhausted() { @@ -526,7 +530,10 @@ func (pr *Proxy) Shutdown() { pr.logger.Error().Err(err).Msg("Error setting the deadline") span.RecordError(err) } - conn.Close() + if err := conn.Close(); err != nil { + pr.logger.Error().Err(err).Msg("Failed to close the connection") + span.RecordError(err) + } } if client, ok := value.(*Client); ok { if client != nil { diff --git a/network/proxy_test.go b/network/proxy_test.go index ec072740..8bd82452 100644 --- a/network/proxy_test.go +++ b/network/proxy_test.go @@ -71,7 +71,7 @@ func TestNewProxy(t *testing.T) { assert.Equal(t, false, proxy.Elastic) assert.Equal(t, false, proxy.ReuseElasticClients) assert.Equal(t, false, proxy.IsExhausted()) - c, err := proxy.IsHealty(client) + c, err := proxy.IsHealthy(client) assert.Nil(t, err) assert.Equal(t, client, c) } @@ -369,7 +369,7 @@ func BenchmarkProxyIsHealthyAndIsExhausted(b *testing.B) { // Connect to the proxy for i := 0; i < b.N; i++ { - proxy.IsHealty(client) //nolint:errcheck + proxy.IsHealthy(client) //nolint:errcheck proxy.IsExhausted() } } diff --git a/network/server.go b/network/server.go index 1d85c300..7d4ba358 100644 --- a/network/server.go +++ b/network/server.go @@ -6,6 +6,7 @@ import ( "fmt" "net" "os" + "sync" "time" v1 "github.com/gatewayd-io/gatewayd-plugin-sdk/plugin/v1" @@ -25,6 +26,7 @@ type Server struct { pluginRegistry *plugin.Registry ctx context.Context //nolint:containedctx pluginTimeout time.Duration + mu *sync.RWMutex Network string // tcp/udp/unix Address string @@ -58,9 +60,9 @@ func (s *Server) OnBoot(engine Engine) Action { s.engine = engine // Set the server status to running. - s.engine.mu.Lock() + s.mu.Lock() s.Status = config.Running - s.engine.mu.Unlock() + s.mu.Unlock() // Run the OnBooted hooks. _, err = s.pluginRegistry.Run( @@ -173,11 +175,14 @@ func (s *Server) OnClose(conn net.Conn, err error) Action { span.AddEvent("Ran the OnClosing hooks") // Shutdown the server if there are no more connections and the server is stopped. - // This is used to shutdown the server gracefully. + // This is used to shut down the server gracefully. + s.mu.Lock() if uint64(s.engine.CountConnections()) == 0 && s.Status == config.Stopped { span.AddEvent("Shutting down the server") + s.mu.Unlock() return Shutdown } + s.mu.Unlock() // Disconnect the connection from the proxy. This effectively removes the mapping between // the incoming and the server connections in the pool of the busy connections and either @@ -251,11 +256,7 @@ func (s *Server) OnTraffic(conn net.Conn, stopConnection chan struct{}) Action { if err := server.proxy.PassThroughToServer(conn); err != nil { server.logger.Trace().Err(err).Msg("Failed to pass through traffic") span.RecordError(err) - server.engine.mu.Lock() - if server.Status == config.Stopped { - stopConnection <- struct{}{} - } - server.engine.mu.Unlock() + stopConnection <- struct{}{} break } } @@ -269,17 +270,14 @@ func (s *Server) OnTraffic(conn net.Conn, stopConnection chan struct{}) Action { if err := server.proxy.PassThroughToClient(conn); err != nil { server.logger.Trace().Err(err).Msg("Failed to pass through traffic") span.RecordError(err) - server.engine.mu.Lock() - if server.Status == config.Stopped { - stopConnection <- struct{}{} - } - server.engine.mu.Unlock() + stopConnection <- struct{}{} break } } }(s, conn, stopConnection) - return None + <-stopConnection + return Close } // OnShutdown is called when the server is shutting down. It calls the OnShutdown hooks. @@ -306,9 +304,9 @@ func (s *Server) OnShutdown(Engine) { s.proxy.Shutdown() // Set the server status to stopped. This is used to shutdown the server gracefully in OnClose. - s.engine.mu.Lock() + s.mu.Lock() s.Status = config.Stopped - s.engine.mu.Unlock() + s.mu.Unlock() } // OnTick is called every TickInterval. It calls the OnTick hooks. @@ -402,9 +400,9 @@ func (s *Server) Shutdown() { s.proxy.Shutdown() // Set the server status to stopped. This is used to shutdown the server gracefully in OnClose. - s.engine.mu.Lock() + s.mu.Lock() s.Status = config.Stopped - s.engine.mu.Unlock() + s.mu.Unlock() // Shutdown the server. if err := s.engine.Stop(context.Background()); err != nil { @@ -419,8 +417,8 @@ func (s *Server) IsRunning() bool { defer span.End() span.SetAttributes(attribute.Bool("status", s.Status == config.Running)) - s.engine.mu.Lock() - defer s.engine.mu.Unlock() + s.mu.Lock() + defer s.mu.Unlock() return s.Status == config.Running } @@ -450,6 +448,7 @@ func NewServer( logger: logger, pluginRegistry: pluginRegistry, pluginTimeout: pluginTimeout, + mu: &sync.RWMutex{}, } // Try to resolve the address and log an error if it can't be resolved. From fb9fdab16f983fca05238fe4dbe6553ff0df6946 Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Sun, 15 Oct 2023 13:10:10 +0200 Subject: [PATCH 18/35] Internalize net.Conn in Client struct to prevent direct access to methods Use a mutex to prevent data races to client.conn Reset context to prevent early timeouts Check for field value before logging it --- network/client.go | 56 ++++++++++++++++++++++++++++++----------------- network/proxy.go | 54 +++++++++++++++++++++++---------------------- 2 files changed, 64 insertions(+), 46 deletions(-) diff --git a/network/client.go b/network/client.go index faf256b5..c71e2f38 100644 --- a/network/client.go +++ b/network/client.go @@ -5,6 +5,7 @@ import ( "context" "fmt" "net" + "sync" "sync/atomic" "time" @@ -26,11 +27,11 @@ type IClient interface { } type Client struct { - net.Conn - + conn net.Conn logger zerolog.Logger ctx context.Context //nolint:containedctx connected atomic.Bool + mu sync.Mutex TCPKeepAlive bool TCPKeepAlivePeriod time.Duration @@ -69,6 +70,7 @@ func NewClient(ctx context.Context, clientConfig *config.Client, logger zerolog. // Create a resolved client. client = Client{ ctx: clientCtx, + mu: sync.Mutex{}, Network: clientConfig.Network, Address: addr, } @@ -90,14 +92,14 @@ func NewClient(ctx context.Context, clientConfig *config.Client, logger zerolog. return nil } - client.Conn = conn + client.conn = conn client.connected.Store(true) // Set the TCP keep alive. client.TCPKeepAlive = clientConfig.TCPKeepAlive client.TCPKeepAlivePeriod = clientConfig.TCPKeepAlivePeriod - if c, ok := client.Conn.(*net.TCPConn); ok { + if c, ok := client.conn.(*net.TCPConn); ok { if err := c.SetKeepAlive(client.TCPKeepAlive); err != nil { logger.Error().Err(err).Msg("Failed to set keep alive") span.RecordError(err) @@ -112,7 +114,7 @@ func NewClient(ctx context.Context, clientConfig *config.Client, logger zerolog. // Set the receive deadline (timeout). client.ReceiveDeadline = clientConfig.ReceiveDeadline if client.ReceiveDeadline > 0 { - if err := client.Conn.SetReadDeadline(time.Now().Add(client.ReceiveDeadline)); err != nil { + if err := client.conn.SetReadDeadline(time.Now().Add(client.ReceiveDeadline)); err != nil { logger.Error().Err(err).Msg("Failed to set receive deadline") span.RecordError(err) } else { @@ -124,7 +126,7 @@ func NewClient(ctx context.Context, clientConfig *config.Client, logger zerolog. // Set the send deadline (timeout). client.SendDeadline = clientConfig.SendDeadline if client.SendDeadline > 0 { - if err := client.Conn.SetWriteDeadline(time.Now().Add(client.SendDeadline)); err != nil { + if err := client.conn.SetWriteDeadline(time.Now().Add(client.SendDeadline)); err != nil { logger.Error().Err(err).Msg("Failed to set send deadline") span.RecordError(err) } else { @@ -163,7 +165,7 @@ func (c *Client) Send(data []byte) (int, *gerr.GatewayDError) { break } - written, err := c.Conn.Write(data) + written, err := c.conn.Write(data) if err != nil { c.logger.Error().Err(err).Msg("Couldn't send data to the server") span.RecordError(err) @@ -207,7 +209,7 @@ func (c *Client) Receive() (int, []byte, *gerr.GatewayDError) { // Read the data in chunks. for ctx.Err() == nil { chunk := make([]byte, c.ReceiveChunkSize) - read, err := c.Conn.Read(chunk) + read, err := c.conn.Read(chunk) if err != nil { c.logger.Error().Err(err).Msg("Couldn't receive data from the server") span.RecordError(err) @@ -232,7 +234,7 @@ func (c *Client) Reconnect() error { address := c.Address network := c.Network - if c.Conn != nil { + if c.conn != nil { c.Close() } c.connected.Store(false) @@ -248,7 +250,7 @@ func (c *Client) Reconnect() error { return gerr.ErrClientConnectionFailed.Wrap(err) } - c.Conn = conn + c.conn = conn c.ID = GetID( conn.LocalAddr().Network(), conn.LocalAddr().String(), config.DefaultSeed, c.logger) c.connected.Store(true) @@ -263,21 +265,29 @@ func (c *Client) Close() { _, span := otel.Tracer(config.TracerName).Start(c.ctx, "Close") defer span.End() + c.mu.Lock() + defer c.mu.Unlock() + // Set the deadline to now so that the connection is closed immediately. - if err := c.Conn.SetDeadline(time.Now()); err != nil { + // This will stop all the Conn.Read() and Conn.Write() calls. + // Ref: https://groups.google.com/g/golang-nuts/c/VPVWFrpIEyo + if err := c.conn.SetDeadline(time.Now()); err != nil { c.logger.Error().Err(err).Msg("Failed to set deadline") span.RecordError(err) } + c.connected.Store(false) c.logger.Debug().Str("address", c.Address).Msg("Closing connection to server") - if c.Conn != nil { - c.Conn.Close() + if c.conn != nil { + if err := c.conn.Close(); err != nil { + c.logger.Error().Err(err).Msg("Failed to close connection") + span.RecordError(err) + } } c.ID = "" - c.Conn = nil + c.conn = nil c.Address = "" c.Network = "" - c.connected.Store(false) metrics.ServerConnections.Dec() } @@ -298,7 +308,7 @@ func (c *Client) IsConnected() bool { return false } - if c != nil && c.Conn == nil || c.ID == "" { + if c != nil && c.conn == nil || c.ID == "" { c.logger.Debug().Fields( map[string]interface{}{ "address": c.Address, @@ -316,8 +326,11 @@ func (c *Client) RemoteAddr() string { return "" } - if c.Conn != nil && c.Conn.RemoteAddr() != nil { - return c.Conn.RemoteAddr().String() + c.mu.Lock() + defer c.mu.Unlock() + + if c.conn != nil && c.conn.RemoteAddr() != nil { + return c.conn.RemoteAddr().String() } return "" @@ -329,8 +342,11 @@ func (c *Client) LocalAddr() string { return "" } - if c.Conn != nil && c.Conn.LocalAddr() != nil { - return c.Conn.LocalAddr().String() + c.mu.Lock() + defer c.mu.Unlock() + + if c.conn != nil && c.conn.LocalAddr() != nil { + return c.conn.LocalAddr().String() } return "" diff --git a/network/proxy.go b/network/proxy.go index a6ac55b1..157939de 100644 --- a/network/proxy.go +++ b/network/proxy.go @@ -341,6 +341,9 @@ func (pr *Proxy) PassThroughToServer(conn net.Conn) *gerr.GatewayDError { _, err = pr.sendTrafficToServer(client, request) span.AddEvent("Sent traffic to server") + pluginTimeoutCtx, cancel = context.WithTimeout(context.Background(), pr.pluginTimeout) + defer cancel() + // Run the OnTrafficToServer hooks. _, err = pr.pluginRegistry.Run( pluginTimeoutCtx, @@ -396,12 +399,14 @@ func (pr *Proxy) PassThroughToClient(conn net.Conn) *gerr.GatewayDError { // If the response is empty, don't send anything, instead just close the ingress connection. if received == 0 || err != nil { - pr.logger.Debug().Fields( - map[string]interface{}{ - "function": "proxy.passthrough", - "local": client.LocalAddr(), - "remote": client.RemoteAddr(), - }).Msg("No data to send to client") + fields := map[string]interface{}{"function": "proxy.passthrough"} + if client.LocalAddr() != "" { + fields["local_addr"] = client.LocalAddr() + } + if client.RemoteAddr() != "" { + fields["remote_addr"] = client.RemoteAddr() + } + pr.logger.Debug().Fields(fields).Msg("No data to send to client") span.AddEvent("No data to send to client") span.RecordError(err) return err @@ -442,6 +447,9 @@ func (pr *Proxy) PassThroughToClient(conn net.Conn) *gerr.GatewayDError { span.AddEvent("Sent traffic to client") // Run the OnTrafficToClient hooks. + pluginTimeoutCtx, cancel = context.WithTimeout(context.Background(), pr.pluginTimeout) + defer cancel() + _, err = pr.pluginRegistry.Run( pluginTimeoutCtx, trafficData( @@ -509,12 +517,6 @@ func (pr *Proxy) Shutdown() { pr.availableConnections.ForEach(func(key, value interface{}) bool { if client, ok := value.(*Client); ok { if client.IsConnected() { - // This will stop all the Conn.Read() and Conn.Write() calls. - // Ref: https://groups.google.com/g/golang-nuts/c/VPVWFrpIEyo - if err := client.Conn.SetDeadline(time.Now()); err != nil { - pr.logger.Error().Err(err).Msg("Error setting the deadline") - span.RecordError(err) - } client.Close() } } @@ -537,11 +539,6 @@ func (pr *Proxy) Shutdown() { } if client, ok := value.(*Client); ok { if client != nil { - // This will stop all the Conn.Read() and Conn.Write() calls. - if err := client.Conn.SetDeadline(time.Now()); err != nil { - pr.logger.Error().Err(err).Msg("Error setting the deadline") - span.RecordError(err) - } client.Close() } } @@ -561,7 +558,7 @@ func (pr *Proxy) AvailableConnections() []string { connections := make([]string, 0) pr.availableConnections.ForEach(func(_, value interface{}) bool { if cl, ok := value.(*Client); ok { - connections = append(connections, cl.Conn.LocalAddr().String()) + connections = append(connections, cl.LocalAddr()) } return true }) @@ -668,14 +665,19 @@ func (pr *Proxy) receiveTrafficFromServer(client *Client) (int, []byte, *gerr.Ga // Receive the response from the server. received, response, err := client.Receive() - pr.logger.Debug().Fields( - map[string]interface{}{ - "function": "proxy.passthrough", - "length": received, - "local": client.LocalAddr(), - "remote": client.RemoteAddr(), - }, - ).Msg("Received data from database") + + fields := map[string]interface{}{ + "function": "proxy.passthrough", + "length": received, + } + if client.LocalAddr() != "" { + fields["local"] = client.LocalAddr() + } + if client.RemoteAddr() != "" { + fields["remote"] = client.RemoteAddr() + } + + pr.logger.Debug().Fields(fields).Msg("Received data from database") metrics.BytesReceivedFromServer.Observe(float64(received)) metrics.TotalTrafficBytes.Observe(float64(received)) From e549e586f5a1cd2ca5c8cde587ddd98e92cf06b8 Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Sun, 15 Oct 2023 13:13:34 +0200 Subject: [PATCH 19/35] Limit run time of integration test --- .github/workflows/test.yaml | 1 + 1 file changed, 1 insertion(+) diff --git a/.github/workflows/test.yaml b/.github/workflows/test.yaml index 959f312c..85ce866b 100644 --- a/.github/workflows/test.yaml +++ b/.github/workflows/test.yaml @@ -69,6 +69,7 @@ jobs: name: Test GatewayD Plugins runs-on: ubuntu-latest needs: test + timeout-minutes: 3 services: postgres: image: postgres From f02a31fe0f821f0c0534e50032dfaf9cdc26c459 Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Sun, 15 Oct 2023 13:18:48 +0200 Subject: [PATCH 20/35] Add test for Reconnect Fix tests --- cmd/run_test.go | 2 +- network/client_test.go | 22 ++++++++++++++++++++-- 2 files changed, 21 insertions(+), 3 deletions(-) diff --git a/cmd/run_test.go b/cmd/run_test.go index 0969e6e8..0f73f4c2 100644 --- a/cmd/run_test.go +++ b/cmd/run_test.go @@ -165,7 +165,7 @@ func Test_runCmdWithCachePlugin(t *testing.T) { var waitGroup sync.WaitGroup waitGroup.Add(1) go func(waitGroup *sync.WaitGroup) { - time.Sleep(500 * time.Millisecond) + time.Sleep(time.Second) StopGracefully( runCmd.Context(), diff --git a/network/client_test.go b/network/client_test.go index ed431883..8896415e 100644 --- a/network/client_test.go +++ b/network/client_test.go @@ -49,7 +49,7 @@ func TestNewClient(t *testing.T) { assert.Equal(t, "tcp", client.Network) assert.Equal(t, "127.0.0.1:5432", client.Address) assert.NotEmpty(t, client.ID) - assert.NotNil(t, client.Conn) + assert.NotNil(t, client.conn) } // TestSend tests the Send function. @@ -96,7 +96,7 @@ func TestClose(t *testing.T) { assert.Equal(t, "", client.ID) assert.Equal(t, "", client.Network) assert.Equal(t, "", client.Address) - assert.Nil(t, client.Conn) + assert.Nil(t, client.conn) } // TestIsConnected tests the IsConnected function. @@ -108,6 +108,24 @@ func TestIsConnected(t *testing.T) { assert.False(t, client.IsConnected()) } +func TestReconnect(t *testing.T) { + client := CreateNewClient(t) + defer client.Close() + + assert.NotNil(t, client) + assert.NotNil(t, client.conn) + assert.NotEmpty(t, client.ID) + localAddr := client.LocalAddr() + assert.NotEmpty(t, localAddr) + + assert.NoError(t, client.Reconnect()) + assert.NotNil(t, client) + assert.NotNil(t, client.conn) + assert.NotEmpty(t, client.ID) + assert.NotEmpty(t, client.LocalAddr()) + assert.NotEqual(t, localAddr, client.LocalAddr()) // This is a new connection. +} + func BenchmarkNewClient(b *testing.B) { cfg := logging.LoggerConfig{ Output: []config.LogOutput{config.Console}, From 8013cb933e9eb307c4c1bee462252f3a59b5a52b Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Sun, 15 Oct 2023 13:56:01 +0200 Subject: [PATCH 21/35] Fix metrics --- network/client.go | 2 ++ network/server_test.go | 4 ++++ 2 files changed, 6 insertions(+) diff --git a/network/client.go b/network/client.go index c71e2f38..27c119a4 100644 --- a/network/client.go +++ b/network/client.go @@ -236,6 +236,8 @@ func (c *Client) Reconnect() error { if c.conn != nil { c.Close() + } else { + metrics.ServerConnections.Dec() } c.connected.Store(false) diff --git a/network/server_test.go b/network/server_test.go index 4dbb7d24..9b0643c2 100644 --- a/network/server_test.go +++ b/network/server_test.go @@ -16,6 +16,7 @@ import ( "github.com/gatewayd-io/gatewayd/logging" "github.com/gatewayd-io/gatewayd/plugin" "github.com/gatewayd-io/gatewayd/pool" + "github.com/prometheus/client_golang/prometheus" "github.com/rs/zerolog" "github.com/stretchr/testify/assert" "google.golang.org/grpc" @@ -25,6 +26,9 @@ import ( func TestRunServer(t *testing.T) { errs := make(chan error) + // Reset prometheus metrics. + prometheus.DefaultRegisterer = prometheus.NewRegistry() + logger := logging.NewLogger(context.Background(), logging.LoggerConfig{ Output: []config.LogOutput{ config.File, From 3e734cb3067e652c3545729fda0487d9a1ecb8a3 Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Sun, 15 Oct 2023 14:12:41 +0200 Subject: [PATCH 22/35] Remove unused variable --- network/engine.go | 6 +++--- network/server.go | 2 +- plugin/plugin_registry.go | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/network/engine.go b/network/engine.go index c7158ab2..05002238 100644 --- a/network/engine.go +++ b/network/engine.go @@ -100,7 +100,7 @@ func Run(network, address string, server *Server) *gerr.GatewayDError { go func(server *Server) { <-server.engine.stopServer - server.OnShutdown(server.engine) + server.OnShutdown() server.logger.Debug().Msg("Server stopped") }(server) @@ -116,7 +116,7 @@ func Run(network, address string, server *Server) *gerr.GatewayDError { default: interval, action := server.OnTick() if action == Shutdown { - server.OnShutdown(server.engine) + server.OnShutdown() return } if interval == time.Duration(0) { @@ -150,7 +150,7 @@ func Run(network, address string, server *Server) *gerr.GatewayDError { } conn.Close() if action == Shutdown { - server.OnShutdown(server.engine) + server.OnShutdown() return nil } } diff --git a/network/server.go b/network/server.go index 7d4ba358..38b36eaa 100644 --- a/network/server.go +++ b/network/server.go @@ -281,7 +281,7 @@ func (s *Server) OnTraffic(conn net.Conn, stopConnection chan struct{}) Action { } // OnShutdown is called when the server is shutting down. It calls the OnShutdown hooks. -func (s *Server) OnShutdown(Engine) { +func (s *Server) OnShutdown() { _, span := otel.Tracer("gatewayd").Start(s.ctx, "OnShutdown") defer span.End() diff --git a/plugin/plugin_registry.go b/plugin/plugin_registry.go index e8d0d993..1a72116f 100644 --- a/plugin/plugin_registry.go +++ b/plugin/plugin_registry.go @@ -6,7 +6,7 @@ import ( "encoding/hex" "sort" - semver "github.com/Masterminds/semver/v3" + "github.com/Masterminds/semver/v3" sdkPlugin "github.com/gatewayd-io/gatewayd-plugin-sdk/plugin" v1 "github.com/gatewayd-io/gatewayd-plugin-sdk/plugin/v1" "github.com/gatewayd-io/gatewayd/config" From e13389a71eee2d5807a6e8303314b531fce6c77d Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Sun, 15 Oct 2023 15:21:59 +0200 Subject: [PATCH 23/35] Fix connection hanging --- network/proxy.go | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/network/proxy.go b/network/proxy.go index 157939de..2cf608a8 100644 --- a/network/proxy.go +++ b/network/proxy.go @@ -264,8 +264,8 @@ func (pr *Proxy) PassThroughToServer(conn net.Conn) *gerr.GatewayDError { _, span := otel.Tracer(config.TracerName).Start(pr.ctx, "PassThrough") defer span.End() - // Check if the proxy has a egress client for the incoming connection. var client *Client + // Check if the proxy has a egress client for the incoming connection. if pr.busyConnections.Get(conn) == nil { span.RecordError(gerr.ErrClientNotFound) return gerr.ErrClientNotFound @@ -280,7 +280,7 @@ func (pr *Proxy) PassThroughToServer(conn net.Conn) *gerr.GatewayDError { } span.AddEvent("Got the client from the busy connection pool") - if !client.IsConnected() || !pr.isConnectionHealthy(conn) { + if !client.IsConnected() { return gerr.ErrClientNotConnected } @@ -375,6 +375,7 @@ func (pr *Proxy) PassThroughToClient(conn net.Conn) *gerr.GatewayDError { defer span.End() var client *Client + // Check if the proxy has a egress client for the incoming connection. if pr.busyConnections.Get(conn) == nil { span.RecordError(gerr.ErrClientNotFound) return gerr.ErrClientNotFound @@ -389,7 +390,7 @@ func (pr *Proxy) PassThroughToClient(conn net.Conn) *gerr.GatewayDError { } span.AddEvent("Got the client from the busy connection pool") - if !client.IsConnected() || !pr.isConnectionHealthy(conn) { + if !client.IsConnected() { return gerr.ErrClientNotConnected } From f2017364fe442207e6df4c09393fc326e593072e Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Sun, 15 Oct 2023 16:10:09 +0200 Subject: [PATCH 24/35] Check if the value is nil before putting it into the pool --- errors/errors.go | 3 +++ pool/pool.go | 12 ++++++++++++ 2 files changed, 15 insertions(+) diff --git a/errors/errors.go b/errors/errors.go index a8799bbc..28c697d5 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -27,6 +27,7 @@ const ( ErrCodeAcceptFailed ErrCodeReadFailed ErrCodePutFailed + ErrCodeNilPointer ErrCodeCastFailed ErrCodeHookVerificationFailed ErrCodeHookReturnedError @@ -96,6 +97,8 @@ var ( ErrPutFailed = NewGatewayDError( ErrCodePutFailed, "failed to put in pool", nil) + ErrNilPointer = NewGatewayDError( + ErrCodeNilPointer, "nil pointer", nil) ErrCastFailed = NewGatewayDError( ErrCodeCastFailed, "failed to cast", nil) diff --git a/pool/pool.go b/pool/pool.go index 88eab31a..e902bde6 100644 --- a/pool/pool.go +++ b/pool/pool.go @@ -57,6 +57,12 @@ func (p *Pool) Put(key, value interface{}) *gerr.GatewayDError { span.RecordError(gerr.ErrPoolExhausted) return gerr.ErrPoolExhausted } + + if value == nil { + span.RecordError(gerr.ErrNilPointer) + return gerr.ErrNilPointer + } + p.pool.Store(key, value) return nil } @@ -80,6 +86,12 @@ func (p *Pool) GetOrPut(key, value interface{}) (interface{}, bool, *gerr.Gatewa span.RecordError(gerr.ErrPoolExhausted) return nil, false, gerr.ErrPoolExhausted } + + if value == nil { + span.RecordError(gerr.ErrNilPointer) + return nil, false, gerr.ErrNilPointer + } + val, loaded := p.pool.LoadOrStore(key, value) return val, loaded, nil } From fa790c6fda76cfdc4b40e4712a947cd6da6ac1d2 Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Sun, 15 Oct 2023 16:16:30 +0200 Subject: [PATCH 25/35] Check if connection exists before trying to close it --- network/client.go | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/network/client.go b/network/client.go index 27c119a4..05408ff4 100644 --- a/network/client.go +++ b/network/client.go @@ -273,9 +273,11 @@ func (c *Client) Close() { // Set the deadline to now so that the connection is closed immediately. // This will stop all the Conn.Read() and Conn.Write() calls. // Ref: https://groups.google.com/g/golang-nuts/c/VPVWFrpIEyo - if err := c.conn.SetDeadline(time.Now()); err != nil { - c.logger.Error().Err(err).Msg("Failed to set deadline") - span.RecordError(err) + if c.conn != nil { + if err := c.conn.SetDeadline(time.Now()); err != nil { + c.logger.Error().Err(err).Msg("Failed to set deadline") + span.RecordError(err) + } } c.connected.Store(false) From 068f03c555f149b69150017ca33995b9720c6fcf Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Sun, 15 Oct 2023 16:29:21 +0200 Subject: [PATCH 26/35] Close channel before returning --- network/engine.go | 1 + 1 file changed, 1 insertion(+) diff --git a/network/engine.go b/network/engine.go index 05002238..a1820b79 100644 --- a/network/engine.go +++ b/network/engine.go @@ -54,6 +54,7 @@ func (engine *Engine) Stop(ctx context.Context) error { engine.running.Store(false) if err := engine.listener.Close(); err != nil { engine.stopServer <- struct{}{} + close(engine.stopServer) return gerr.ErrCloseListenerFailed.Wrap(err) } engine.stopServer <- struct{}{} From 91998ea96b436114c91c571568a0f8a530ceed4f Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Sun, 15 Oct 2023 16:36:48 +0200 Subject: [PATCH 27/35] Move Run function to server.Run Check if listener is not nil before closing it --- errors/errors.go | 6 -- network/engine.go | 138 +++------------------------------------------- network/server.go | 129 ++++++++++++++++++++++++++++++++++++++++--- 3 files changed, 130 insertions(+), 143 deletions(-) diff --git a/errors/errors.go b/errors/errors.go index 28c697d5..e5b1c1a5 100644 --- a/errors/errors.go +++ b/errors/errors.go @@ -9,7 +9,6 @@ const ( ErrCodeNetworkNotSupported ErrCodeResolveFailed ErrCodePoolExhausted - ErrCodeStartServerFailed ErrCodePluginNotFound ErrCodePluginNotReady ErrCodeStartPluginFailed @@ -23,7 +22,6 @@ const ( ErrCodeServerSendFailed ErrCodeServerListenFailed ErrCodeSplitHostPortFailed - ErrCodeCloseListenerFailed ErrCodeAcceptFailed ErrCodeReadFailed ErrCodePutFailed @@ -56,8 +54,6 @@ var ( ErrCodeResolveFailed, "failed to resolve address", nil) ErrPoolExhausted = NewGatewayDError( ErrCodePoolExhausted, "pool is exhausted", nil) - ErrFailedToStartServer = NewGatewayDError( - ErrCodeStartServerFailed, "failed to start server", nil) ErrPluginNotFound = NewGatewayDError( ErrCodePluginNotFound, "plugin not found", nil) @@ -87,8 +83,6 @@ var ( ErrCodeServerListenFailed, "couldn't listen on the server", nil) ErrSplitHostPortFailed = NewGatewayDError( ErrCodeSplitHostPortFailed, "failed to split host:port", nil) - ErrCloseListenerFailed = NewGatewayDError( - ErrCodeCloseListenerFailed, "failed to close listener", nil) ErrAcceptFailed = NewGatewayDError( ErrCodeAcceptFailed, "failed to accept connection", nil) diff --git a/network/engine.go b/network/engine.go index a1820b79..2f552ed3 100644 --- a/network/engine.go +++ b/network/engine.go @@ -3,13 +3,12 @@ package network import ( "context" "net" - "strconv" "sync" "sync/atomic" "time" "github.com/gatewayd-io/gatewayd/config" - gerr "github.com/gatewayd-io/gatewayd/errors" + "github.com/rs/zerolog" ) type Option struct { @@ -36,6 +35,7 @@ type Engine struct { host string port int connections uint32 + logger zerolog.Logger running *atomic.Bool stopServer chan struct{} mu *sync.RWMutex @@ -52,136 +52,14 @@ func (engine *Engine) Stop(ctx context.Context) error { defer cancel() engine.running.Store(false) - if err := engine.listener.Close(); err != nil { - engine.stopServer <- struct{}{} - close(engine.stopServer) - return gerr.ErrCloseListenerFailed.Wrap(err) + if engine.listener != nil { + if err := engine.listener.Close(); err != nil { + engine.logger.Error().Err(err).Msg("Failed to close listener") + } + } else { + engine.logger.Error().Msg("Listener is not initialized") } engine.stopServer <- struct{}{} close(engine.stopServer) return nil } - -// Run starts a server and connects all the handlers. -func Run(network, address string, server *Server) *gerr.GatewayDError { - server.engine = Engine{ - connections: 0, - stopServer: make(chan struct{}), - mu: &sync.RWMutex{}, - running: &atomic.Bool{}, - } - - if action := server.OnBoot(server.engine); action != None { - return nil - } - - var err error - server.engine.listener, err = net.Listen(network, address) - if err != nil { - server.logger.Error().Err(err).Msg("Server failed to start listening") - return gerr.ErrServerListenFailed.Wrap(err) - } - - if server.engine.listener == nil { - server.logger.Error().Msg("Server is not properly initialized") - return nil - } - - var port string - server.engine.host, port, err = net.SplitHostPort(server.engine.listener.Addr().String()) - if err != nil { - server.logger.Error().Err(err).Msg("Failed to split host and port") - return gerr.ErrSplitHostPortFailed.Wrap(err) - } - - if server.engine.port, err = strconv.Atoi(port); err != nil { - server.logger.Error().Err(err).Msg("Failed to convert port to integer") - return gerr.ErrCastFailed.Wrap(err) - } - - go func(server *Server) { - <-server.engine.stopServer - server.OnShutdown() - server.logger.Debug().Msg("Server stopped") - }(server) - - go func(server *Server) { - if !server.Options.EnableTicker { - return - } - - for { - select { - case <-server.engine.stopServer: - return - default: - interval, action := server.OnTick() - if action == Shutdown { - server.OnShutdown() - return - } - if interval == time.Duration(0) { - return - } - time.Sleep(interval) - } - } - }(server) - - server.engine.running.Store(true) - - for { - select { - case <-server.engine.stopServer: - server.logger.Info().Msg("Server stopped") - return nil - default: - conn, err := server.engine.listener.Accept() - if err != nil { - if !server.engine.running.Load() { - return nil - } - server.logger.Error().Err(err).Msg("Failed to accept connection") - return gerr.ErrAcceptFailed.Wrap(err) - } - - if out, action := server.OnOpen(conn); action != None { - if _, err := conn.Write(out); err != nil { - server.logger.Error().Err(err).Msg("Failed to write to connection") - } - conn.Close() - if action == Shutdown { - server.OnShutdown() - return nil - } - } - server.engine.mu.Lock() - server.engine.connections++ - server.engine.mu.Unlock() - - // For every new connection, a new unbuffered channel is created to help - // stop the proxy, recycle the server connection and close stale connections. - stopConnection := make(chan struct{}) - go func(server *Server, conn net.Conn, stopConnection chan struct{}) { - if action := server.OnTraffic(conn, stopConnection); action == Close { - stopConnection <- struct{}{} - } - }(server, conn, stopConnection) - - go func(server *Server, conn net.Conn, stopConnection chan struct{}) { - for { - select { - case <-stopConnection: - server.engine.mu.Lock() - server.engine.connections-- - server.engine.mu.Unlock() - server.OnClose(conn, err) - return - case <-server.engine.stopServer: - return - } - } - }(server, conn, stopConnection) - } - } -} diff --git a/network/server.go b/network/server.go index 38b36eaa..ae9fd48a 100644 --- a/network/server.go +++ b/network/server.go @@ -6,7 +6,9 @@ import ( "fmt" "net" "os" + "strconv" "sync" + "sync/atomic" "time" v1 "github.com/gatewayd-io/gatewayd-plugin-sdk/plugin/v1" @@ -341,7 +343,7 @@ func (s *Server) OnTick() (time.Duration, Action) { } // Run starts the server and blocks until the server is stopped. It calls the OnRun hooks. -func (s *Server) Run() error { +func (s *Server) Run() *gerr.GatewayDError { _, span := otel.Tracer("gatewayd").Start(s.ctx, "Run") defer span.End() @@ -381,14 +383,127 @@ func (s *Server) Run() error { } // Start the server. - origErr := Run(s.Network, addr, s) - if origErr != nil && origErr.Unwrap() != nil { - s.logger.Error().Err(origErr).Msg("Failed to start server") - span.RecordError(origErr) - return gerr.ErrFailedToStartServer.Wrap(origErr) + s.engine = Engine{ + connections: 0, + logger: s.logger, + stopServer: make(chan struct{}), + mu: &sync.RWMutex{}, + running: &atomic.Bool{}, } - return nil + if action := s.OnBoot(s.engine); action != None { + return nil + } + + listener, origErr := net.Listen(s.Network, addr) + if origErr != nil { + s.logger.Error().Err(origErr).Msg("Server failed to start listening") + return gerr.ErrServerListenFailed.Wrap(origErr) + } + s.engine.listener = listener + + if s.engine.listener == nil { + s.logger.Error().Msg("Server is not properly initialized") + return nil + } + + var port string + s.engine.host, port, origErr = net.SplitHostPort(s.engine.listener.Addr().String()) + if origErr != nil { + s.logger.Error().Err(origErr).Msg("Failed to split host and port") + return gerr.ErrSplitHostPortFailed.Wrap(origErr) + } + + if s.engine.port, origErr = strconv.Atoi(port); origErr != nil { + s.logger.Error().Err(origErr).Msg("Failed to convert port to integer") + return gerr.ErrCastFailed.Wrap(origErr) + } + + go func(server *Server) { + <-server.engine.stopServer + server.OnShutdown() + server.logger.Debug().Msg("Server stopped") + }(s) + + go func(server *Server) { + if !server.Options.EnableTicker { + return + } + + for { + select { + case <-server.engine.stopServer: + return + default: + interval, action := server.OnTick() + if action == Shutdown { + server.OnShutdown() + return + } + if interval == time.Duration(0) { + return + } + time.Sleep(interval) + } + } + }(s) + + s.engine.running.Store(true) + + for { + select { + case <-s.engine.stopServer: + s.logger.Info().Msg("Server stopped") + return nil + default: + conn, err := s.engine.listener.Accept() + if err != nil { + if !s.engine.running.Load() { + return nil + } + s.logger.Error().Err(err).Msg("Failed to accept connection") + return gerr.ErrAcceptFailed.Wrap(err) + } + + if out, action := s.OnOpen(conn); action != None { + if _, err := conn.Write(out); err != nil { + s.logger.Error().Err(err).Msg("Failed to write to connection") + } + conn.Close() + if action == Shutdown { + s.OnShutdown() + return nil + } + } + s.engine.mu.Lock() + s.engine.connections++ + s.engine.mu.Unlock() + + // For every new connection, a new unbuffered channel is created to help + // stop the proxy, recycle the server connection and close stale connections. + stopConnection := make(chan struct{}) + go func(server *Server, conn net.Conn, stopConnection chan struct{}) { + if action := server.OnTraffic(conn, stopConnection); action == Close { + stopConnection <- struct{}{} + } + }(s, conn, stopConnection) + + go func(server *Server, conn net.Conn, stopConnection chan struct{}) { + for { + select { + case <-stopConnection: + server.engine.mu.Lock() + server.engine.connections-- + server.engine.mu.Unlock() + server.OnClose(conn, err) + return + case <-server.engine.stopServer: + return + } + } + }(s, conn, stopConnection) + } + } } // Shutdown stops the server. From f1c70bfeb990e4efd9dd589bc9a420947cf13234 Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Sun, 15 Oct 2023 17:54:51 +0200 Subject: [PATCH 28/35] Add a NewEngine function to create a new instance of the Engine struct --- network/engine.go | 39 ++++++++++++++++++--------------------- network/server.go | 23 +++++++++++++---------- 2 files changed, 31 insertions(+), 31 deletions(-) diff --git a/network/engine.go b/network/engine.go index 2f552ed3..093754de 100644 --- a/network/engine.go +++ b/network/engine.go @@ -11,25 +11,8 @@ import ( "github.com/rs/zerolog" ) -type Option struct { - EnableTicker bool -} - -type Action int - -const ( - None Action = iota - Close - Shutdown -) - -type TCPSocketOpt int - -const ( - TCPNoDelay TCPSocketOpt = iota - TCPDelay -) - +// Engine is the network engine. +// TODO: Move this to the Server struct. type Engine struct { listener net.Listener host string @@ -41,19 +24,22 @@ type Engine struct { mu *sync.RWMutex } +// CountConnections returns the current number of connections. func (engine *Engine) CountConnections() int { engine.mu.RLock() defer engine.mu.RUnlock() return int(engine.connections) } +// Stop stops the engine. func (engine *Engine) Stop(ctx context.Context) error { _, cancel := context.WithDeadline(ctx, time.Now().Add(config.DefaultEngineStopTimeout)) defer cancel() + var err error engine.running.Store(false) if engine.listener != nil { - if err := engine.listener.Close(); err != nil { + if err = engine.listener.Close(); err != nil { engine.logger.Error().Err(err).Msg("Failed to close listener") } } else { @@ -61,5 +47,16 @@ func (engine *Engine) Stop(ctx context.Context) error { } engine.stopServer <- struct{}{} close(engine.stopServer) - return nil + return err +} + +// NewEngine creates a new engine. +func NewEngine(logger zerolog.Logger) Engine { + return Engine{ + connections: 0, + logger: logger, + running: &atomic.Bool{}, + stopServer: make(chan struct{}), + mu: &sync.RWMutex{}, + } } diff --git a/network/server.go b/network/server.go index ae9fd48a..14ed58ed 100644 --- a/network/server.go +++ b/network/server.go @@ -8,7 +8,6 @@ import ( "os" "strconv" "sync" - "sync/atomic" "time" v1 "github.com/gatewayd-io/gatewayd-plugin-sdk/plugin/v1" @@ -21,6 +20,18 @@ import ( "go.opentelemetry.io/otel/attribute" ) +type Option struct { + EnableTicker bool +} + +type Action int + +const ( + None Action = iota + Close + Shutdown +) + type Server struct { engine Engine proxy IProxy @@ -382,15 +393,6 @@ func (s *Server) Run() *gerr.GatewayDError { } } - // Start the server. - s.engine = Engine{ - connections: 0, - logger: s.logger, - stopServer: make(chan struct{}), - mu: &sync.RWMutex{}, - running: &atomic.Bool{}, - } - if action := s.OnBoot(s.engine); action != None { return nil } @@ -564,6 +566,7 @@ func NewServer( pluginRegistry: pluginRegistry, pluginTimeout: pluginTimeout, mu: &sync.RWMutex{}, + engine: NewEngine(logger), } // Try to resolve the address and log an error if it can't be resolved. From f377c83b70f26a741604c343543fab3415cb1c00 Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Sun, 15 Oct 2023 17:55:07 +0200 Subject: [PATCH 29/35] Add test for Engine --- network/engine.go | 2 +- network/engine_test.go | 51 ++++++++++++++++++++++++++++++++++++++++++ 2 files changed, 52 insertions(+), 1 deletion(-) create mode 100644 network/engine_test.go diff --git a/network/engine.go b/network/engine.go index 093754de..be2a5c5e 100644 --- a/network/engine.go +++ b/network/engine.go @@ -47,7 +47,7 @@ func (engine *Engine) Stop(ctx context.Context) error { } engine.stopServer <- struct{}{} close(engine.stopServer) - return err + return err //nolint:wrapcheck } // NewEngine creates a new engine. diff --git a/network/engine_test.go b/network/engine_test.go new file mode 100644 index 00000000..c985a1fd --- /dev/null +++ b/network/engine_test.go @@ -0,0 +1,51 @@ +package network + +import ( + "context" + "testing" + "time" + + "github.com/gatewayd-io/gatewayd/config" + "github.com/gatewayd-io/gatewayd/logging" + "github.com/rs/zerolog" + "github.com/stretchr/testify/assert" + "github.com/zenizh/go-capturer" +) + +func TestEngine(t *testing.T) { + output := capturer.CaptureOutput(func() { + logger := logging.NewLogger(context.Background(), logging.LoggerConfig{ + Output: []config.LogOutput{config.Console}, + TimeFormat: zerolog.TimeFormatUnix, + ConsoleTimeFormat: time.RFC3339, + Level: zerolog.WarnLevel, + NoColor: true, + }) + engine := NewEngine(logger) + assert.NotNil(t, engine) + assert.NotNil(t, engine.logger) + assert.Zero(t, engine.connections) + assert.Zero(t, engine.CountConnections()) + assert.Empty(t, engine.host) + assert.Empty(t, engine.port) + assert.False(t, engine.running.Load()) + + go func(engine Engine) { + v := <-engine.stopServer + // Empty struct is expected to be received and + // it means that the server is stopped. + assert.Equal(t, v, struct{}{}) + }(engine) + + err := engine.Stop(context.Background()) + // This is expected to fail because the engine is not running. + assert.Nil(t, err) + assert.False(t, engine.running.Load()) + assert.Zero(t, engine.connections) + }) + + t.Log(output) + + // Expected output: + assert.Contains(t, output, "ERR Listener is not initialized") +} From 91efb669c4990762062939352dc8753d314f7d58 Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Sun, 15 Oct 2023 18:30:05 +0200 Subject: [PATCH 30/35] Use background context instead of Cmd.Context to prevent races --- cmd/run_test.go | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/cmd/run_test.go b/cmd/run_test.go index 0f73f4c2..cb865636 100644 --- a/cmd/run_test.go +++ b/cmd/run_test.go @@ -1,6 +1,7 @@ package cmd import ( + "context" "os" "sync" "testing" @@ -29,8 +30,8 @@ func Test_runCmd(t *testing.T) { time.Sleep(100 * time.Millisecond) StopGracefully( - runCmd.Context(), - runCmd.Context(), + context.Background(), + context.Background(), nil, nil, nil, @@ -88,8 +89,8 @@ func Test_runCmdWithMultiTenancy(t *testing.T) { time.Sleep(500 * time.Millisecond) StopGracefully( - runCmd.Context(), - runCmd.Context(), + context.Background(), + context.Background(), nil, nil, nil, @@ -168,8 +169,8 @@ func Test_runCmdWithCachePlugin(t *testing.T) { time.Sleep(time.Second) StopGracefully( - runCmd.Context(), - runCmd.Context(), + context.Background(), + context.Background(), nil, nil, nil, From ac85264606737358e58adde7174ccd9e0148079b Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Sun, 15 Oct 2023 18:30:43 +0200 Subject: [PATCH 31/35] Close listener after use Add new make target for finding race conditions in tests --- Makefile | 3 +++ network/server.go | 1 + 2 files changed, 4 insertions(+) diff --git a/Makefile b/Makefile index b66c1e09..1e1fb437 100644 --- a/Makefile +++ b/Makefile @@ -110,6 +110,9 @@ clean: test: @go test -v ./... +test-race: + @go test -race -v ./... + benchmark: @go test -bench=. -benchmem -run=^# ./... diff --git a/network/server.go b/network/server.go index 14ed58ed..d95d5f8c 100644 --- a/network/server.go +++ b/network/server.go @@ -403,6 +403,7 @@ func (s *Server) Run() *gerr.GatewayDError { return gerr.ErrServerListenFailed.Wrap(origErr) } s.engine.listener = listener + defer s.engine.listener.Close() if s.engine.listener == nil { s.logger.Error().Msg("Server is not properly initialized") From 909a23cb85b323541d59670b42a493010a2e619b Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Sun, 15 Oct 2023 18:43:14 +0200 Subject: [PATCH 32/35] Don't send on a closed channel --- network/engine.go | 13 ++++++++++--- network/engine_test.go | 2 -- 2 files changed, 10 insertions(+), 5 deletions(-) diff --git a/network/engine.go b/network/engine.go index be2a5c5e..6f8192ee 100644 --- a/network/engine.go +++ b/network/engine.go @@ -45,9 +45,16 @@ func (engine *Engine) Stop(ctx context.Context) error { } else { engine.logger.Error().Msg("Listener is not initialized") } - engine.stopServer <- struct{}{} - close(engine.stopServer) - return err //nolint:wrapcheck + + select { + case <-engine.stopServer: + engine.logger.Info().Msg("Server stopped") + return err //nolint:wrapcheck + default: + engine.stopServer <- struct{}{} + close(engine.stopServer) + return err //nolint:wrapcheck + } } // NewEngine creates a new engine. diff --git a/network/engine_test.go b/network/engine_test.go index c985a1fd..a9ee7e2e 100644 --- a/network/engine_test.go +++ b/network/engine_test.go @@ -44,8 +44,6 @@ func TestEngine(t *testing.T) { assert.Zero(t, engine.connections) }) - t.Log(output) - // Expected output: assert.Contains(t, output, "ERR Listener is not initialized") } From b009dc123f7349eb6f07afa423dbde04717ec321 Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Sun, 15 Oct 2023 22:54:40 +0200 Subject: [PATCH 33/35] Renew timeout of plugin context --- cmd/run.go | 29 ++++++++++++++++++++++++----- cmd/run_test.go | 3 --- network/server.go | 9 +++++++++ 3 files changed, 33 insertions(+), 8 deletions(-) diff --git a/cmd/run.go b/cmd/run.go index 03653c00..aeb3504c 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -70,7 +70,6 @@ var ( func StopGracefully( runCtx context.Context, - pluginTimeoutCtx context.Context, sig os.Signal, metricsMerger *metrics.Merger, metricsServer *http.Server, @@ -87,6 +86,9 @@ func StopGracefully( logger.Info().Msg("Notifying the plugins that the server is shutting down") if pluginRegistry != nil { + pluginTimeoutCtx, cancel := context.WithTimeout(context.Background(), conf.Plugin.Timeout) + defer cancel() + _, err := pluginRegistry.Run( pluginTimeoutCtx, map[string]interface{}{"signal": signal}, @@ -461,6 +463,9 @@ var runCmd = &cobra.Command{ }(conf.Global.Metrics[config.Default], logger) // This is a notification hook, so we don't care about the result. + pluginTimeoutCtx, cancel = context.WithTimeout(context.Background(), conf.Plugin.Timeout) + defer cancel() + if data, ok := conf.GlobalKoanf.Get("loggers").(map[string]interface{}); ok { _, err = pluginRegistry.Run( pluginTimeoutCtx, data, v1.HookName_HOOK_NAME_ON_NEW_LOGGER) @@ -527,6 +532,10 @@ var runCmd = &cobra.Command{ span.AddEvent("Create client", eventOptions) + pluginTimeoutCtx, cancel = context.WithTimeout( + context.Background(), conf.Plugin.Timeout) + defer cancel() + clientCfg := map[string]interface{}{ "id": client.ID, "network": client.Network, @@ -571,6 +580,10 @@ var runCmd = &cobra.Command{ os.Exit(gerr.FailedToInitializePool) } + pluginTimeoutCtx, cancel = context.WithTimeout( + context.Background(), conf.Plugin.Timeout) + defer cancel() + _, err = pluginRegistry.Run( pluginTimeoutCtx, map[string]interface{}{"name": name, "size": cfg.GetSize()}, @@ -610,6 +623,10 @@ var runCmd = &cobra.Command{ attribute.String("healthCheckPeriod", cfg.HealthCheckPeriod.String()), )) + pluginTimeoutCtx, cancel = context.WithTimeout( + context.Background(), conf.Plugin.Timeout) + defer cancel() + if data, ok := conf.GlobalKoanf.Get("proxies").(map[string]interface{}); ok { _, err = pluginRegistry.Run( pluginTimeoutCtx, data, v1.HookName_HOOK_NAME_ON_NEW_PROXY) @@ -651,6 +668,10 @@ var runCmd = &cobra.Command{ attribute.String("pluginTimeout", conf.Plugin.Timeout.String()), )) + pluginTimeoutCtx, cancel = context.WithTimeout( + context.Background(), conf.Plugin.Timeout) + defer cancel() + if data, ok := conf.GlobalKoanf.Get("servers").(map[string]interface{}); ok { _, err = pluginRegistry.Run( pluginTimeoutCtx, data, v1.HookName_HOOK_NAME_ON_NEW_SERVER) @@ -750,8 +771,7 @@ var runCmd = &cobra.Command{ ) signalsCh := make(chan os.Signal, 1) signal.Notify(signalsCh, signals...) - go func(pluginTimeoutCtx context.Context, - pluginRegistry *plugin.Registry, + go func(pluginRegistry *plugin.Registry, logger zerolog.Logger, servers map[string]*network.Server, metricsMerger *metrics.Merger, @@ -763,7 +783,6 @@ var runCmd = &cobra.Command{ if sig != s { StopGracefully( runCtx, - pluginTimeoutCtx, sig, metricsMerger, metricsServer, @@ -776,7 +795,7 @@ var runCmd = &cobra.Command{ } } } - }(pluginTimeoutCtx, pluginRegistry, logger, servers, metricsMerger, metricsServer, stopChan) + }(pluginRegistry, logger, servers, metricsMerger, metricsServer, stopChan) _, span = otel.Tracer(config.TracerName).Start(runCtx, "Start servers") // Start the server. diff --git a/cmd/run_test.go b/cmd/run_test.go index cb865636..40761ca6 100644 --- a/cmd/run_test.go +++ b/cmd/run_test.go @@ -30,7 +30,6 @@ func Test_runCmd(t *testing.T) { time.Sleep(100 * time.Millisecond) StopGracefully( - context.Background(), context.Background(), nil, nil, @@ -89,7 +88,6 @@ func Test_runCmdWithMultiTenancy(t *testing.T) { time.Sleep(500 * time.Millisecond) StopGracefully( - context.Background(), context.Background(), nil, nil, @@ -169,7 +167,6 @@ func Test_runCmdWithCachePlugin(t *testing.T) { time.Sleep(time.Second) StopGracefully( - context.Background(), context.Background(), nil, nil, diff --git a/network/server.go b/network/server.go index d95d5f8c..3c9270be 100644 --- a/network/server.go +++ b/network/server.go @@ -78,6 +78,9 @@ func (s *Server) OnBoot(engine Engine) Action { s.mu.Unlock() // Run the OnBooted hooks. + pluginTimeoutCtx, cancel = context.WithTimeout(context.Background(), s.pluginTimeout) + defer cancel() + _, err = s.pluginRegistry.Run( pluginTimeoutCtx, map[string]interface{}{"status": fmt.Sprint(s.Status)}, @@ -136,6 +139,9 @@ func (s *Server) OnOpen(conn net.Conn) ([]byte, Action) { } // Run the OnOpened hooks. + pluginTimeoutCtx, cancel = context.WithTimeout(context.Background(), s.pluginTimeout) + defer cancel() + onOpenedData := map[string]interface{}{ "client": map[string]interface{}{ "local": LocalAddr(conn), @@ -214,6 +220,9 @@ func (s *Server) OnClose(conn net.Conn, err error) Action { } // Run the OnClosed hooks. + pluginTimeoutCtx, cancel = context.WithTimeout(context.Background(), s.pluginTimeout) + defer cancel() + data = map[string]interface{}{ "client": map[string]interface{}{ "local": LocalAddr(conn), From 4399644abe96e939c6e5fff0c585665606004cc8 Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Sun, 15 Oct 2023 22:58:16 +0200 Subject: [PATCH 34/35] Use a stack data structure to push and restore the last request --- cmd/run.go | 1 + network/proxy.go | 35 +++++++++++++++--- network/proxy_test.go | 6 ++-- network/server.go | 16 +++++---- network/stack.go | 82 +++++++++++++++++++++++++++++++++++++++++++ 5 files changed, 128 insertions(+), 12 deletions(-) create mode 100644 network/stack.go diff --git a/cmd/run.go b/cmd/run.go index aeb3504c..286ec85f 100644 --- a/cmd/run.go +++ b/cmd/run.go @@ -89,6 +89,7 @@ func StopGracefully( pluginTimeoutCtx, cancel := context.WithTimeout(context.Background(), conf.Plugin.Timeout) defer cancel() + //nolint:contextcheck _, err := pluginRegistry.Run( pluginTimeoutCtx, map[string]interface{}{"signal": signal}, diff --git a/network/proxy.go b/network/proxy.go index 2cf608a8..56e9d799 100644 --- a/network/proxy.go +++ b/network/proxy.go @@ -23,8 +23,8 @@ import ( type IProxy interface { Connect(conn net.Conn) *gerr.GatewayDError Disconnect(conn net.Conn) *gerr.GatewayDError - PassThroughToServer(conn net.Conn) *gerr.GatewayDError - PassThroughToClient(conn net.Conn) *gerr.GatewayDError + PassThroughToServer(conn net.Conn, stack *Stack) *gerr.GatewayDError + PassThroughToClient(conn net.Conn, stack *Stack) *gerr.GatewayDError IsHealthy(cl *Client) (*Client, *gerr.GatewayDError) IsExhausted() bool Shutdown() @@ -260,7 +260,7 @@ func (pr *Proxy) Disconnect(conn net.Conn) *gerr.GatewayDError { } // PassThroughToServer sends the data from the client to the server. -func (pr *Proxy) PassThroughToServer(conn net.Conn) *gerr.GatewayDError { +func (pr *Proxy) PassThroughToServer(conn net.Conn, stack *Stack) *gerr.GatewayDError { _, span := otel.Tracer(config.TracerName).Start(pr.ctx, "PassThrough") defer span.End() @@ -317,6 +317,9 @@ func (pr *Proxy) PassThroughToServer(conn net.Conn) *gerr.GatewayDError { return gerr.ErrClientNotConnected.Wrap(origErr) } + // Push the client's request to the stack. + stack.Push(&Request{Data: request}) + // If the hook wants to terminate the connection, do it. if pr.shouldTerminate(result) { if modResponse, modReceived := pr.getPluginModifiedResponse(result); modResponse != nil { @@ -326,6 +329,10 @@ func (pr *Proxy) PassThroughToServer(conn net.Conn) *gerr.GatewayDError { metrics.TotalTrafficBytes.Observe(float64(modReceived)) span.AddEvent("Terminating connection") + + // Remove the request from the stack if the response is modified. + stack.PopLastRequest() + return pr.sendTrafficToClient(conn, modResponse, modReceived) } span.RecordError(gerr.ErrHookTerminatedConnection) @@ -337,6 +344,8 @@ func (pr *Proxy) PassThroughToServer(conn net.Conn) *gerr.GatewayDError { span.AddEvent("Plugin(s) modified the request") } + stack.UpdateLastRequest(&Request{Data: request}) + // Send the request to the server. _, err = pr.sendTrafficToServer(client, request) span.AddEvent("Sent traffic to server") @@ -370,7 +379,7 @@ func (pr *Proxy) PassThroughToServer(conn net.Conn) *gerr.GatewayDError { } // PassThroughToClient sends the data from the server to the client. -func (pr *Proxy) PassThroughToClient(conn net.Conn) *gerr.GatewayDError { +func (pr *Proxy) PassThroughToClient(conn net.Conn, stack *Stack) *gerr.GatewayDError { _, span := otel.Tracer(config.TracerName).Start(pr.ctx, "PassThrough") defer span.End() @@ -410,12 +419,22 @@ func (pr *Proxy) PassThroughToClient(conn net.Conn) *gerr.GatewayDError { pr.logger.Debug().Fields(fields).Msg("No data to send to client") span.AddEvent("No data to send to client") span.RecordError(err) + + stack.PopLastRequest() + return err } pluginTimeoutCtx, cancel := context.WithTimeout(context.Background(), pr.pluginTimeout) defer cancel() + // Get the last request from the stack. + lastRequest := stack.PopLastRequest() + request := make([]byte, 0) + if lastRequest != nil { + request = lastRequest.Data + } + // Run the OnTrafficFromServer hooks. result, err := pr.pluginRegistry.Run( pluginTimeoutCtx, @@ -423,6 +442,10 @@ func (pr *Proxy) PassThroughToClient(conn net.Conn) *gerr.GatewayDError { conn, client, []Field{ + { + Name: "request", + Value: request, + }, { Name: "response", Value: response[:received], @@ -457,6 +480,10 @@ func (pr *Proxy) PassThroughToClient(conn net.Conn) *gerr.GatewayDError { conn, client, []Field{ + { + Name: "request", + Value: request, + }, { Name: "response", Value: response[:received], diff --git a/network/proxy_test.go b/network/proxy_test.go index 8bd82452..379b6df0 100644 --- a/network/proxy_test.go +++ b/network/proxy_test.go @@ -310,10 +310,12 @@ func BenchmarkProxyPassThrough(b *testing.B) { proxy.Connect(conn.Conn) //nolint:errcheck defer proxy.Disconnect(&conn) //nolint:errcheck + stack := NewStack() + // Connect to the proxy for i := 0; i < b.N; i++ { - proxy.PassThroughToClient(&conn) //nolint:errcheck - proxy.PassThroughToServer(&conn) //nolint:errcheck + proxy.PassThroughToClient(&conn, stack) //nolint:errcheck + proxy.PassThroughToServer(&conn, stack) //nolint:errcheck } } diff --git a/network/server.go b/network/server.go index 3c9270be..71738b08 100644 --- a/network/server.go +++ b/network/server.go @@ -270,35 +270,39 @@ func (s *Server) OnTraffic(conn net.Conn, stopConnection chan struct{}) Action { } span.AddEvent("Ran the OnTraffic hooks") + stack := NewStack() + // Pass the traffic from the client to server. // If there is an error, log it and close the connection. - go func(server *Server, conn net.Conn, stopConnection chan struct{}) { + go func(server *Server, conn net.Conn, stopConnection chan struct{}, stack *Stack) { for { server.logger.Trace().Msg("Passing through traffic from client to server") - if err := server.proxy.PassThroughToServer(conn); err != nil { + if err := server.proxy.PassThroughToServer(conn, stack); err != nil { server.logger.Trace().Err(err).Msg("Failed to pass through traffic") span.RecordError(err) stopConnection <- struct{}{} break } } - }(s, conn, stopConnection) + }(s, conn, stopConnection, stack) // Pass the traffic from the server to client. // If there is an error, log it and close the connection. - go func(server *Server, conn net.Conn, stopConnection chan struct{}) { + go func(server *Server, conn net.Conn, stopConnection chan struct{}, stack *Stack) { for { server.logger.Debug().Msg("Passing through traffic from server to client") - if err := server.proxy.PassThroughToClient(conn); err != nil { + if err := server.proxy.PassThroughToClient(conn, stack); err != nil { server.logger.Trace().Err(err).Msg("Failed to pass through traffic") span.RecordError(err) stopConnection <- struct{}{} break } } - }(s, conn, stopConnection) + }(s, conn, stopConnection, stack) <-stopConnection + stack.Clear() + return Close } diff --git a/network/stack.go b/network/stack.go new file mode 100644 index 00000000..a3dd6cfb --- /dev/null +++ b/network/stack.go @@ -0,0 +1,82 @@ +package network + +import "sync" + +type Request struct { + Data []byte +} + +type Stack struct { + items []*Request + mu sync.RWMutex +} + +func (s *Stack) Push(req *Request) { + s.mu.Lock() + defer s.mu.Unlock() + + s.items = append(s.items, req) +} + +func (s *Stack) GetLastRequest() *Request { + s.mu.RLock() + defer s.mu.RUnlock() + + if len(s.items) == 0 { + return nil + } + + //nolint:staticcheck + for i := len(s.items) - 1; i >= 0; i-- { + return s.items[i] + } + + return nil +} + +func (s *Stack) PopLastRequest() *Request { + s.mu.Lock() + defer s.mu.Unlock() + + if len(s.items) == 0 { + return nil + } + + //nolint:staticcheck + for i := len(s.items) - 1; i >= 0; i-- { + req := s.items[i] + s.items = append(s.items[:i], s.items[i+1:]...) + return req + } + + return nil +} + +func (s *Stack) UpdateLastRequest(req *Request) { + s.mu.Lock() + defer s.mu.Unlock() + + if len(s.items) == 0 { + return + } + + //nolint:staticcheck + for i := len(s.items) - 1; i >= 0; i-- { + s.items[i] = req + return + } +} + +func (s *Stack) Clear() { + s.mu.Lock() + defer s.mu.Unlock() + + s.items = make([]*Request, 0) +} + +func NewStack() *Stack { + return &Stack{ + items: make([]*Request, 0), + mu: sync.RWMutex{}, + } +} From fc41aedcccfb7c55c0a69649a5417b3e72005e6e Mon Sep 17 00:00:00 2001 From: Mostafa Moradian Date: Tue, 17 Oct 2023 22:52:20 +0200 Subject: [PATCH 35/35] Update dependencies --- go.mod | 10 +++++----- go.sum | 18 ++++++++++-------- 2 files changed, 15 insertions(+), 13 deletions(-) diff --git a/go.mod b/go.mod index c6be2933..135b4932 100644 --- a/go.mod +++ b/go.mod @@ -32,7 +32,7 @@ require ( go.opentelemetry.io/otel/sdk v1.19.0 go.opentelemetry.io/otel/trace v1.19.0 golang.org/x/exp v0.0.0-20231006140011-7918f672742d - google.golang.org/genproto/googleapis/api v0.0.0-20231012201019-e917dd12ba7a + google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b google.golang.org/grpc v1.58.3 google.golang.org/protobuf v1.31.0 gopkg.in/natefinch/lumberjack.v2 v2.2.1 @@ -46,7 +46,7 @@ require ( github.com/buger/jsonparser v1.1.1 // indirect github.com/cenkalti/backoff/v4 v4.2.1 // indirect github.com/cespare/xxhash/v2 v2.2.0 // indirect - github.com/cloudflare/circl v1.3.3 // indirect + github.com/cloudflare/circl v1.3.5 // indirect github.com/davecgh/go-spew v1.1.1 // indirect github.com/fatih/color v1.15.0 // indirect github.com/fatih/structs v1.1.0 // indirect @@ -60,7 +60,7 @@ require ( github.com/inconshreveable/mousetrap v1.1.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect github.com/mattn/go-colorable v0.1.13 // indirect - github.com/mattn/go-isatty v0.0.19 // indirect + github.com/mattn/go-isatty v0.0.20 // indirect github.com/matttproud/golang_protobuf_extensions v1.0.4 // indirect github.com/mitchellh/copystructure v1.2.0 // indirect github.com/mitchellh/go-testing-interface v1.14.1 // indirect @@ -81,6 +81,6 @@ require ( golang.org/x/sys v0.13.0 // indirect golang.org/x/text v0.13.0 // indirect google.golang.org/appengine v1.6.8 // indirect - google.golang.org/genproto v0.0.0-20231012201019-e917dd12ba7a // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20231012201019-e917dd12ba7a // indirect + google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b // indirect ) diff --git a/go.sum b/go.sum index 34a522b1..a198162b 100644 --- a/go.sum +++ b/go.sum @@ -46,8 +46,9 @@ github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XL github.com/cespare/xxhash/v2 v2.2.0 h1:DC2CZ1Ep5Y4k3ZQ899DldepgrayRUGE6BBZ/cd9Cj44= github.com/cespare/xxhash/v2 v2.2.0/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/client9/misspell v0.3.4/go.mod h1:qj6jICC3Q7zFZvVWo7KLAzC3yx5G7kyvSDkc90ppPyw= -github.com/cloudflare/circl v1.3.3 h1:fE/Qz0QdIGqeWfnwq0RE0R7MI51s0M2E4Ga9kq5AEMs= github.com/cloudflare/circl v1.3.3/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= +github.com/cloudflare/circl v1.3.5 h1:g+wWynZqVALYAlpSQFAa7TscDnUK8mKYtrxMpw6AUKo= +github.com/cloudflare/circl v1.3.5/go.mod h1:5XYMA4rFBvNIrhs50XuiBJ15vF2pZn4nnUKZrLbUZFA= github.com/cncf/udpa/go v0.0.0-20191209042840-269d4d468f6f/go.mod h1:M8M6+tZqaGXZJjfX53e64911xZQV5JYwmTeXPW+k8Sc= github.com/cncf/udpa/go v0.0.0-20201120205902-5459f2c99403/go.mod h1:WmhPx2Nbnhtbo57+VJT5O0JRkEi1Wbu0z5j0R8u5Hbk= github.com/codingsince1985/checksum v1.3.0 h1:kqqIqWBwjidGmt/pO4yXCEX+np7HACGx72EB+MkKcVY= @@ -242,8 +243,9 @@ github.com/mattn/go-isatty v0.0.11/go.mod h1:PhnuNfih5lzO57/f3n+odYbM4JtupLOxQOA github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-isatty v0.0.14/go.mod h1:7GGIvUiUoEMVVmxf/4nioHXj79iQHKdU27kJ6hsGG94= github.com/mattn/go-isatty v0.0.16/go.mod h1:kYGgaQfpe5nmfYZH+SKPsOc2e4SrIfOl2e/yFXSvRLM= -github.com/mattn/go-isatty v0.0.19 h1:JITubQf0MOLdlGRuRq+jtsDlekdYPia9ZFsB8h/APPA= github.com/mattn/go-isatty v0.0.19/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= +github.com/mattn/go-isatty v0.0.20 h1:xfD0iDuEKnDkl03q4limB+vH+GxLEtL/jb4xVJSWWEY= +github.com/mattn/go-isatty v0.0.20/go.mod h1:W+V8PltTTMOvKvAeJH7IuucS94S2C6jfK/D7dTCTo3Y= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/matttproud/golang_protobuf_extensions v1.0.4 h1:mmDVorXM7PCGKw94cs5zkfA9PSy5pEvNWRP0ET0TIVo= github.com/matttproud/golang_protobuf_extensions v1.0.4/go.mod h1:BSXmuO+STAnVfrANrmjBb36TMTDstsz7MSK+HVaYKv4= @@ -544,12 +546,12 @@ google.golang.org/genproto v0.0.0-20190819201941-24fa4b261c55/go.mod h1:DMBHOl98 google.golang.org/genproto v0.0.0-20200513103714-09dca8ec2884/go.mod h1:55QSHmfGQM9UVYDPBsyGGes0y52j32PQ3BqQfXhyH3c= google.golang.org/genproto v0.0.0-20200526211855-cb27e3aa2013/go.mod h1:NbSheEEYHJ7i3ixzK3sjbqSGDJWnxyFXZblF3eUsNvo= google.golang.org/genproto v0.0.0-20210602131652-f16073e35f0c/go.mod h1:UODoCrxHCcBojKKwX1terBiRUaqAsFqJiF615XL43r0= -google.golang.org/genproto v0.0.0-20231012201019-e917dd12ba7a h1:fwgW9j3vHirt4ObdHoYNwuO24BEZjSzbh+zPaNWoiY8= -google.golang.org/genproto v0.0.0-20231012201019-e917dd12ba7a/go.mod h1:EMfReVxb80Dq1hhioy0sOsY9jCE46YDgHlJ7fWVUWRE= -google.golang.org/genproto/googleapis/api v0.0.0-20231012201019-e917dd12ba7a h1:myvhA4is3vrit1a6NZCWBIwN0kNEnX21DJOJX/NvIfI= -google.golang.org/genproto/googleapis/api v0.0.0-20231012201019-e917dd12ba7a/go.mod h1:SUBoKXbI1Efip18FClrQVGjWcyd0QZd8KkvdP34t7ww= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231012201019-e917dd12ba7a h1:a2MQQVoTo96JC9PMGtGBymLp7+/RzpFc2yX/9WfFg1c= -google.golang.org/genproto/googleapis/rpc v0.0.0-20231012201019-e917dd12ba7a/go.mod h1:4cYg8o5yUbm77w8ZX00LhMVNl/YVBFJRYWDc0uYWMs0= +google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b h1:+YaDE2r2OG8t/z5qmsh7Y+XXwCbvadxxZ0YY6mTdrVA= +google.golang.org/genproto v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:CgAqfJo+Xmu0GwA0411Ht3OU3OntXwsGmrmjI8ioGXI= +google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b h1:CIC2YMXmIhYw6evmhPxBKJ4fmLbOFtXQN/GV3XOZR8k= +google.golang.org/genproto/googleapis/api v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:IBQ646DjkDkvUIsVq/cc03FUFQ9wbZu7yE396YcL870= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b h1:ZlWIi1wSK56/8hn4QcBp/j9M7Gt3U/3hZw3mC7vDICo= +google.golang.org/genproto/googleapis/rpc v0.0.0-20231016165738-49dd2c1f3d0b/go.mod h1:swOH3j0KzcDDgGUWr+SNpyTen5YrXjS3eyPzFYKc6lc= google.golang.org/grpc v1.14.0/go.mod h1:yo6s7OP7yaDglbqo1J04qKzAhqBH6lvTonzMVmEdcZw= google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c= google.golang.org/grpc v1.22.0/go.mod h1:Y5yQAOtifL1yxbo5wqy6BxZv8vAUGQwXBOALyacEbxg=