Review feedback

pkg/endpoints/endpoints.go

@@ -18,8 +18,7 @@ func newEndpointGroup() *endpointGroup {
 }
 
 type endpoint struct {
-	inFlight   *atomic.Int64
-	terminated chan struct{}
+	inFlight *atomic.Int64
 }
 
 type endpointGroup struct {
@@ -107,13 +106,12 @@ func (g *endpointGroup) setIPs(ips map[string]struct{}, ports map[string]int32)
 	g.ports = ports
 	for ip := range ips {
 		if _, ok := g.endpoints[ip]; !ok {
-			g.endpoints[ip] = endpoint{inFlight: &atomic.Int64{}, terminated: make(chan struct{})}
+			g.endpoints[ip] = endpoint{inFlight: &atomic.Int64{}}
 		}
 	}
-	for ip, endpoint := range g.endpoints {
+	for ip := range g.endpoints {
 		if _, ok := ips[ip]; !ok {
 			delete(g.endpoints, ip)
-			close(endpoint.terminated)
 		}
 	}
 	g.mtx.Unlock()

pkg/proxy/handler.go

@@ -131,6 +131,7 @@ func parseModel(r *http.Request) (string, *http.Request, error) {
 	}
 	var body []byte
 	if mb, ok := r.Body.(*lazyBodyCapturer); ok && mb.capturedBody != nil {
+		// reuse buffer
 		body = mb.capturedBody
 	} else {
 		// parse request body for model name, ignore errors

pkg/proxy/middleware.go

@@ -14,7 +14,6 @@ var _ http.Handler = &RetryMiddleware{}
 type RetryMiddleware struct {
 	nextHandler      http.Handler
 	maxRetries       int
-	rSource          *rand.Rand
 	retryStatusCodes map[int]struct{}
 }
 
@@ -39,7 +38,6 @@ func NewRetryMiddleware(maxRetries int, other http.Handler, optRetryStatusCodes
 		nextHandler:      other,
 		maxRetries:       maxRetries,
 		retryStatusCodes: statusCodeIndex,
-		rSource:          rand.New(rand.NewSource(time.Now().UnixNano())),
 	}
 }
 
@@ -72,7 +70,7 @@ func (r RetryMiddleware) ServeHTTP(writer http.ResponseWriter, request *http.Req
 
 		totalRetries.Inc()
 		// exponential backoff
-		jitter := time.Duration(r.rSource.Intn(50))
+		jitter := time.Duration(rand.Intn(50))
 		time.Sleep(time.Millisecond*time.Duration(1<<uint(i)) + jitter)
 	}
 }