From b8dc5d9ed07dc83099a09b6353530af80000000d Mon Sep 17 00:00:00 2001 From: Thibault Jamet Date: Thu, 20 Feb 2020 13:52:52 +0100 Subject: [PATCH] Remove terminating pods from endpoints The standard endpoint controller removes, by default, terminating pods from an endpoint: https://github.com/kubernetes/kubernetes/commit/2aaf8bddc253737fefb77e4d0306872c553ddbde#diff-a1a9c0efe93384ed9a010e65e8ad4604R316 This behaviour let the service clients time to react to stop sending traffic to the terminating pods while finishing processing current requests, as pods in "terminating" state are technically still able to accept connections and handle requests. It allows to smoothly replace the pods by the activator when all pods are entering the terminating state, for the reason described above. --- pkg/endpoints/controller/endpoints_manager.go | 30 +++++++++++++++++++ 1 file changed, 30 insertions(+) diff --git a/pkg/endpoints/controller/endpoints_manager.go b/pkg/endpoints/controller/endpoints_manager.go index 7a06e3e5..0dc144dd 100644 --- a/pkg/endpoints/controller/endpoints_manager.go +++ b/pkg/endpoints/controller/endpoints_manager.go @@ -5,6 +5,7 @@ import ( "encoding/base64" "encoding/json" "fmt" + "strconv" "sync" "github.com/deislabs/osiris/pkg/kubernetes" @@ -18,6 +19,14 @@ import ( endpointsv1 "k8s.io/kubernetes/pkg/api/v1/endpoints" ) +const ( + + // tolerateAnnoration backport annotation + // TolerateUnreadyEndpointsAnnotation definition from + // k8s.io/kubernetes/pkg/controller/endpoint + tolerateAnnoration = "service.alpha.kubernetes.io/tolerate-unready-endpoints" +) + // endpointsManager is a controller responsible for the on-going management of // the endpoints resource corresponding to a single Osiris-enabled service type endpointsManager struct { @@ -127,6 +136,27 @@ func (e *endpointsManager) syncAppPod(obj interface{}) { break } } + + // If the user specified the older (deprecated) annotation, + // we have to respect it. + tolerateUnreadyEndpoints := e.service.Spec.PublishNotReadyAddresses + v, ok := e.service.Annotations[tolerateAnnoration] + if ok { + b, err := strconv.ParseBool(v) + if err == nil { + tolerateUnreadyEndpoints = b + } else { + glog.Errorf( + "Failed to parse annotation %v: %v", + tolerateAnnoration, + err, + ) + } + } + + if !tolerateUnreadyEndpoints && pod.DeletionTimestamp != nil { + ready = false + } glog.Infof( "Informed about pod %s for service %s in namespace %s; its IP is %s and "+ "its ready condition is %t",