Remove terminating pods from endpoints

The standard endpoint controller removes, by default, terminating pods
from an endpoint: kubernetes/kubernetes@2aaf8bd#diff-a1a9c0efe93384ed9a010e65e8ad4604R316

This behaviour let the service clients time to react to stop sending
traffic to the terminating pods while finishing processing current
requests, as pods in "terminating" state are technically still able to
accept connections and handle requests.

It allows to smoothly replace the pods by the activator when all pods
are entering the terminating state, for the reason described above.

pkg/endpoints/controller/endpoints_manager.go

@@ -5,6 +5,7 @@ import (
 	"encoding/base64"
 	"encoding/json"
 	"fmt"
+	"strconv"
 	"sync"
 
 	"github.com/deislabs/osiris/pkg/kubernetes"
@@ -18,6 +19,23 @@ import (
 	endpointsv1 "k8s.io/kubernetes/pkg/api/v1/endpoints"
 )
 
+const (
+
+	// TolerateUnreadyEndpointsAnnotation is an annotation on the Service denoting if the endpoints
+	// controller should go ahead and create endpoints for unready pods. This annotation is
+	// currently only used by StatefulSets, where we need the pod to be DNS
+	// resolvable during initialization and termination. In this situation we
+	// create a headless Service just for the StatefulSet, and clients shouldn't
+	// be using this Service for anything so unready endpoints don't matter.
+	// Endpoints of these Services retain their DNS records and continue
+	// receiving traffic for the Service from the moment the kubelet starts all
+	// containers in the pod and marks it "Running", till the kubelet stops all
+	// containers and deletes the pod from the apiserver.
+	// This field is deprecated. v1.Service.PublishNotReadyAddresses will replace it
+	// subsequent releases.  It will be removed no sooner than 1.13.
+	TolerateUnreadyEndpointsAnnotation = "service.alpha.kubernetes.io/tolerate-unready-endpoints"
+)
+
 // endpointsManager is a controller responsible for the on-going management of
 // the endpoints resource corresponding to a single Osiris-enabled service
 type endpointsManager struct {
@@ -127,6 +145,27 @@ func (e *endpointsManager) syncAppPod(obj interface{}) {
 			break
 		}
 	}
+
+	// If the user specified the older (deprecated) annotation,
+	// we have to respect it.
+	tolerateUnreadyEndpoints := e.service.Spec.PublishNotReadyAddresses
+	v, ok := e.service.Annotations[TolerateUnreadyEndpointsAnnotation]
+	if ok {
+		b, err := strconv.ParseBool(v)
+		if err == nil {
+			tolerateUnreadyEndpoints = b
+		} else {
+			glog.Errorf(
+				"Failed to parse annotation %v: %v",
+				TolerateUnreadyEndpointsAnnotation,
+				err,
+			)
+		}
+	}
+
+	if !tolerateUnreadyEndpoints && pod.DeletionTimestamp != nil {
+		ready = false
+	}
 	glog.Infof(
 		"Informed about pod %s for service %s in namespace %s; its IP is %s and "+
 			"its ready condition is %t",