add retry to leader election process

pkg/leader/election.go

@@ -10,6 +10,7 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/leaderelection"
 	"k8s.io/client-go/tools/leaderelection/resourcelock"
+	"k8s.io/client-go/util/flowcontrol"
 )
 
 func NewElection(clientset kubernetes.Interface, id, namespace string) *Election {
@@ -63,5 +64,27 @@ type Election struct {
 }
 
 func (le *Election) Start(ctx context.Context) {
-	leaderelection.RunOrDie(ctx, le.config)
+	backoff := flowcontrol.NewBackOff(1*time.Second, 15*time.Second)
+	const backoffID = "lingo-leader-election"
+	retryCount := 0
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		default:
+			if retryCount > 0 {
+				backoff.Next(backoffID, backoff.Clock.Now())
+				delay := backoff.Get(backoffID)
+				log.Printf("Leader election failed, retrying in %v. RetryCount: %v", delay, retryCount+1)
+				select {
+				case <-time.After(delay):
+				case <-ctx.Done():
+					return
+				}
+			}
+			log.Printf("Starting leader election process. RetryCount: %v", retryCount+1)
+			leaderelection.RunOrDie(ctx, le.config)
+			retryCount++
+		}
+	}
 }