From e365dd724fd9648afbc340586d7ce05cba25eb1d Mon Sep 17 00:00:00 2001
From: Sam Stoelinga <sammiestoel@gmail.com>
Date: Sat, 3 Feb 2024 15:05:13 -0800
Subject: [PATCH] add retry to leader election process

---
 pkg/leader/election.go | 25 ++++++++++++++++++++++++-
 1 file changed, 24 insertions(+), 1 deletion(-)

diff --git a/pkg/leader/election.go b/pkg/leader/election.go
index 1bdcaf55..78bd2ba9 100644
--- a/pkg/leader/election.go
+++ b/pkg/leader/election.go
@@ -10,6 +10,7 @@ import (
 	"k8s.io/client-go/kubernetes"
 	"k8s.io/client-go/tools/leaderelection"
 	"k8s.io/client-go/tools/leaderelection/resourcelock"
+	"k8s.io/client-go/util/flowcontrol"
 )
 
 func NewElection(clientset kubernetes.Interface, id, namespace string) *Election {
@@ -63,5 +64,27 @@ type Election struct {
 }
 
 func (le *Election) Start(ctx context.Context) {
-	leaderelection.RunOrDie(ctx, le.config)
+	backoff := flowcontrol.NewBackOff(1*time.Second, 15*time.Second)
+	const backoffID = "lingo-leader-election"
+	retryCount := 0
+	for {
+		select {
+		case <-ctx.Done():
+			return
+		default:
+			if retryCount > 0 {
+				backoff.Next(backoffID, backoff.Clock.Now())
+				delay := backoff.Get(backoffID)
+				log.Printf("Leader election failed, retrying in %v. RetryCount: %v", delay, retryCount+1)
+				select {
+				case <-time.After(delay):
+				case <-ctx.Done():
+					return
+				}
+			}
+			log.Printf("Starting leader election process. RetryCount: %v", retryCount+1)
+			leaderelection.RunOrDie(ctx, le.config)
+			retryCount++
+		}
+	}
 }