retry limit on rolloutHook

fluxcd · Sep 8, 2023 · 61e128c · 61e128c
1 parent 8dbd8d5
commit 61e128c
Show file tree

Hide file tree

Showing 3 changed files with 24 additions and 5 deletions.
diff --git a/charts/flagger/crds/crd.yaml b/charts/flagger/crds/crd.yaml
@@ -1077,6 +1077,9 @@ spec:
                             description: URL address of this webhook
                             type: string
                             format: url
+                          retrylimit:
+                            type: integer
+                            description: Number of retry if Rollout hook failed
                           timeout:
                             description: Request timeout for this webhook
                             type: string

diff --git a/pkg/apis/flagger/v1beta1/canary.go b/pkg/apis/flagger/v1beta1/canary.go
@@ -390,6 +390,11 @@ type CanaryWebhook struct {
 	// Metadata (key-value pairs) for this webhook
 	// +optional
 	Metadata *map[string]string `json:"metadata,omitempty"`
+
+	// FailureThreshold for rollout hook
+	// +optional
+	RetryLimit int `json:"retrylimit,omitempty"`
+
 }
 
 // CanaryWebhookPayload holds the deployment info and metadata sent to webhooks

diff --git a/pkg/controller/scheduler.go b/pkg/controller/scheduler.go
@@ -727,11 +727,22 @@ func (c *Controller) runAnalysis(canary *flaggerv1.Canary) bool {
 	// run external checks
 	for _, webhook := range canary.GetAnalysis().Webhooks {
 		if webhook.Type == "" || webhook.Type == flaggerv1.RolloutHook {
-			err := CallWebhook(canary.Name, canary.Namespace, flaggerv1.CanaryPhaseProgressing, webhook)
-			if err != nil {
-				c.recordEventWarningf(canary, "Halt %s.%s advancement external check %s failed %v",
-					canary.Name, canary.Namespace, webhook.Name, err)
-				return false
+			retries := 0 
+			for {
+				err := CallWebhook(canary.Name, canary.Namespace, flaggerv1.CanaryPhaseProgressing, webhook)
+				if err != nil {
+					c.recordEventWarningf(canary, "Retrying %s.%s advancement external check %s failed %v",
+						canary.Name, canary.Namespace, webhook.Name, err)
+					retries++ 
+					if retries >= retrylimit { 
+						c.recordEventWarningf(canary, "Halt %s.%s advancement external check %s failed %v",
+						canary.Name, canary.Namespace, webhook.Name, err)
+						return false // retry limit crossed retrylimit
+					} 
+				} else {
+					break // Success, exit the retry loop
+				}
+
 			}
 		}
 	}