feat: Webhook retries

Signed-off-by: Joseph Kwasniewski <[email protected]>
fluxcd · Oct 18, 2023 · 7fa50da · 7fa50da
1 parent c5369e9
commit 7fa50da
Show file tree

Hide file tree

Showing 4 changed files with 22 additions and 6 deletions.
diff --git a/artifacts/flagger/crd.yaml b/artifacts/flagger/crd.yaml
@@ -1132,6 +1132,9 @@ spec:
                             description: Request timeout for this webhook
                             type: string
                             pattern: "^[0-9]+(m|s)"
+                          retries:
+                            description: Number of retries for this webhook
+                            type: number
                           metadata:
                             description: Metadata (key-value pairs) for this webhook
                             type: object

diff --git a/charts/flagger/crds/crd.yaml b/charts/flagger/crds/crd.yaml
@@ -1132,6 +1132,9 @@ spec:
                             description: Request timeout for this webhook
                             type: string
                             pattern: "^[0-9]+(m|s)"
+                          retries:
+                            description: Number of retries for this webhook
+                            type: number
                           metadata:
                             description: Metadata (key-value pairs) for this webhook
                             type: object

diff --git a/pkg/apis/flagger/v1beta1/canary.go b/pkg/apis/flagger/v1beta1/canary.go
@@ -394,6 +394,9 @@ type CanaryWebhook struct {
 	// Metadata (key-value pairs) for this webhook
 	// +optional
 	Metadata *map[string]string `json:"metadata,omitempty"`
+
+	// Number of retries for this webhook
+	Retries int `json:"retries,omitempty"`
 }
 
 // CanaryWebhookPayload holds the deployment info and metadata sent to webhooks

diff --git a/pkg/controller/webhook.go b/pkg/controller/webhook.go
@@ -23,16 +23,17 @@ import (
 	"errors"
 	"fmt"
 	"io"
-	"net/http"
 	"net/url"
 	"strconv"
 	"time"
 
+	"github.com/hashicorp/go-retryablehttp"
+
 	flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1"
 	"github.com/fluxcd/flagger/pkg/canary"
 )
 
-func callWebhook(webhook string, payload interface{}, timeout string) error {
+func callWebhook(webhook string, payload interface{}, timeout string, retries int) error {
 	payloadBin, err := json.Marshal(payload)
 	if err != nil {
 		return err
@@ -43,7 +44,13 @@ func callWebhook(webhook string, payload interface{}, timeout string) error {
 		return err
 	}
 
-	req, err := http.NewRequest("POST", hook.String(), bytes.NewBuffer(payloadBin))
+	httpClient := retryablehttp.NewClient()
+	httpClient.RetryWaitMin = 5 * time.Second
+	httpClient.RetryWaitMax = 30 * time.Second
+	httpClient.RetryMax = retries
+	httpClient.Logger = nil
+
+	req, err := retryablehttp.NewRequest("POST", hook.String(), bytes.NewBuffer(payloadBin))
 	if err != nil {
 		return err
 	}
@@ -62,7 +69,7 @@ func callWebhook(webhook string, payload interface{}, timeout string) error {
 	ctx, cancel := context.WithTimeout(req.Context(), t)
 	defer cancel()
 
-	r, err := http.DefaultClient.Do(req.WithContext(ctx))
+	r, err := httpClient.Do(req.WithContext(ctx))
 	if err != nil {
 		return err
 	}
@@ -98,7 +105,7 @@ func CallWebhook(canary flaggerv1.Canary, phase flaggerv1.CanaryPhase, w flagger
 		w.Timeout = "10s"
 	}
 
-	return callWebhook(w.URL, payload, w.Timeout)
+	return callWebhook(w.URL, payload, w.Timeout, w.Retries)
 }
 
 func CallEventWebhook(r *flaggerv1.Canary, w flaggerv1.CanaryWebhook, message, eventtype string) error {
@@ -124,7 +131,7 @@ func CallEventWebhook(r *flaggerv1.Canary, w flaggerv1.CanaryWebhook, message, e
 			payload.Metadata[key] = value
 		}
 	}
-	return callWebhook(w.URL, payload, "5s")
+	return callWebhook(w.URL, payload, "5s", 0)
 }
 
 func canaryChecksum(c flaggerv1.Canary) string {