gatewayapi: add support for session affinity

Add support for Canary releases with session affinity for Gateway API. This enables any Gateway API implementation that supports [`ResponseHeaderModifier`](https://github.com/kubernetes-sigs/gateway-api/blob/3d22aa5a08413222cb79e6b2e245870360434614/apis/v1beta1/httproute_types.go#L651) to be used with session affinity. Signed-off-by: Sanskar Jaiswal <[email protected]>
fluxcd · Sep 7, 2023 · 5136e62 · 5136e62
1 parent 8dbd8d5
commit 5136e62
Show file tree

Hide file tree

Showing 4 changed files with 534 additions and 21 deletions.
diff --git a/docs/gitbook/tutorials/gatewayapi-progressive-delivery.md b/docs/gitbook/tutorials/gatewayapi-progressive-delivery.md
@@ -137,7 +137,7 @@ Save the above resource as metric-templates.yaml and then apply it:
 kubectl apply -f metric-templates.yaml
 ```
 
-Create a canary custom resource \(replace "loaclproject.contour.io" with your own domain\):
+Create a canary custom resource \(replace "localproject.contour.io" with your own domain\):
 
 ```yaml
 apiVersion: flagger.app/v1beta1
@@ -382,13 +382,121 @@ Events:
   Warning  Synced  1m    flagger  Canary failed! Scaling down podinfo.test
 ```
 
+## Session Affinity
+
+While Flagger can perform weighted routing and A/B testing individually, with Istio it can combine the two leading to a Canary
+release with session affinity. For more information you can read the [deployment strategies docs](../usage/deployment-strategies.md#canary-release-with-session-affinity).
+
+Create a canary custom resource \(replace localproject.contour.io with your own domain\):
+
+```yaml
+apiVersion: flagger.app/v1beta1
+kind: Canary
+metadata:
+  name: podinfo
+  namespace: test
+spec:
+  # deployment reference
+  targetRef:
+    apiVersion: apps/v1
+    kind: Deployment
+    name: podinfo
+  # the maximum time in seconds for the canary deployment
+  # to make progress before it is rollback (default 600s)
+  progressDeadlineSeconds: 60
+  # HPA reference (optional)
+  autoscalerRef:
+    apiVersion: autoscaling/v2beta2
+    kind: HorizontalPodAutoscaler
+    name: podinfo
+  service:
+    # service port number
+    port: 9898
+    # container port number or name (optional)
+    targetPort: 9898
+    # Gateway API HTTPRoute host names
+    hosts:
+     - localproject.contour.io
+    # Reference to the Gateway that the generated HTTPRoute would attach to.
+    gatewayRefs:
+      - name: contour
+        namespace: projectcontour
+  analysis:
+    # schedule interval (default 60s)
+    interval: 1m
+    # max number of failed metric checks before rollback
+    threshold: 5
+    # max traffic percentage routed to canary
+    # percentage (0-100)
+    maxWeight: 50
+    # canary increment step
+    # percentage (0-100)
+    stepWeight: 10
+    # session affinity config
+    sessionAffinity:
+      # name of the cookie used
+      cookieName: flagger-cookie
+      # max age of the cookie (in seconds)
+      # optional; defaults to 86400
+      maxAge: 21600
+    metrics:
+    - name: error-rate
+      # max error rate (5xx responses)
+      # percentage (0-100)
+      templateRef:
+        name: error-rate
+        namespace: flagger-system
+      thresholdRange:
+        max: 1
+      interval: 1m
+    - name: latency
+      templateRef:
+        name: latency
+        namespace: flagger-system
+      # seconds
+      thresholdRange:
+         max: 0.5
+      interval: 30s
+    # testing (optional)
+    webhooks:
+      - name: smoke-test
+        type: pre-rollout
+        url: http://flagger-loadtester.test/
+        timeout: 15s
+        metadata:
+          type: bash
+          cmd: "curl -sd 'anon' http://podinfo-canary.test:9898/token | grep token"
+      - name: load-test
+        url: http://flagger-loadtester.test/
+        timeout: 5s
+        metadata:
+          cmd: "hey -z 2m -q 10 -c 2 -host localproject.contour.io http://envoy.projectcontour/"
+```
+
+Save the above resource as podinfo-canary-session-affinity.yaml and then apply it:
+
+```bash
+kubectl apply -f ./podinfo-canary-session-affinity.yaml
+```
+
+Trigger a canary deployment by updating the container image:
+
+```bash
+kubectl -n test set image deployment/podinfo \
+podinfod=ghcr.io/stefanprodan/podinfo:6.0.1
+```
+
+You can load `localproject.contour.io` in your browser and refresh it until you see the requests being served by `podinfo:6.0.1`.
+All subsequent requests after that will be served by `podinfo:6.0.1` and not `podinfo:6.0.0` because of the session affinity
+configured by Flagger in the HTTPRoute object.
+
 # A/B Testing
 
 Besides weighted routing, Flagger can be configured to route traffic to the canary based on HTTP match conditions. In an A/B testing scenario, you'll be using HTTP headers or cookies to target a certain segment of your users. This is particularly useful for frontend applications that require session affinity.
 
 ![Flagger A/B Testing Stages](https://raw.githubusercontent.com/fluxcd/flagger/main/docs/diagrams/flagger-abtest-steps.png)
 
-Create a canary custom resource \(replace "loaclproject.contour.io" with your own domain\):
+Create a canary custom resource \(replace "localproject.contour.io" with your own domain\):
 
 ```yaml
 apiVersion: flagger.app/v1beta1

diff --git a/docs/gitbook/usage/deployment-strategies.md b/docs/gitbook/usage/deployment-strategies.md
@@ -11,7 +11,7 @@ Flagger can run automated application analysis, promotion and rollback for the f
 * **Blue/Green Mirroring** \(traffic shadowing\)
   * Istio
 * **Canary Release with Session Affinity** \(progressive traffic shifting combined with cookie based routing\)
-  * Istio
+  * Istio, Gateway API
 
 For Canary releases and A/B testing you'll need a Layer 7 traffic management solution like
 a service mesh or an ingress controller. For Blue/Green deployments no service mesh or ingress controller is required.
@@ -408,7 +408,7 @@ cookie based routing with regular weight based routing. This means once a user i
 version of our application (based on the traffic weights), they're always routed to that version, i.e.
 they're never routed back to the old version of our application.
 
-You can enable this, by specifying `.spec.analsyis.sessionAffinity` in the Canary (only Istio is supported):
+You can enable this, by specifying `.spec.analsyis.sessionAffinity` in the Canary:
 
 ```yaml
   analysis:

diff --git a/pkg/router/gateway_api_v1beta1.go b/pkg/router/gateway_api_v1beta1.go
@@ -20,6 +20,7 @@ import (
 	"context"
 	"fmt"
 	"reflect"
+	"strings"
 
 	flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1"
 	"github.com/fluxcd/flagger/pkg/apis/gatewayapi/v1beta1"
@@ -162,10 +163,32 @@ func (gwr *GatewayAPIV1Beta1Router) Reconcile(canary *flaggerv1.Canary) error {
 		return fmt.Errorf("HTTPRoute %s.%s get error: %w", apexSvcName, hrNamespace, err)
 	}
 
+	ignoreCmpOptions := []cmp.Option{
+		cmpopts.IgnoreFields(v1beta1.BackendRef{}, "Weight"),
+		cmpopts.EquateEmpty(),
+	}
+	if canary.Spec.Analysis.SessionAffinity != nil {
+		ignoreRoute := cmpopts.IgnoreSliceElements(func(r v1beta1.HTTPRouteRule) bool {
+			// Ignore the rule that does sticky routing, i.e. matches against the `Cookie` header.
+			for _, match := range r.Matches {
+				for _, headerMatch := range match.Headers {
+					if *headerMatch.Type == v1beta1HeaderMatchRegex && headerMatch.Name == cookieHeader &&
+						strings.Contains(headerMatch.Value, canary.Spec.Analysis.SessionAffinity.CookieName) {
+						return true
+					}
+				}
+			}
+			return false
+		})
+		ignoreCmpOptions = append(ignoreCmpOptions, ignoreRoute)
+		// Ignore backend specific filters, since we use that to insert the `Set-Cookie` header in responses.
+		ignoreCmpOptions = append(ignoreCmpOptions, cmpopts.IgnoreFields(v1beta1.HTTPBackendRef{}, "Filters"))
+	}
+
 	if httpRoute != nil {
 		specDiff := cmp.Diff(
 			httpRoute.Spec, httpRouteSpec,
-			cmpopts.IgnoreFields(v1beta1.BackendRef{}, "Weight"),
+			ignoreCmpOptions...,
 		)
 		labelsDiff := cmp.Diff(newMetadata.Labels, httpRoute.Labels, cmpopts.EquateEmpty())
 		annotationsDiff := cmp.Diff(newMetadata.Annotations, httpRoute.Annotations, cmpopts.EquateEmpty())
@@ -200,7 +223,19 @@ func (gwr *GatewayAPIV1Beta1Router) GetRoutes(canary *flaggerv1.Canary) (
 		err = fmt.Errorf("HTTPRoute %s.%s get error: %w", apexSvcName, hrNamespace, err)
 		return
 	}
+	var weightedRule *v1beta1.HTTPRouteRule
 	for _, rule := range httpRoute.Spec.Rules {
+		// If session affinity is enabled, then we are only interested in the rule
+		// that has backend-specific filters, as that's the rule that does weighted
+		// routing.
+		if canary.Spec.Analysis.SessionAffinity != nil {
+			for _, backendRef := range rule.BackendRefs {
+				if len(backendRef.Filters) > 0 {
+					weightedRule = &rule
+				}
+			}
+		}
+
 		// A/B testing: Avoid reading the rule with only for backendRef.
 		if len(rule.BackendRefs) == 2 {
 			for _, backendRef := range rule.BackendRefs {
@@ -212,7 +247,17 @@ func (gwr *GatewayAPIV1Beta1Router) GetRoutes(canary *flaggerv1.Canary) (
 				}
 			}
 		}
+	}
 
+	if weightedRule != nil {
+		for _, backendRef := range weightedRule.BackendRefs {
+			if backendRef.Name == v1beta1.ObjectName(primarySvcName) {
+				primaryWeight = int(*backendRef.Weight)
+			}
+			if backendRef.Name == v1beta1.ObjectName(canarySvcName) {
+				canaryWeight = int(*backendRef.Weight)
+			}
+		}
 	}
 	return
 }
@@ -248,25 +293,35 @@ func (gwr *GatewayAPIV1Beta1Router) SetRoutes(
 			},
 		})
 	}
+	weightedRouteRule := &v1beta1.HTTPRouteRule{
+		Matches: matches,
+		BackendRefs: []v1beta1.HTTPBackendRef{
+			{
+				BackendRef: gwr.makeBackendRef(primarySvcName, pWeight, canary.Spec.Service.Port),
+			},
+			{
+				BackendRef: gwr.makeBackendRef(canarySvcName, cWeight, canary.Spec.Service.Port),
+			},
+		},
+	}
 	httpRouteSpec := v1beta1.HTTPRouteSpec{
 		CommonRouteSpec: v1beta1.CommonRouteSpec{
 			ParentRefs: canary.Spec.Service.GatewayRefs,
 		},
 		Hostnames: hostNames,
 		Rules: []v1beta1.HTTPRouteRule{
-			{
-				Matches: matches,
-				BackendRefs: []v1beta1.HTTPBackendRef{
-					{
-						BackendRef: gwr.makeBackendRef(primarySvcName, pWeight, canary.Spec.Service.Port),
-					},
-					{
-						BackendRef: gwr.makeBackendRef(canarySvcName, cWeight, canary.Spec.Service.Port),
-					},
-				},
-			},
+			*weightedRouteRule,
 		},
 	}
+
+	if canary.Spec.Analysis.SessionAffinity != nil {
+		rules, err := gwr.getSessionAffinityRouteRules(canary, canaryWeight, weightedRouteRule)
+		if err != nil {
+			return err
+		}
+		httpRouteSpec.Rules = rules
+	}
+
 	hrClone.Spec = httpRouteSpec
 
 	// A/B testing
@@ -295,6 +350,112 @@ func (gwr *GatewayAPIV1Beta1Router) Finalize(_ *flaggerv1.Canary) error {
 	return nil
 }
 
+// getSessionAffinityRouteRules returns the HTTPRouteRule objects required to perform
+// session affinity based Canary releases.
+func (gwr *GatewayAPIV1Beta1Router) getSessionAffinityRouteRules(canary *flaggerv1.Canary, canaryWeight int,
+	weightedRouteRule *v1beta1.HTTPRouteRule) ([]v1beta1.HTTPRouteRule, error) {
+	_, primarySvcName, canarySvcName := canary.GetServiceNames()
+	stickyRouteRule := *weightedRouteRule
+
+	// If a canary run is active, we want all responses corresponding to requests hitting the canary deployment
+	// (due to weighted routing) to include a `Set-Cookie` header. All requests that have the `Cookie` header
+	// and match the value of the `Set-Cookie` header will be routed to the canary deployment.
+	if canaryWeight != 0 {
+		if canary.Status.SessionAffinityCookie == "" {
+			canary.Status.SessionAffinityCookie = fmt.Sprintf("%s=%s", canary.Spec.Analysis.SessionAffinity.CookieName, randSeq())
+		}
+
+		// Add `Set-Cookie` header modifier to the primary backend in the weighted routing rule.
+		for i, backendRef := range weightedRouteRule.BackendRefs {
+			if string(backendRef.BackendObjectReference.Name) == canarySvcName {
+				backendRef.Filters = append(backendRef.Filters, v1beta1.HTTPRouteFilter{
+					Type: v1beta1.HTTPRouteFilterResponseHeaderModifier,
+					ResponseHeaderModifier: &v1beta1.HTTPHeaderFilter{
+						Add: []v1beta1.HTTPHeader{
+							{
+								Name: setCookieHeader,
+								Value: fmt.Sprintf("%s; %s=%d", canary.Status.SessionAffinityCookie, maxAgeAttr,
+									canary.Spec.Analysis.SessionAffinity.GetMaxAge(),
+								),
+							},
+						},
+					},
+				})
+			}
+			weightedRouteRule.BackendRefs[i] = backendRef
+		}
+
+		// Add `Cookie` header matcher to the sticky routing rule.
+		cookieKeyAndVal := strings.Split(canary.Status.SessionAffinityCookie, "=")
+		regexMatchType := v1beta1.HeaderMatchRegularExpression
+		cookieMatch := v1beta1.HTTPRouteMatch{
+			Headers: []v1beta1.HTTPHeaderMatch{
+				{
+					Type:  &regexMatchType,
+					Name:  cookieHeader,
+					Value: fmt.Sprintf(".*%s.*%s.*", cookieKeyAndVal[0], cookieKeyAndVal[1]),
+				},
+			},
+		}
+
+		svcMatches, err := gwr.mapRouteMatches(canary.Spec.Service.Match)
+		if err != nil {
+			return nil, err
+		}
+
+		mergedMatches := gwr.mergeMatchConditions([]v1beta1.HTTPRouteMatch{cookieMatch}, svcMatches)
+		stickyRouteRule.Matches = mergedMatches
+		stickyRouteRule.BackendRefs = []v1beta1.HTTPBackendRef{
+			{
+				BackendRef: gwr.makeBackendRef(primarySvcName, 0, canary.Spec.Service.Port),
+			},
+			{
+				BackendRef: gwr.makeBackendRef(canarySvcName, 100, canary.Spec.Service.Port),
+			},
+		}
+	} else {
+		// If canary weight is 0 and SessionAffinityCookie is non-blank, then it belongs to a previous canary run.
+		if canary.Status.SessionAffinityCookie != "" {
+			canary.Status.PreviousSessionAffinityCookie = canary.Status.SessionAffinityCookie
+		}
+		previousCookie := canary.Status.PreviousSessionAffinityCookie
+
+		// Match against the previous session cookie and delete that cookie
+		if previousCookie != "" {
+			cookieKeyAndVal := strings.Split(previousCookie, "=")
+			regexMatchType := v1beta1.HeaderMatchRegularExpression
+			cookieMatch := v1beta1.HTTPRouteMatch{
+				Headers: []v1beta1.HTTPHeaderMatch{
+					{
+						Type:  &regexMatchType,
+						Name:  cookieHeader,
+						Value: fmt.Sprintf(".*%s.*%s.*", cookieKeyAndVal[0], cookieKeyAndVal[1]),
+					},
+				},
+			}
+			svcMatches, _ := gwr.mapRouteMatches(canary.Spec.Service.Match)
+			mergedMatches := gwr.mergeMatchConditions([]v1beta1.HTTPRouteMatch{cookieMatch}, svcMatches)
+			stickyRouteRule.Matches = mergedMatches
+
+			stickyRouteRule.Filters = append(stickyRouteRule.Filters, v1beta1.HTTPRouteFilter{
+				Type: v1beta1.HTTPRouteFilterResponseHeaderModifier,
+				ResponseHeaderModifier: &v1beta1.HTTPHeaderFilter{
+					Add: []v1beta1.HTTPHeader{
+						{
+							Name:  setCookieHeader,
+							Value: fmt.Sprintf("%s; %s=%d", previousCookie, maxAgeAttr, -1),
+						},
+					},
+				},
+			})
+		}
+
+		canary.Status.SessionAffinityCookie = ""
+	}
+
+	return []v1beta1.HTTPRouteRule{stickyRouteRule, *weightedRouteRule}, nil
+}
+
 func (gwr *GatewayAPIV1Beta1Router) mapRouteMatches(requestMatches []v1alpha3.HTTPMatchRequest) ([]v1beta1.HTTPRouteMatch, error) {
 	matches := []v1beta1.HTTPRouteMatch{}
 
@@ -389,6 +550,9 @@ func (gwr *GatewayAPIV1Beta1Router) mergeMatchConditions(analysis, service []v1b
 	if len(analysis) == 0 {
 		return service
 	}
+	if len(service) == 0 {
+		return analysis
+	}
 
 	merged := make([]v1beta1.HTTPRouteMatch, len(service)*len(analysis))
 	num := 0