feat: add feature flag to enable/disable Gateway agent service (#88)

* feat: add feature flag to enable/disable Gateway agent service Signed-off-by: Lin Yang <[email protected]> * fix: import golang.org/x/exp/slices instead of slices Signed-off-by: Lin Yang <[email protected]> * fix: make manifests Signed-off-by: Lin Yang <[email protected]> --------- Signed-off-by: Lin Yang <[email protected]>
flomesh-io · Oct 27, 2023 · fdfcf0c · fdfcf0c
1 parent 1c67658
commit fdfcf0c
Show file tree

Hide file tree

Showing 12 changed files with 157 additions and 73 deletions.
diff --git a/charts/fsm/README.md b/charts/fsm/README.md
@@ -142,6 +142,7 @@ The following table lists the configurable parameters of the fsm chart and their
 | fsm.featureFlags.enableAsyncProxyServiceMapping | bool | `false` | Enable async proxy-service mapping |
 | fsm.featureFlags.enableAutoDefaultRoute | bool | `false` | Enable AutoDefaultRoute |
 | fsm.featureFlags.enableEgressPolicy | bool | `true` | Enable FSM's Egress policy API. When enabled, fine grained control over Egress (external) traffic is enforced |
+| fsm.featureFlags.enableGatewayAgentService | bool | `false` | Enable Gateway Agent Service |
 | fsm.featureFlags.enableIngressBackendPolicy | bool | `true` | Enables FSM's IngressBackend policy API. When enabled, FSM will use the IngressBackend API allow ingress traffic to mesh backends |
 | fsm.featureFlags.enableMeshRootCertificate | bool | `false` | Enable the MeshRootCertificate to configure the FSM certificate provider |
 | fsm.featureFlags.enablePluginPolicy | bool | `false` | Enable Plugin Policy for extend |

diff --git a/charts/fsm/templates/preset-mesh-config.yaml b/charts/fsm/templates/preset-mesh-config.yaml
@@ -88,7 +88,8 @@ data:
         "enableValidateGatewayListenerHostname": {{.Values.fsm.featureFlags.enableValidateGatewayListenerHostname | mustToJson}},
         "enableValidateHTTPRouteHostnames": {{.Values.fsm.featureFlags.enableValidateHTTPRouteHostnames | mustToJson}},
         "enableValidateGRPCRouteHostnames": {{.Values.fsm.featureFlags.enableValidateGRPCRouteHostnames | mustToJson}},
-        "enableValidateTLSRouteHostnames": {{.Values.fsm.featureFlags.enableValidateTLSRouteHostnames | mustToJson}}
+        "enableValidateTLSRouteHostnames": {{.Values.fsm.featureFlags.enableValidateTLSRouteHostnames | mustToJson}},
+        "enableGatewayAgentService": {{.Values.fsm.featureFlags.enableGatewayAgentService | mustToJson}}
       },
       "pluginChains": {{.Values.fsm.pluginChains | mustToJson }},
       "ingress": {

diff --git a/charts/fsm/values.schema.json b/charts/fsm/values.schema.json
@@ -2017,7 +2017,8 @@
                         "enableValidateGatewayListenerHostname",
                         "enableValidateHTTPRouteHostnames",
                         "enableValidateGRPCRouteHostnames",
-                        "enableValidateTLSRouteHostnames"
+                        "enableValidateTLSRouteHostnames",
+                        "enableGatewayAgentService"
                     ],
                     "properties": {
                         "enableEgressPolicy": {
@@ -2154,6 +2155,15 @@
                           "examples": [
                             true
                           ]
+                        },
+                        "enableGatewayAgentService": {
+                          "$id": "#/properties/fsm/properties/featureFlags/properties/enableGatewayAgentService",
+                          "type": "boolean",
+                          "title": "Enable Agent Service",
+                          "description": "Enable Agent Service",
+                          "examples": [
+                            false
+                          ]
                         }
                     },
                     "additionalProperties": false

diff --git a/charts/fsm/values.yaml b/charts/fsm/values.yaml
@@ -776,6 +776,8 @@ fsm:
     enableValidateGRPCRouteHostnames: true
     # -- Enable validate TLS route hostnames, enforce the hostname is DNS name not IP address
     enableValidateTLSRouteHostnames: true
+    # -- Enable Gateway Agent Service
+    enableGatewayAgentService: false
 
   # -- Node tolerations applied to control plane pods.
   # The specified tolerations allow pods to schedule onto nodes with matching taints.

diff --git a/cmd/fsm-bootstrap/crds/config.flomesh.io_meshconfigs.yaml b/cmd/fsm-bootstrap/crds/config.flomesh.io_meshconfigs.yaml
@@ -1361,6 +1361,10 @@ spec:
                     description: EnableEgressPolicy defines if FSM's Egress policy
                       is enabled.
                     type: boolean
+                  enableGatewayAgentService:
+                    description: EnableGatewayAgentService defines if agent service
+                      is enabled.
+                    type: boolean
                   enableIngressBackendPolicy:
                     description: EnableIngressBackendPolicy defines if FSM will use
                       the IngressBackend API to allow ingress traffic to service mesh
@@ -1402,6 +1406,7 @@ spec:
                 - enableAsyncProxyServiceMapping
                 - enableAutoDefaultRoute
                 - enableEgressPolicy
+                - enableGatewayAgentService
                 - enableIngressBackendPolicy
                 - enablePluginPolicy
                 - enableRetryPolicy

diff --git a/go.mod b/go.mod
@@ -93,6 +93,7 @@ require (
 	github.com/pkg/errors v0.9.1
 	github.com/tidwall/gjson v1.14.4
 	github.com/tidwall/sjson v1.2.5
+	golang.org/x/exp v0.0.0-20220827204233-334a2380cb91
 	k8s.io/kubectl v0.26.6
 	k8s.io/kubernetes v1.26.6
 	sigs.k8s.io/yaml v1.3.0

diff --git a/go.sum b/go.sum
@@ -1661,6 +1661,8 @@ golang.org/x/exp v0.0.0-20191227195350-da58074b4299/go.mod h1:2RIsYlXP63K8oxa1u0
 golang.org/x/exp v0.0.0-20200119233911-0405dc783f0a/go.mod h1:2RIsYlXP63K8oxa1u096TMicItID8zy7Y6sNkU49FU4=
 golang.org/x/exp v0.0.0-20200207192155-f17229e696bd/go.mod h1:J/WKrq2StrnmMY6+EHIKF9dgMWnmCNThgcyBT1FY9mM=
 golang.org/x/exp v0.0.0-20200224162631-6cc2880d07d6/go.mod h1:3jZMyOhIsHpP37uCMkUooju7aAi5cS1Q23tOzKc+0MU=
+golang.org/x/exp v0.0.0-20220827204233-334a2380cb91 h1:tnebWN09GYg9OLPss1KXj8txwZc6X6uMr6VFdcGNbHw=
+golang.org/x/exp v0.0.0-20220827204233-334a2380cb91/go.mod h1:cyybsKvd6eL0RnXn6p/Grxp8F5bW7iYuBgsNCOHpMYE=
 golang.org/x/image v0.0.0-20190227222117-0694c2d4d067/go.mod h1:kZ7UVZpmo3dzQBMxlp+ypCbDeSB+sBbTgSJuh5dn5js=
 golang.org/x/image v0.0.0-20190802002840-cff245a6509b/go.mod h1:FeLwcggjj3mMvU+oOTbSwawSJRM1uh48EjtB4UJZlP0=
 golang.org/x/lint v0.0.0-20181026193005-c67002cb31c3/go.mod h1:UVdnD1Gm6xHRNCYTkRU2/jEulfH38KcIWyp/GAMgvoE=

diff --git a/pkg/apis/config/v1alpha3/mesh_config.go b/pkg/apis/config/v1alpha3/mesh_config.go
@@ -357,6 +357,9 @@ type FeatureFlags struct {
 
 	// EnableValidateTCPRouteHostnames defines if validate tcp route hostnames is enabled.
 	EnableValidateTLSRouteHostnames bool `json:"enableValidateTLSRouteHostnames"`
+
+	// EnableGatewayAgentService defines if agent service is enabled.
+	EnableGatewayAgentService bool `json:"enableGatewayAgentService"`
 }
 
 // SidecarDriverSpec is the type to represent FSM's sidecar driver define.

diff --git a/pkg/gateway/cache/config.go b/pkg/gateway/cache/config.go
@@ -52,7 +52,7 @@ func (c *GatewayCache) BuildConfigs() {
 			Listeners:  listenerCfg,
 			RouteRules: rules,
 			Services:   svcConfigs,
-			Chains:     chains(),
+			Chains:     c.chains(),
 		}
 		configSpec.Version = utils.SimpleHash(configSpec)
 		configs[ns] = configSpec
@@ -577,77 +577,23 @@ func (c *GatewayCache) serviceConfigs(services map[string]serviceInfo) map[strin
 	return configs
 }
 
-func chains() routecfg.Chains {
+func (c *GatewayCache) chains() routecfg.Chains {
+	if c.cfg.GetFeatureFlags().EnableGatewayAgentService {
+		return routecfg.Chains{
+			HTTPRoute:      insertAgentServiceScript(defaultHTTPChains),
+			HTTPSRoute:     insertAgentServiceScript(defaultHTTPSChains),
+			TLSPassthrough: defaultTLSPassthroughChains,
+			TLSTerminate:   defaultTLSTerminateChains,
+			TCPRoute:       defaultTCPChains,
+		}
+	}
+
 	return routecfg.Chains{
-		HTTPRoute: []string{
-			"common/access-control.js",
-			"common/ratelimit.js",
-			"common/consumer.js",
-			"http/codec.js",
-			"extension/agent-service.js",
-			"http/access-log.js",
-			"http/auth.js",
-			"http/route.js",
-			"http/fault-injection.js",
-			"filter/request-redirect.js",
-			"filter/header-modifier.js",
-			"filter/url-rewrite.js",
-			"http/service.js",
-			"http/metrics.js",
-			"http/tracing.js",
-			"http/logging.js",
-			"http/circuit-breaker.js",
-			"http/throttle-domain.js",
-			"http/throttle-route.js",
-			"http/error-page.js",
-			"http/proxy-redirect.js",
-			"http/forward.js",
-			"http/default.js",
-		},
-		HTTPSRoute: []string{
-			"common/access-control.js",
-			"common/ratelimit.js",
-			"common/tls-termination.js",
-			"common/consumer.js",
-			"http/codec.js",
-			"extension/agent-service.js",
-			"http/access-log.js",
-			"http/auth.js",
-			"http/route.js",
-			"http/fault-injection.js",
-			"filter/request-redirect.js",
-			"filter/header-modifier.js",
-			"filter/url-rewrite.js",
-			"http/service.js",
-			"http/metrics.js",
-			"http/tracing.js",
-			"http/logging.js",
-			"http/circuit-breaker.js",
-			"http/throttle-domain.js",
-			"http/throttle-route.js",
-			"http/error-page.js",
-			"http/proxy-redirect.js",
-			"http/forward.js",
-			"http/default.js",
-		},
-		TLSPassthrough: []string{
-			"common/access-control.js",
-			"common/ratelimit.js",
-			"tls/passthrough.js",
-			"common/consumer.js",
-		},
-		TLSTerminate: []string{
-			"common/access-control.js",
-			"common/ratelimit.js",
-			"common/tls-termination.js",
-			"common/consumer.js",
-			"tls/forward.js",
-		},
-		TCPRoute: []string{
-			"common/access-control.js",
-			"common/ratelimit.js",
-			"tcp/forward.js",
-		},
+		HTTPRoute:      defaultHTTPChains,
+		HTTPSRoute:     defaultHTTPSChains,
+		TLSPassthrough: defaultTLSPassthroughChains,
+		TLSTerminate:   defaultTLSTerminateChains,
+		TCPRoute:       defaultTCPChains,
 	}
 }
 

diff --git a/pkg/gateway/cache/types.go b/pkg/gateway/cache/types.go
@@ -111,3 +111,82 @@ type endpointInfo struct {
 var (
 	log = logger.New("fsm-gateway/cache")
 )
+
+var (
+	defaultHTTPChains = []string{
+		"common/access-control.js",
+		"common/ratelimit.js",
+		"common/consumer.js",
+		"http/codec.js",
+		"http/access-log.js",
+		"http/auth.js",
+		"http/route.js",
+		"http/fault-injection.js",
+		"filter/request-redirect.js",
+		"filter/header-modifier.js",
+		"filter/url-rewrite.js",
+		"http/service.js",
+		"http/metrics.js",
+		"http/tracing.js",
+		"http/logging.js",
+		"http/circuit-breaker.js",
+		"http/throttle-domain.js",
+		"http/throttle-route.js",
+		"http/error-page.js",
+		"http/proxy-redirect.js",
+		"http/forward.js",
+		"http/default.js",
+	}
+
+	defaultHTTPSChains = []string{
+		"common/access-control.js",
+		"common/ratelimit.js",
+		"common/tls-termination.js",
+		"common/consumer.js",
+		"http/codec.js",
+		"http/access-log.js",
+		"http/auth.js",
+		"http/route.js",
+		"http/fault-injection.js",
+		"filter/request-redirect.js",
+		"filter/header-modifier.js",
+		"filter/url-rewrite.js",
+		"http/service.js",
+		"http/metrics.js",
+		"http/tracing.js",
+		"http/logging.js",
+		"http/circuit-breaker.js",
+		"http/throttle-domain.js",
+		"http/throttle-route.js",
+		"http/error-page.js",
+		"http/proxy-redirect.js",
+		"http/forward.js",
+		"http/default.js",
+	}
+
+	defaultTLSPassthroughChains = []string{
+		"common/access-control.js",
+		"common/ratelimit.js",
+		"tls/passthrough.js",
+		"common/consumer.js",
+	}
+
+	defaultTLSTerminateChains = []string{
+		"common/access-control.js",
+		"common/ratelimit.js",
+		"common/tls-termination.js",
+		"common/consumer.js",
+		"tls/forward.js",
+	}
+
+	defaultTCPChains = []string{
+		"common/access-control.js",
+		"common/ratelimit.js",
+		"tcp/forward.js",
+	}
+)
+
+const (
+	httpCodecScript    = "http/codec.js"
+	agentServiceScript = "extension/agent-service.js"
+)
diff --git a/pkg/gateway/cache/utils.go b/pkg/gateway/cache/utils.go
@@ -3,6 +3,8 @@ package cache
 import (
 	"fmt"
 
+	"golang.org/x/exp/slices"
+
 	"sigs.k8s.io/controller-runtime/pkg/client"
 
 	corev1 "k8s.io/api/core/v1"
@@ -564,3 +566,12 @@ func toFSMPortNumber(port *gwv1beta1.PortNumber) *int32 {
 
 	return pointer.Int32(int32(*port))
 }
+
+func insertAgentServiceScript(chains []string) []string {
+	httpCodecIndex := slices.Index(chains, httpCodecScript)
+	if httpCodecIndex != -1 {
+		return slices.Insert(chains, httpCodecIndex+1, agentServiceScript)
+	}
+
+	return chains
+}
diff --git a/pkg/messaging/broker.go b/pkg/messaging/broker.go
@@ -806,11 +806,34 @@ func getGatewayUpdateEvent(msg events.PubSubMessage) *gatewayUpdateEvent {
 			msg:   msg,
 			topic: announcements.GatewayUpdate.String(),
 		}
+	case announcements.MeshConfigUpdated:
+		return gatewayInterestedConfigChanged(msg)
 	default:
 		return nil
 	}
 }
 
+func gatewayInterestedConfigChanged(msg events.PubSubMessage) *gatewayUpdateEvent {
+	prevMeshConfig, okPrevCast := msg.OldObj.(*configv1alpha3.MeshConfig)
+	newMeshConfig, okNewCast := msg.NewObj.(*configv1alpha3.MeshConfig)
+	if !okPrevCast || !okNewCast {
+		log.Error().Msgf("Expected MeshConfig type, got previous=%T, new=%T", okPrevCast, okNewCast)
+		return nil
+	}
+	prevSpec := prevMeshConfig.Spec
+	newSpec := newMeshConfig.Spec
+
+	if prevSpec.GatewayAPI.LogLevel != newSpec.GatewayAPI.LogLevel ||
+		prevSpec.FeatureFlags.EnableGatewayAgentService != newSpec.FeatureFlags.EnableGatewayAgentService {
+		return &gatewayUpdateEvent{
+			msg:   msg,
+			topic: announcements.GatewayUpdate.String(),
+		}
+	}
+
+	return nil
+}
+
 // getMCSUpdateEvent returns a mcsUpdateEvent type indicating whether the given PubSubMessage should
 // result in a gateway configuration update on an appropriate topic. Nil is returned if the PubSubMessage
 // does not result in a gateway update event.