feat: Support Client IP Detection using XFF on ClientTrafficPolicy (e…

…nvoyproxy#2535) * feat: HTTPConnectionManager settings for XFF headers on ClientTrafficPolicy Signed-off-by: David Alger <[email protected]> * Remove useRemoteAddress configuration and rename section to originalIpDetection Signed-off-by: David Alger <[email protected]> * Add example curl request demonstrating request headers Signed-off-by: David Alger <[email protected]> * Implement support for original ip detection extensions Signed-off-by: David Alger <[email protected]> * Rename to clientIP Signed-off-by: David Alger <[email protected]> * Update tests Signed-off-by: David Alger <[email protected]> * Update docs Signed-off-by: David Alger <[email protected]> * Refactor xff / custom_header implementation Signed-off-by: David Alger <[email protected]> * Update tests Signed-off-by: David Alger <[email protected]> * Strip out customHeader support for now Signed-off-by: David Alger <[email protected]> * Make optional Signed-off-by: David Alger <[email protected]> * Cleanup Signed-off-by: David Alger <[email protected]> --------- Signed-off-by: David Alger <[email protected]>
deszhou · Feb 3, 2024 · e3994ec · e3994ec
1 parent ca4657c
commit e3994ec
Show file tree

Hide file tree

Showing 17 changed files with 513 additions and 1 deletion.
diff --git a/api/v1alpha1/clienttrafficpolicy_types.go b/api/v1alpha1/clienttrafficpolicy_types.go
@@ -66,6 +66,10 @@ type ClientTrafficPolicySpec struct {
 	//
 	// +optional
 	EnableProxyProtocol *bool `json:"enableProxyProtocol,omitempty"`
+	// ClientIPDetectionSettings provides configuration for determining the original client IP address for requests.
+	//
+	// +optional
+	ClientIPDetection *ClientIPDetectionSettings `json:"clientIPDetection,omitempty"`
 	// HTTP3 provides HTTP/3 configuration on the listener.
 	//
 	// +optional
@@ -84,6 +88,25 @@ type ClientTrafficPolicySpec struct {
 	HTTP1 *HTTP1Settings `json:"http1,omitempty"`
 }
 
+// ClientIPDetectionSettings provides configuration for determining the original client IP address for requests.
+type ClientIPDetectionSettings struct {
+	// XForwardedForSettings provides configuration for using X-Forwarded-For headers for determining the client IP address.
+	//
+	// +optional
+	XForwardedFor *XForwardedForSettings `json:"xForwardedFor,omitempty"`
+}
+
+// XForwardedForSettings provides configuration for using X-Forwarded-For headers for determining the client IP address.
+type XForwardedForSettings struct {
+	// NumTrustedHops controls the number of additional ingress proxy hops from the right side of XFF HTTP
+	// headers to trust when determining the origin client's IP address.
+	// Refer to https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_conn_man/headers#x-forwarded-for
+	// for more details.
+	//
+	// +optional
+	NumTrustedHops *uint32 `json:"numTrustedHops,omitempty"`
+}
+
 // HTTP3Settings provides HTTP/3 configuration on the listener.
 type HTTP3Settings struct {
 }

diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go
diff --git a/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_clienttrafficpolicies.yaml b/charts/gateway-helm/crds/generated/gateway.envoyproxy.io_clienttrafficpolicies.yaml
@@ -44,6 +44,25 @@ spec:
           spec:
             description: Spec defines the desired state of ClientTrafficPolicy.
             properties:
+              clientIPDetection:
+                description: ClientIPDetectionSettings provides configuration for
+                  determining the original client IP address for requests.
+                properties:
+                  xForwardedFor:
+                    description: XForwardedForSettings provides configuration for
+                      using X-Forwarded-For headers for determining the client IP
+                      address.
+                    properties:
+                      numTrustedHops:
+                        description: NumTrustedHops controls the number of additional
+                          ingress proxy hops from the right side of XFF HTTP headers
+                          to trust when determining the origin client's IP address.
+                          Refer to https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_conn_man/headers#x-forwarded-for
+                          for more details.
+                        format: int32
+                        type: integer
+                    type: object
+                type: object
               enableProxyProtocol:
                 description: EnableProxyProtocol interprets the ProxyProtocol header
                   and adds the Client Address into the X-Forwarded-For header. Note

diff --git a/internal/gatewayapi/clienttrafficpolicy.go b/internal/gatewayapi/clienttrafficpolicy.go
@@ -292,6 +292,9 @@ func (t *Translator) translateClientTrafficPolicyForListener(policySpec *egv1a1.
 		// Translate Proxy Protocol
 		translateListenerProxyProtocol(policySpec.EnableProxyProtocol, httpIR)
 
+		// Translate Client IP Detection
+		translateClientIPDetection(policySpec.ClientIPDetection, httpIR)
+
 		// Translate Suppress Envoy Headers
 		translateListenerSuppressEnvoyHeaders(policySpec.SuppressEnvoyHeaders, httpIR)
 
@@ -375,6 +378,15 @@ func translateListenerProxyProtocol(enableProxyProtocol *bool, httpIR *ir.HTTPLi
 	}
 }
 
+func translateClientIPDetection(clientIPDetection *egv1a1.ClientIPDetectionSettings, httpIR *ir.HTTPListener) {
+	// Return early if not set
+	if clientIPDetection == nil {
+		return
+	}
+
+	httpIR.ClientIPDetection = (*ir.ClientIPDetectionSettings)(clientIPDetection)
+}
+
 func translateListenerSuppressEnvoyHeaders(suppressEnvoyHeaders *bool, httpIR *ir.HTTPListener) {
 	if suppressEnvoyHeaders != nil {
 		httpIR.SuppressEnvoyHeaders = *suppressEnvoyHeaders

diff --git a/internal/gatewayapi/testdata/clienttrafficpolicy-client-ip-detection.in.yaml b/internal/gatewayapi/testdata/clienttrafficpolicy-client-ip-detection.in.yaml
@@ -0,0 +1,30 @@
+clientTrafficPolicies:
+- apiVersion: gateway.envoyproxy.io/v1alpha1
+  kind: ClientTrafficPolicy
+  metadata:
+    namespace: envoy-gateway
+    name: target-gateway-1
+  spec:
+    clientIPDetection:
+      xForwardedFor:
+        numTrustedHops: 2
+    targetRef:
+      group: gateway.networking.k8s.io
+      kind: Gateway
+      name: gateway-1
+      namespace: envoy-gateway
+gateways:
+- apiVersion: gateway.networking.k8s.io/v1
+  kind: Gateway
+  metadata:
+    namespace: envoy-gateway
+    name: gateway-1
+  spec:
+    gatewayClassName: envoy-gateway-class
+    listeners:
+    - name: http-1
+      protocol: HTTP
+      port: 8081
+      allowedRoutes:
+        namespaces:
+          from: Same
diff --git a/internal/gatewayapi/testdata/clienttrafficpolicy-client-ip-detection.out.yaml b/internal/gatewayapi/testdata/clienttrafficpolicy-client-ip-detection.out.yaml
@@ -0,0 +1,98 @@
+clientTrafficPolicies:
+- apiVersion: gateway.envoyproxy.io/v1alpha1
+  kind: ClientTrafficPolicy
+  metadata:
+    creationTimestamp: null
+    name: target-gateway-1
+    namespace: envoy-gateway
+  spec:
+    clientIPDetection:
+      xForwardedFor:
+        numTrustedHops: 2
+    targetRef:
+      group: gateway.networking.k8s.io
+      kind: Gateway
+      name: gateway-1
+      namespace: envoy-gateway
+  status:
+    conditions:
+    - lastTransitionTime: null
+      message: ClientTrafficPolicy has been accepted.
+      reason: Accepted
+      status: "True"
+      type: Accepted
+gateways:
+- apiVersion: gateway.networking.k8s.io/v1
+  kind: Gateway
+  metadata:
+    creationTimestamp: null
+    name: gateway-1
+    namespace: envoy-gateway
+  spec:
+    gatewayClassName: envoy-gateway-class
+    listeners:
+    - allowedRoutes:
+        namespaces:
+          from: Same
+      name: http-1
+      port: 8081
+      protocol: HTTP
+  status:
+    listeners:
+    - attachedRoutes: 0
+      conditions:
+      - lastTransitionTime: null
+        message: Sending translated listener configuration to the data plane
+        reason: Programmed
+        status: "True"
+        type: Programmed
+      - lastTransitionTime: null
+        message: Listener has been successfully translated
+        reason: Accepted
+        status: "True"
+        type: Accepted
+      - lastTransitionTime: null
+        message: Listener references have been resolved
+        reason: ResolvedRefs
+        status: "True"
+        type: ResolvedRefs
+      name: http-1
+      supportedKinds:
+      - group: gateway.networking.k8s.io
+        kind: HTTPRoute
+      - group: gateway.networking.k8s.io
+        kind: GRPCRoute
+infraIR:
+  envoy-gateway/gateway-1:
+    proxy:
+      listeners:
+      - address: null
+        name: envoy-gateway/gateway-1/http-1
+        ports:
+        - containerPort: 8081
+          name: http-1
+          protocol: HTTP
+          servicePort: 8081
+      metadata:
+        labels:
+          gateway.envoyproxy.io/owning-gateway-name: gateway-1
+          gateway.envoyproxy.io/owning-gateway-namespace: envoy-gateway
+      name: envoy-gateway/gateway-1
+xdsIR:
+  envoy-gateway/gateway-1:
+    accessLog:
+      text:
+      - path: /dev/stdout
+    http:
+    - address: 0.0.0.0
+      clientIPDetection:
+        xForwardedFor:
+          numTrustedHops: 2
+      hostnames:
+      - '*'
+      isHTTP2: false
+      name: envoy-gateway/gateway-1/http-1
+      path:
+        escapedSlashesAction: UnescapeAndRedirect
+        mergeSlashes: true
+      port: 8081
diff --git a/internal/ir/xds.go b/internal/ir/xds.go
@@ -214,6 +214,8 @@ type HTTPListener struct {
 	SuppressEnvoyHeaders bool `json:"suppressEnvoyHeaders,omitempty" yaml:"suppressEnvoyHeaders,omitempty"`
 	// EnableProxyProtocol enables the listener to interpret proxy protocol header
 	EnableProxyProtocol bool `json:"enableProxyProtocol,omitempty" yaml:"enableProxyProtocol,omitempty"`
+	// ClientIPDetection controls how the original client IP address is determined for requests.
+	ClientIPDetection *ClientIPDetectionSettings `json:"clientIPDetection,omitempty" yaml:"clientIPDetection,omitempty"`
 	// HTTP3 provides HTTP/3 configuration on the listener.
 	// +optional
 	HTTP3 *HTTP3Settings `json:"http3,omitempty"`
@@ -340,6 +342,10 @@ type PathSettings struct {
 	EscapedSlashesAction PathEscapedSlashAction `json:"escapedSlashesAction" yaml:"escapedSlashesAction"`
 }
 
+// ClientIPDetectionSettings provides configuration for determining the original client IP address for requests.
+// +k8s:deepcopy-gen=true
+type ClientIPDetectionSettings egv1a1.ClientIPDetectionSettings
+
 // BackendWeights stores the weights of valid and invalid backends for the route so that 500 error responses can be returned in the same proportions
 type BackendWeights struct {
 	Valid   uint32 `json:"valid" yaml:"valid"`

diff --git a/internal/ir/zz_generated.deepcopy.go b/internal/ir/zz_generated.deepcopy.go
diff --git a/internal/xds/translator/listener.go b/internal/xds/translator/listener.go
@@ -24,6 +24,7 @@ import (
 	"github.com/golang/protobuf/ptypes/wrappers"
 	"google.golang.org/protobuf/types/known/anypb"
 	"google.golang.org/protobuf/types/known/wrapperspb"
+	"k8s.io/utils/ptr"
 
 	"github.com/envoyproxy/gateway/internal/ir"
 	"github.com/envoyproxy/gateway/internal/utils/protocov"
@@ -146,6 +147,13 @@ func (t *Translator) addXdsHTTPFilterChain(xdsListener *listenerv3.Listener, irL
 	} else {
 		statPrefix = "http"
 	}
+
+	// Client IP detection
+	var xffNumTrustedHops uint32
+	if irListener.ClientIPDetection != nil && irListener.ClientIPDetection.XForwardedFor != nil {
+		xffNumTrustedHops = ptr.Deref(irListener.ClientIPDetection.XForwardedFor.NumTrustedHops, 0)
+	}
+
 	mgr := &hcmv3.HttpConnectionManager{
 		AccessLog:  al,
 		CodecType:  hcmv3.HttpConnectionManager_AUTO,
@@ -162,7 +170,8 @@ func (t *Translator) addXdsHTTPFilterChain(xdsListener *listenerv3.Listener, irL
 		// Set it by default to also support HTTP1.1 to HTTP2 Upgrades
 		Http2ProtocolOptions: http2ProtocolOptions(),
 		// https://www.envoyproxy.io/docs/envoy/latest/configuration/http/http_conn_man/headers#x-forwarded-for
-		UseRemoteAddress: &wrappers.BoolValue{Value: true},
+		UseRemoteAddress:  &wrappers.BoolValue{Value: true},
+		XffNumTrustedHops: xffNumTrustedHops,
 		// normalize paths according to RFC 3986
 		NormalizePath:                &wrapperspb.BoolValue{Value: true},
 		MergeSlashes:                 irListener.Path.MergeSlashes,

diff --git a/internal/xds/translator/testdata/in/xds-ir/client-ip-detection.yaml b/internal/xds/translator/testdata/in/xds-ir/client-ip-detection.yaml
@@ -0,0 +1,18 @@
+http:
+- name: "first-listener"
+  address: "0.0.0.0"
+  port: 8081
+  hostnames:
+  - "*"
+  routes:
+  - name: "first-route"
+    hostname: "*"
+    destination:
+      name: "first-route-dest"
+      settings:
+      - endpoints:
+        - host: "1.1.1.1"
+          port: 8081
+  clientIPDetection:
+    xForwardedFor:
+      numTrustedHops: 2