From cf4988a3d4d5860b9d5b76976c3d9a62ae512b51 Mon Sep 17 00:00:00 2001 From: Dave Protasowski Date: Tue, 18 Jun 2024 15:40:05 -0400 Subject: [PATCH] Add envoy gateway testing (#738) * add envoy gateway * set gateway svc namespace * set the svc name of the gateway for testing reasons * test in kind * external gateway needs LB * install envoy-gateway gateways * include config to use the right gateways * fix no service test * disable host-rewrite - it uses K8s Service type=ExternalName * update envoy config to work with latest * add port 443 to the listener * include a test for the config changes --- .github/workflows/kind-e2e.yaml | 1 + hack/test-env.sh | 3 + test/e2e-common.sh | 41 ++++++++++++-- test/e2e/gateway_config_test.go | 2 + .../envoy-gateway-no-service-vis.yaml | 37 +++++++++++++ third_party/envoy-gateway/config-gateway.yaml | 38 +++++++++++++ third_party/envoy-gateway/config_test.go | 44 +++++++++++++++ third_party/envoy-gateway/external.yaml | 55 +++++++++++++++++++ third_party/envoy-gateway/internal.yaml | 45 +++++++++++++++ third_party/istio/300-gateway.yaml | 11 ++++ 10 files changed, 271 insertions(+), 6 deletions(-) create mode 100644 test/e2e/testdata/envoy-gateway-no-service-vis.yaml create mode 100644 third_party/envoy-gateway/config-gateway.yaml create mode 100644 third_party/envoy-gateway/config_test.go create mode 100644 third_party/envoy-gateway/external.yaml create mode 100644 third_party/envoy-gateway/internal.yaml diff --git a/.github/workflows/kind-e2e.yaml b/.github/workflows/kind-e2e.yaml index 3f11d9d44..765b196a4 100644 --- a/.github/workflows/kind-e2e.yaml +++ b/.github/workflows/kind-e2e.yaml @@ -16,6 +16,7 @@ jobs: ingress: - istio - contour + - envoy-gateway env: KO_DOCKER_REPO: kind.local diff --git a/hack/test-env.sh b/hack/test-env.sh index 28eb799ad..6e2d261e1 100755 --- a/hack/test-env.sh +++ b/hack/test-env.sh @@ -19,3 +19,6 @@ export ISTIO_VERSION="1.22.0" export ISTIO_UNSUPPORTED_E2E_TESTS="retry,httpoption" export CONTOUR_VERSION="v1.29.0" export CONTOUR_UNSUPPORTED_E2E_TESTS="httpoption" + +export ENVOY_GATEWAY_VERSION="latest" +export ENVOY_GATEWAY_UNSUPPORTED_E2E_TESTS="httpoption,host-rewrite" diff --git a/test/e2e-common.sh b/test/e2e-common.sh index c3b2c7357..2fb434920 100755 --- a/test/e2e-common.sh +++ b/test/e2e-common.sh @@ -53,6 +53,14 @@ function parse_flags() { readonly UNSUPPORTED_E2E_TESTS="${CONTOUR_UNSUPPORTED_E2E_TESTS}" return 1 ;; + --envoy-gateway) + readonly INGRESS=envoy-gateway + readonly GATEWAY_OVERRIDE=knative-external + readonly GATEWAY_NAMESPACE_OVERRIDE=envoy-gateway-system + readonly GATEWAY_CLASS=eg-external + readonly UNSUPPORTED_E2E_TESTS="${ENVOY_GATEWAY_UNSUPPORTED_E2E_TESTS}" + return 1 + ;; --kind) readonly KIND=1 return 1 @@ -106,6 +114,8 @@ function setup_networking() { if [[ "${INGRESS}" == "contour" ]]; then setup_contour + elif [[ "${INGRESS}" == "envoy-gateway" ]]; then + setup_envoy_gateway else setup_istio fi @@ -116,16 +126,23 @@ function teardown_networking() { kubectl delete -f "${REPO_ROOT_DIR}/third_party/gateway-api/gateway-api.yaml" if [[ "$INGRESS" == "contour" ]]; then - for file in ${CONTOUR_FILES[@]}; do - kubectl delete -f \ - "https://raw.githubusercontent.com/projectcontour/contour/${CONTOUR_VERSION}/${file}" - done + teardown_contour + elif [[ "${INGRESS}" == "envoy-gateway" ]]; then + teardown_envoy_gateway else - istioctl uninstall -y --purge - kubectl delete namespace istio-system + teardown_istio fi } +function setup_envoy_gateway() { + kubectl apply --server-side -f https://github.com/envoyproxy/gateway/releases/download/${ENVOY_GATEWAY_VERSION}/install.yaml + kubectl apply -f "${REPO_ROOT_DIR}/third_party/envoy-gateway" +} + +function teardown_envoy_gateway() { + kubectl delete -f https://github.com/envoyproxy/gateway/releases/download/${ENVOY_GATEWAY_VERSION}/install.yaml +} + function setup_contour() { # Version is selected is in $REPO_ROOT/hack/test-env.sh for file in ${CONTOUR_FILES[@]}; do @@ -142,6 +159,18 @@ function setup_contour() { fi } +function teardown_contour() { + for file in ${CONTOUR_FILES[@]}; do + kubectl delete -f \ + "https://raw.githubusercontent.com/projectcontour/contour/${CONTOUR_VERSION}/${file}" + done +} + +function teardown_istio() { + istioctl uninstall -y --purge + kubectl delete namespace istio-system +} + function setup_istio() { # Version is selected by ISTIO_VERSION that's source in $REPO_ROOT/hack/test-env.sh curl -L https://istio.io/downloadIstio | sh - && \ diff --git a/test/e2e/gateway_config_test.go b/test/e2e/gateway_config_test.go index e5f926fe1..04d6c92c4 100644 --- a/test/e2e/gateway_config_test.go +++ b/test/e2e/gateway_config_test.go @@ -55,6 +55,8 @@ func TestGatewayWithNoService(t *testing.T) { configGateway = ConfigMapFromTestFile(t, "testdata/contour-no-service-vis.yaml") case "istio": configGateway = ConfigMapFromTestFile(t, "testdata/istio-no-service-vis.yaml") + case "envoy-gateway": + configGateway = ConfigMapFromTestFile(t, "testdata/envoy-gateway-no-service-vis.yaml") case "default": t.Fatalf("value for INGRESS (%s) not supported", ingress) } diff --git a/test/e2e/testdata/envoy-gateway-no-service-vis.yaml b/test/e2e/testdata/envoy-gateway-no-service-vis.yaml new file mode 100644 index 000000000..9c9c6b674 --- /dev/null +++ b/test/e2e/testdata/envoy-gateway-no-service-vis.yaml @@ -0,0 +1,37 @@ +# Copyright 2024 The Knative Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: config-gateway + namespace: knative-serving + labels: + app.kubernetes.io/component: net-gateway-api + app.kubernetes.io/name: knative-serving + serving.knative.dev/release: devel +data: + external-gateways: | + - class: eg-external + gateway: eg-external/eg-external + supported-features: + - HTTPRouteRequestTimeout + + # local-gateways defines the Gateway to be used for cluster local traffic + local-gateways: | + - class: eg-internal + gateway: eg-internal/eg-internal + service: envoy-gateway-system/knative-internal + supported-features: + - HTTPRouteRequestTimeout diff --git a/third_party/envoy-gateway/config-gateway.yaml b/third_party/envoy-gateway/config-gateway.yaml new file mode 100644 index 000000000..98050a1c2 --- /dev/null +++ b/third_party/envoy-gateway/config-gateway.yaml @@ -0,0 +1,38 @@ +# Copyright 2024 The Knative Authors +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# https://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +apiVersion: v1 +kind: ConfigMap +metadata: + name: config-gateway + namespace: knative-serving + labels: + app.kubernetes.io/component: net-gateway-api + app.kubernetes.io/name: knative-serving + serving.knative.dev/release: devel +data: + external-gateways: | + - class: eg-external + gateway: eg-external/eg-external + service: envoy-gateway-system/knative-external + supported-features: + - HTTPRouteRequestTimeout + + # local-gateways defines the Gateway to be used for cluster local traffic + local-gateways: | + - class: eg-internal + gateway: eg-internal/eg-internal + service: envoy-gateway-system/knative-internal + supported-features: + - HTTPRouteRequestTimeout diff --git a/third_party/envoy-gateway/config_test.go b/third_party/envoy-gateway/config_test.go new file mode 100644 index 000000000..9628c24b5 --- /dev/null +++ b/third_party/envoy-gateway/config_test.go @@ -0,0 +1,44 @@ +/* +Copyright 2024 The Knative Authors + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package main + +import ( + "os" + "testing" + + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/yaml" + + "knative.dev/net-gateway-api/pkg/reconciler/ingress/config" +) + +func TestFromConfigMap(t *testing.T) { + bytes, err := os.ReadFile(config.GatewayConfigName + ".yaml") + if err != nil { + t.Fatalf("failed to read %q: %s", config.GatewayConfigName, err) + } + + cm := &corev1.ConfigMap{} + err = yaml.Unmarshal(bytes, cm) + if err != nil { + t.Fatalf("failed to unmarshal %q: %s", config.GatewayConfigName, err) + } + + if _, err := config.FromConfigMap(cm); err != nil { + t.Error("FromConfigMap(actual) =", err) + } +} diff --git a/third_party/envoy-gateway/external.yaml b/third_party/envoy-gateway/external.yaml new file mode 100644 index 000000000..12c5cd716 --- /dev/null +++ b/third_party/envoy-gateway/external.yaml @@ -0,0 +1,55 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: eg-external +--- +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyProxy +metadata: + name: knative-external-config + namespace: envoy-gateway-system +spec: + provider: + type: Kubernetes + kubernetes: + envoyService: + name: knative-external +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: eg-external +spec: + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parametersRef: + group: gateway.envoyproxy.io + kind: EnvoyProxy + name: knative-external-config + namespace: envoy-gateway-system +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: eg-external + namespace: eg-external +spec: + gatewayClassName: eg-external + listeners: + - name: http + port: 80 + protocol: HTTP + allowedRoutes: + namespaces: + from: All + # We've observed when adding and removing a listener on port 443 this + # causes the GKE LB to have downtime. By adding this tls listener + # we keep that LB port open. + - name: tls + port: 443 + protocol: TLS + tls: + mode: Passthrough + allowedRoutes: + namespaces: + from: All diff --git a/third_party/envoy-gateway/internal.yaml b/third_party/envoy-gateway/internal.yaml new file mode 100644 index 000000000..c786c94d6 --- /dev/null +++ b/third_party/envoy-gateway/internal.yaml @@ -0,0 +1,45 @@ +--- +apiVersion: v1 +kind: Namespace +metadata: + name: eg-internal +--- +apiVersion: gateway.envoyproxy.io/v1alpha1 +kind: EnvoyProxy +metadata: + name: knative-internal-config + namespace: envoy-gateway-system +spec: + provider: + type: Kubernetes + kubernetes: + envoyService: + type: ClusterIP + name: knative-internal +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: GatewayClass +metadata: + name: eg-internal +spec: + controllerName: gateway.envoyproxy.io/gatewayclass-controller + parametersRef: + group: gateway.envoyproxy.io + kind: EnvoyProxy + name: knative-internal-config + namespace: envoy-gateway-system +--- +apiVersion: gateway.networking.k8s.io/v1 +kind: Gateway +metadata: + name: eg-internal + namespace: eg-internal +spec: + gatewayClassName: eg-internal + listeners: + - name: http + port: 80 + protocol: HTTP + allowedRoutes: + namespaces: + from: All diff --git a/third_party/istio/300-gateway.yaml b/third_party/istio/300-gateway.yaml index bfc9d7326..6d0885ef6 100644 --- a/third_party/istio/300-gateway.yaml +++ b/third_party/istio/300-gateway.yaml @@ -29,3 +29,14 @@ spec: allowedRoutes: namespaces: from: All + # We've observed when adding and removing a listener on port 443 this + # causes the GKE LB to have downtime. By adding this tls listener + # we keep that LB port open. + - name: tls + port: 443 + protocol: TLS + tls: + mode: Passthrough + allowedRoutes: + namespaces: + from: All