Skip to content

Commit

Permalink
[release-v1.15] Add missing changes from knative#702 (knative#871)
Browse files Browse the repository at this point in the history
* Add missing changes from openshift-knative#702

* Fix e2e-tls tests
  • Loading branch information
ReToCode authored Sep 23, 2024
1 parent 2b10f33 commit 34e65a6
Show file tree
Hide file tree
Showing 5 changed files with 145 additions and 34 deletions.
2 changes: 1 addition & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -39,7 +39,7 @@ perf-tests:
.PHONY: perf-tests

test-e2e-tls:
ENABLE_INTERNAL_TLS="true" ./openshift/e2e-tests.sh
ENABLE_TLS="true" ./openshift/e2e-tests.sh
.PHONY: test-e2e-tls

# Target used by github actions.
Expand Down
77 changes: 49 additions & 28 deletions openshift/e2e-common.sh
Original file line number Diff line number Diff line change
Expand Up @@ -130,11 +130,9 @@ function install_serverless(){
export GOPATH=/tmp/go
export ON_CLUSTER_BUILDS=true
export DOCKER_REPO_OVERRIDE=image-registry.openshift-image-registry.svc:5000/openshift-marketplace
#TODO: enable back when we have the feature ready again downstream
sed -i 's/internal-encryption: "true"/internal-encryption: "false"/g' ./test/v1beta1/resources/operator.knative.dev_v1beta1_knativeserving_cr.yaml
OPENSHIFT_CI="true" make generated-files images install-serving || return $?

# Create a secret for https test.
# Ensure tests trust the OpenShift router CA
trust_router_ca || return $?
popd
}
Expand All @@ -150,30 +148,36 @@ function install_knative(){
wait_until_service_has_external_ip $SERVING_INGRESS_NAMESPACE kourier || fail_test "Ingress has no external IP"
wait_until_hostname_resolves "$(kubectl get svc -n $SERVING_INGRESS_NAMESPACE kourier -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')"

# TODO: Only one cluster enables internal-tls but it should be enabled by default when the feature is stable.
if [[ ${ENABLE_INTERNAL_TLS:-} == "true" ]]; then
configure_cm network internal-encryption:true || fail_test
# As config-kourier is in ingress namespace, don't use configure_cm.
oc patch knativeserving knative-serving \
-n "${SERVING_NAMESPACE}" \
--type merge --patch '{"spec": {"config": {"kourier": {"cluster-cert-secret": "server-certs"}}}}'
# Deploy certificates for testing TLS with cluster-local gateway
timeout 600 '[[ $(oc get ns $SERVING_INGRESS_NAMESPACE -oname | wc -l) == 0 ]]' || return 1
yq read --doc 1 ./test/config/tls/cert-secret.yaml | yq write - metadata.namespace ${SERVING_INGRESS_NAMESPACE} | oc apply -f -
if [[ ${ENABLE_TLS:-} == "true" ]]; then
configure_cm network system-internal-tls:enabled || fail_test
configure_cm network cluster-local-domain-tls:enabled || fail_test

echo "Restart controller to enable cert-manager integration"
oc delete pod -n ${SERVING_NAMESPACE} -l app=controller
oc wait --timeout=60s --for=condition=Available deployment -n ${SERVING_NAMESPACE} controller

echo "Restart activator to mount the certificates"
oc delete pod -n ${SERVING_NAMESPACE} -l app=activator
oc wait --timeout=60s --for=condition=Available deployment -n ${SERVING_NAMESPACE} activator
echo "internal-encryption is enabled"

echo "cluster-local-domain-tls and system-internal-tls are ENABLED"
else
# disable internal-encryption. S-O repo would enable by default.
configure_cm network internal-encryption:false || fail_test
configure_cm network system-internal-tls:disabled || fail_test
configure_cm network cluster-local-domain-tls:disabled || fail_test

echo "Restart activator to unmount the certificates"
oc delete pod -n ${SERVING_NAMESPACE} -l app=activator
oc wait --timeout=60s --for=condition=Available deployment -n ${SERVING_NAMESPACE} activator
echo "internal-encryption is disabled"

echo "Restart controller to disable cert-manager integration"
oc delete pod -n ${SERVING_NAMESPACE} -l app=controller
oc wait --timeout=60s --for=condition=Available deployment -n ${SERVING_NAMESPACE} controller

echo "cluster-local-domain-tls and system-internal-tls are DISABLED"
fi

header "Knative Installed successfully"
header "Successfully installed Knative"
}

function prepare_knative_serving_tests_nightly {
Expand All @@ -195,15 +199,6 @@ function prepare_knative_serving_tests_nightly {
export GATEWAY_OVERRIDE=kourier
export GATEWAY_NAMESPACE_OVERRIDE="$SERVING_INGRESS_NAMESPACE"
export INGRESS_CLASS=kourier.ingress.networking.knative.dev

if [[ ${ENABLE_INTERNAL_TLS} == "true" ]]; then
# Deploy CA cert for testing TLS with cluster-local gateway
yq read --doc 0 ./test/config/tls/cert-secret.yaml | oc apply -f -
# This needs to match the name of Secret in test/config/tls/cert-secret.yaml
export CA_CERT=ca-cert
# This needs to match $san from test/config/tls/generate.sh
export SERVER_NAME=knative.dev
fi
}

function run_e2e_tests(){
Expand Down Expand Up @@ -232,7 +227,7 @@ function run_e2e_tests(){
sleep 30
subdomain=$(oc get ingresses.config.openshift.io cluster -o jsonpath="{.spec.domain}")

readonly OPENSHIFT_TEST_OPTIONS="--kubeconfig $KUBECONFIG --enable-beta --enable-alpha --resolvabledomain --customdomain=$subdomain --https --skip-cleanup-on-fail"
readonly OPENSHIFT_TEST_OPTIONS="--kubeconfig $KUBECONFIG --enable-beta --enable-alpha --resolvabledomain --customdomain=$subdomain --ingress-class=${INGRESS_CLASS} --https --skip-cleanup-on-fail"

# Enable secure pod defaults for all tests.
enable_feature_flags secure-pod-defaults || fail_test
Expand All @@ -254,7 +249,7 @@ function run_e2e_tests(){
parallel=2
fi

go_test_e2e -tags=e2e -timeout=30m -parallel=$parallel \
go_test_e2e -tags=e2e -timeout=40m -parallel=$parallel \
./test/e2e ./test/conformance/api/... ./test/conformance/runtime/... \
--imagetemplate "$TEST_IMAGE_TEMPLATE" \
${OPENSHIFT_TEST_OPTIONS} || failed=1
Expand All @@ -265,6 +260,23 @@ function run_e2e_tests(){
${OPENSHIFT_TEST_OPTIONS} || failed=1
disable_feature_flags tag-header-based-routing || fail_test

if [[ ${ENABLE_TLS:-} == "true" ]]; then
go_test_e2e -timeout=5m ./test/e2e/clusterlocaldomaintls \
--imagetemplate "$TEST_IMAGE_TEMPLATE" \
${OPENSHIFT_TEST_OPTIONS} || failed=1

# get existing request-log-template
existingTemplate=$(oc get cm -n "${SYSTEM_NAMESPACE}" config-observability -o jsonpath='{.data.logging\.request-log-template}' | sed 's/\"/\\"/g')
patch_request_log_template "TLS: {{.Request.TLS}}" || fail_test

go_test_e2e -timeout=5m ./test/e2e/systeminternaltls \
--imagetemplate "$TEST_IMAGE_TEMPLATE" \
${OPENSHIFT_TEST_OPTIONS} || failed=1

# restore request-log-template
patch_request_log_template "$existingTemplate" || fail_test
fi

configure_cm autoscaler allow-zero-initial-scale:true || fail_test
# wait 10 sec until sync.
sleep 10
Expand Down Expand Up @@ -385,6 +397,15 @@ function run_e2e_tests(){
return $failed
}

function patch_request_log_template {
# do not use configure_cm as it would split on the :
local failed=0
oc -n ${SERVING_NAMESPACE} patch knativeserving/knative-serving --type=merge \
--patch="{\"spec\": {\"config\": { \"observability\": {\"logging.request-log-template\": \"$1\" }}}}" || failed=1
timeout 30 "[[ ! \$(oc get cm -n ${SERVING_NAMESPACE} config-observability -o jsonpath='{.data.logging\.request-log-template}') == \"$1\" ]]" || failed=1
return $failed
}

function gather_knative_state {
logger.info 'Gather knative state'
local gather_dir="${ARTIFACT_DIR:-/tmp}/gather-knative"
Expand Down
4 changes: 1 addition & 3 deletions openshift/e2e-tests.sh
Original file line number Diff line number Diff line change
Expand Up @@ -3,13 +3,11 @@
# shellcheck disable=SC1090
source "$(dirname "$0")/e2e-common.sh"

set -x

env

failed=0

export ENABLE_INTERNAL_TLS="${ENABLE_INTERNAL_TLS:-false}"
export ENABLE_TLS="${ENABLE_TLS:-false}"

(( !failed )) && install_knative || failed=1
(( !failed )) && prepare_knative_serving_tests_nightly || failed=2
Expand Down
67 changes: 67 additions & 0 deletions openshift/patches/014-ocp-system-internal-tls-test.patch
Original file line number Diff line number Diff line change
@@ -0,0 +1,67 @@
diff --git a/test/e2e/systeminternaltls/system_internal_tls_test.go b/test/e2e/systeminternaltls/system_internal_tls_test.go
--- a/test/e2e/systeminternaltls/system_internal_tls_test.go (revision 9d0fa738d33402b893659610ddf4d4635cfa61f4)
+++ b/test/e2e/systeminternaltls/system_internal_tls_test.go (revision 4f646561395fb011368360e35365f8664383cdc5)
@@ -113,6 +113,7 @@

// TestTLSCertificateRotation tests certificate rotation and automatic reloading of certs.
func TestTLSCertificateRotation(t *testing.T) {
+
if !test.ServingFlags.EnableAlphaFeatures {
t.Skip("Alpha features not enabled")
}
@@ -162,6 +163,7 @@

t.Log("Creating ConfigMap with old and new CA certs")
systemNS := os.Getenv(system.NamespaceEnvKey)
+ ingressNS := os.Getenv(test.GatewayNamespaceOverride)

// Create ConfigMap with networking.knative.dev/trust-bundle label in required namespaces
cm := &corev1.ConfigMap{
@@ -179,7 +181,15 @@
_, err = clients.KubeClient.CoreV1().ConfigMaps(systemNS).
Create(context.Background(), cm, v1.CreateOptions{})
if err != nil {
- t.Fatal("Failed to create configmap:", err)
+ t.Fatal("Failed to create configmap in "+systemNS, err)
+ }
+
+ if ingressNS != "" && systemNS != ingressNS {
+ _, err = clients.KubeClient.CoreV1().ConfigMaps(ingressNS).
+ Create(context.Background(), cm, v1.CreateOptions{})
+ if err != nil {
+ t.Fatal("Failed to create configmap in "+ingressNS, err)
+ }
}

// Clean up on test failure or interrupt
@@ -187,7 +197,14 @@
test.TearDown(clients, &names)
if err := clients.KubeClient.CoreV1().ConfigMaps(systemNS).
Delete(context.Background(), cm.Name, v1.DeleteOptions{}); err != nil && !errors.IsNotFound(err) {
- t.Fatal("Failed to delete configmap:", err)
+ t.Fatal("Failed to delete configmap in "+systemNS, err)
+ }
+
+ if ingressNS != "" && systemNS != ingressNS {
+ if err := clients.KubeClient.CoreV1().ConfigMaps(ingressNS).
+ Delete(context.Background(), cm.Name, v1.DeleteOptions{}); err != nil && !errors.IsNotFound(err) {
+ t.Fatal("Failed to delete configmap in "+ingressNS, err)
+ }
}
})

@@ -218,6 +235,14 @@
if err := clients.KubeClient.CoreV1().Secrets(systemNS).Delete(context.Background(), config.ServingRoutingCertName, v1.DeleteOptions{}); err != nil {
t.Fatalf("Failed to delete secret %s in system namespace: %v", config.ServingRoutingCertName, err)
}
+
+ if ingressNS != "" && systemNS != ingressNS {
+ t.Log("Deleting secret in ingress namespace")
+ if err := clients.KubeClient.CoreV1().Secrets(ingressNS).Delete(context.Background(), config.ServingRoutingCertName, v1.DeleteOptions{}); err != nil {
+ t.Fatalf("Failed to delete secret %s in ingress namespace: %v", config.ServingRoutingCertName, err)
+ }
+ }
+
checkEndpointState(t, clients, url)
}

29 changes: 27 additions & 2 deletions test/e2e/systeminternaltls/system_internal_tls_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -113,6 +113,7 @@ func TestSystemInternalTLS(t *testing.T) {

// TestTLSCertificateRotation tests certificate rotation and automatic reloading of certs.
func TestTLSCertificateRotation(t *testing.T) {

if !test.ServingFlags.EnableAlphaFeatures {
t.Skip("Alpha features not enabled")
}
Expand Down Expand Up @@ -162,6 +163,7 @@ func TestTLSCertificateRotation(t *testing.T) {

t.Log("Creating ConfigMap with old and new CA certs")
systemNS := os.Getenv(system.NamespaceEnvKey)
ingressNS := os.Getenv(test.GatewayNamespaceOverride)

// Create ConfigMap with networking.knative.dev/trust-bundle label in required namespaces
cm := &corev1.ConfigMap{
Expand All @@ -179,15 +181,30 @@ func TestTLSCertificateRotation(t *testing.T) {
_, err = clients.KubeClient.CoreV1().ConfigMaps(systemNS).
Create(context.Background(), cm, v1.CreateOptions{})
if err != nil {
t.Fatal("Failed to create configmap:", err)
t.Fatal("Failed to create configmap in "+systemNS, err)
}

if ingressNS != "" && systemNS != ingressNS {
_, err = clients.KubeClient.CoreV1().ConfigMaps(ingressNS).
Create(context.Background(), cm, v1.CreateOptions{})
if err != nil {
t.Fatal("Failed to create configmap in "+ingressNS, err)
}
}

// Clean up on test failure or interrupt
test.EnsureCleanup(t, func() {
test.TearDown(clients, &names)
if err := clients.KubeClient.CoreV1().ConfigMaps(systemNS).
Delete(context.Background(), cm.Name, v1.DeleteOptions{}); err != nil && !errors.IsNotFound(err) {
t.Fatal("Failed to delete configmap:", err)
t.Fatal("Failed to delete configmap in "+systemNS, err)
}

if ingressNS != "" && systemNS != ingressNS {
if err := clients.KubeClient.CoreV1().ConfigMaps(ingressNS).
Delete(context.Background(), cm.Name, v1.DeleteOptions{}); err != nil && !errors.IsNotFound(err) {
t.Fatal("Failed to delete configmap in "+ingressNS, err)
}
}
})

Expand Down Expand Up @@ -218,6 +235,14 @@ func TestTLSCertificateRotation(t *testing.T) {
if err := clients.KubeClient.CoreV1().Secrets(systemNS).Delete(context.Background(), config.ServingRoutingCertName, v1.DeleteOptions{}); err != nil {
t.Fatalf("Failed to delete secret %s in system namespace: %v", config.ServingRoutingCertName, err)
}

if ingressNS != "" && systemNS != ingressNS {
t.Log("Deleting secret in ingress namespace")
if err := clients.KubeClient.CoreV1().Secrets(ingressNS).Delete(context.Background(), config.ServingRoutingCertName, v1.DeleteOptions{}); err != nil {
t.Fatalf("Failed to delete secret %s in ingress namespace: %v", config.ServingRoutingCertName, err)
}
}

checkEndpointState(t, clients, url)
}

Expand Down

0 comments on commit 34e65a6

Please sign in to comment.