diff --git a/Makefile b/Makefile index 4354d7f50b27..0e3df88be5f6 100644 --- a/Makefile +++ b/Makefile @@ -39,7 +39,7 @@ perf-tests: .PHONY: perf-tests test-e2e-tls: - ENABLE_INTERNAL_TLS="true" ./openshift/e2e-tests.sh + ENABLE_TLS="true" ./openshift/e2e-tests.sh .PHONY: test-e2e-tls # Target used by github actions. diff --git a/openshift/e2e-common.sh b/openshift/e2e-common.sh index 60be0a9eeed8..4154f8477aac 100644 --- a/openshift/e2e-common.sh +++ b/openshift/e2e-common.sh @@ -130,11 +130,9 @@ function install_serverless(){ export GOPATH=/tmp/go export ON_CLUSTER_BUILDS=true export DOCKER_REPO_OVERRIDE=image-registry.openshift-image-registry.svc:5000/openshift-marketplace - #TODO: enable back when we have the feature ready again downstream - sed -i 's/internal-encryption: "true"/internal-encryption: "false"/g' ./test/v1beta1/resources/operator.knative.dev_v1beta1_knativeserving_cr.yaml OPENSHIFT_CI="true" make generated-files images install-serving || return $? - # Create a secret for https test. + # Ensure tests trust the OpenShift router CA trust_router_ca || return $? popd } @@ -150,30 +148,36 @@ function install_knative(){ wait_until_service_has_external_ip $SERVING_INGRESS_NAMESPACE kourier || fail_test "Ingress has no external IP" wait_until_hostname_resolves "$(kubectl get svc -n $SERVING_INGRESS_NAMESPACE kourier -o jsonpath='{.status.loadBalancer.ingress[0].hostname}')" - # TODO: Only one cluster enables internal-tls but it should be enabled by default when the feature is stable. - if [[ ${ENABLE_INTERNAL_TLS:-} == "true" ]]; then - configure_cm network internal-encryption:true || fail_test - # As config-kourier is in ingress namespace, don't use configure_cm. - oc patch knativeserving knative-serving \ - -n "${SERVING_NAMESPACE}" \ - --type merge --patch '{"spec": {"config": {"kourier": {"cluster-cert-secret": "server-certs"}}}}' - # Deploy certificates for testing TLS with cluster-local gateway - timeout 600 '[[ $(oc get ns $SERVING_INGRESS_NAMESPACE -oname | wc -l) == 0 ]]' || return 1 - yq read --doc 1 ./test/config/tls/cert-secret.yaml | yq write - metadata.namespace ${SERVING_INGRESS_NAMESPACE} | oc apply -f - + if [[ ${ENABLE_TLS:-} == "true" ]]; then + configure_cm network system-internal-tls:enabled || fail_test + configure_cm network cluster-local-domain-tls:enabled || fail_test + + echo "Restart controller to enable cert-manager integration" + oc delete pod -n ${SERVING_NAMESPACE} -l app=controller + oc wait --timeout=60s --for=condition=Available deployment -n ${SERVING_NAMESPACE} controller + echo "Restart activator to mount the certificates" oc delete pod -n ${SERVING_NAMESPACE} -l app=activator oc wait --timeout=60s --for=condition=Available deployment -n ${SERVING_NAMESPACE} activator - echo "internal-encryption is enabled" + + echo "cluster-local-domain-tls and system-internal-tls are ENABLED" else # disable internal-encryption. S-O repo would enable by default. - configure_cm network internal-encryption:false || fail_test + configure_cm network system-internal-tls:disabled || fail_test + configure_cm network cluster-local-domain-tls:disabled || fail_test + echo "Restart activator to unmount the certificates" oc delete pod -n ${SERVING_NAMESPACE} -l app=activator oc wait --timeout=60s --for=condition=Available deployment -n ${SERVING_NAMESPACE} activator - echo "internal-encryption is disabled" + + echo "Restart controller to disable cert-manager integration" + oc delete pod -n ${SERVING_NAMESPACE} -l app=controller + oc wait --timeout=60s --for=condition=Available deployment -n ${SERVING_NAMESPACE} controller + + echo "cluster-local-domain-tls and system-internal-tls are DISABLED" fi - header "Knative Installed successfully" + header "Successfully installed Knative" } function prepare_knative_serving_tests_nightly { @@ -195,15 +199,6 @@ function prepare_knative_serving_tests_nightly { export GATEWAY_OVERRIDE=kourier export GATEWAY_NAMESPACE_OVERRIDE="$SERVING_INGRESS_NAMESPACE" export INGRESS_CLASS=kourier.ingress.networking.knative.dev - - if [[ ${ENABLE_INTERNAL_TLS} == "true" ]]; then - # Deploy CA cert for testing TLS with cluster-local gateway - yq read --doc 0 ./test/config/tls/cert-secret.yaml | oc apply -f - - # This needs to match the name of Secret in test/config/tls/cert-secret.yaml - export CA_CERT=ca-cert - # This needs to match $san from test/config/tls/generate.sh - export SERVER_NAME=knative.dev - fi } function run_e2e_tests(){ @@ -232,7 +227,7 @@ function run_e2e_tests(){ sleep 30 subdomain=$(oc get ingresses.config.openshift.io cluster -o jsonpath="{.spec.domain}") - readonly OPENSHIFT_TEST_OPTIONS="--kubeconfig $KUBECONFIG --enable-beta --enable-alpha --resolvabledomain --customdomain=$subdomain --https --skip-cleanup-on-fail" + readonly OPENSHIFT_TEST_OPTIONS="--kubeconfig $KUBECONFIG --enable-beta --enable-alpha --resolvabledomain --customdomain=$subdomain --ingress-class=${INGRESS_CLASS} --https --skip-cleanup-on-fail" # Enable secure pod defaults for all tests. enable_feature_flags secure-pod-defaults || fail_test @@ -254,7 +249,7 @@ function run_e2e_tests(){ parallel=2 fi - go_test_e2e -tags=e2e -timeout=30m -parallel=$parallel \ + go_test_e2e -tags=e2e -timeout=40m -parallel=$parallel \ ./test/e2e ./test/conformance/api/... ./test/conformance/runtime/... \ --imagetemplate "$TEST_IMAGE_TEMPLATE" \ ${OPENSHIFT_TEST_OPTIONS} || failed=1 @@ -265,6 +260,23 @@ function run_e2e_tests(){ ${OPENSHIFT_TEST_OPTIONS} || failed=1 disable_feature_flags tag-header-based-routing || fail_test + if [[ ${ENABLE_TLS:-} == "true" ]]; then + go_test_e2e -timeout=5m ./test/e2e/clusterlocaldomaintls \ + --imagetemplate "$TEST_IMAGE_TEMPLATE" \ + ${OPENSHIFT_TEST_OPTIONS} || failed=1 + + # get existing request-log-template + existingTemplate=$(oc get cm -n "${SYSTEM_NAMESPACE}" config-observability -o jsonpath='{.data.logging\.request-log-template}' | sed 's/\"/\\"/g') + patch_request_log_template "TLS: {{.Request.TLS}}" || fail_test + + go_test_e2e -timeout=5m ./test/e2e/systeminternaltls \ + --imagetemplate "$TEST_IMAGE_TEMPLATE" \ + ${OPENSHIFT_TEST_OPTIONS} || failed=1 + + # restore request-log-template + patch_request_log_template "$existingTemplate" || fail_test + fi + configure_cm autoscaler allow-zero-initial-scale:true || fail_test # wait 10 sec until sync. sleep 10 @@ -385,6 +397,15 @@ function run_e2e_tests(){ return $failed } +function patch_request_log_template { + # do not use configure_cm as it would split on the : + local failed=0 + oc -n ${SERVING_NAMESPACE} patch knativeserving/knative-serving --type=merge \ + --patch="{\"spec\": {\"config\": { \"observability\": {\"logging.request-log-template\": \"$1\" }}}}" || failed=1 + timeout 30 "[[ ! \$(oc get cm -n ${SERVING_NAMESPACE} config-observability -o jsonpath='{.data.logging\.request-log-template}') == \"$1\" ]]" || failed=1 + return $failed +} + function gather_knative_state { logger.info 'Gather knative state' local gather_dir="${ARTIFACT_DIR:-/tmp}/gather-knative" diff --git a/openshift/e2e-tests.sh b/openshift/e2e-tests.sh index 4575200c8091..5ec645cebce9 100755 --- a/openshift/e2e-tests.sh +++ b/openshift/e2e-tests.sh @@ -3,13 +3,11 @@ # shellcheck disable=SC1090 source "$(dirname "$0")/e2e-common.sh" -set -x - env failed=0 -export ENABLE_INTERNAL_TLS="${ENABLE_INTERNAL_TLS:-false}" +export ENABLE_TLS="${ENABLE_TLS:-false}" (( !failed )) && install_knative || failed=1 (( !failed )) && prepare_knative_serving_tests_nightly || failed=2 diff --git a/openshift/patches/014-ocp-system-internal-tls-test.patch b/openshift/patches/014-ocp-system-internal-tls-test.patch new file mode 100644 index 000000000000..2d4256b5b381 --- /dev/null +++ b/openshift/patches/014-ocp-system-internal-tls-test.patch @@ -0,0 +1,67 @@ +diff --git a/test/e2e/systeminternaltls/system_internal_tls_test.go b/test/e2e/systeminternaltls/system_internal_tls_test.go +--- a/test/e2e/systeminternaltls/system_internal_tls_test.go (revision 9d0fa738d33402b893659610ddf4d4635cfa61f4) ++++ b/test/e2e/systeminternaltls/system_internal_tls_test.go (revision 4f646561395fb011368360e35365f8664383cdc5) +@@ -113,6 +113,7 @@ + + // TestTLSCertificateRotation tests certificate rotation and automatic reloading of certs. + func TestTLSCertificateRotation(t *testing.T) { ++ + if !test.ServingFlags.EnableAlphaFeatures { + t.Skip("Alpha features not enabled") + } +@@ -162,6 +163,7 @@ + + t.Log("Creating ConfigMap with old and new CA certs") + systemNS := os.Getenv(system.NamespaceEnvKey) ++ ingressNS := os.Getenv(test.GatewayNamespaceOverride) + + // Create ConfigMap with networking.knative.dev/trust-bundle label in required namespaces + cm := &corev1.ConfigMap{ +@@ -179,7 +181,15 @@ + _, err = clients.KubeClient.CoreV1().ConfigMaps(systemNS). + Create(context.Background(), cm, v1.CreateOptions{}) + if err != nil { +- t.Fatal("Failed to create configmap:", err) ++ t.Fatal("Failed to create configmap in "+systemNS, err) ++ } ++ ++ if ingressNS != "" && systemNS != ingressNS { ++ _, err = clients.KubeClient.CoreV1().ConfigMaps(ingressNS). ++ Create(context.Background(), cm, v1.CreateOptions{}) ++ if err != nil { ++ t.Fatal("Failed to create configmap in "+ingressNS, err) ++ } + } + + // Clean up on test failure or interrupt +@@ -187,7 +197,14 @@ + test.TearDown(clients, &names) + if err := clients.KubeClient.CoreV1().ConfigMaps(systemNS). + Delete(context.Background(), cm.Name, v1.DeleteOptions{}); err != nil && !errors.IsNotFound(err) { +- t.Fatal("Failed to delete configmap:", err) ++ t.Fatal("Failed to delete configmap in "+systemNS, err) ++ } ++ ++ if ingressNS != "" && systemNS != ingressNS { ++ if err := clients.KubeClient.CoreV1().ConfigMaps(ingressNS). ++ Delete(context.Background(), cm.Name, v1.DeleteOptions{}); err != nil && !errors.IsNotFound(err) { ++ t.Fatal("Failed to delete configmap in "+ingressNS, err) ++ } + } + }) + +@@ -218,6 +235,14 @@ + if err := clients.KubeClient.CoreV1().Secrets(systemNS).Delete(context.Background(), config.ServingRoutingCertName, v1.DeleteOptions{}); err != nil { + t.Fatalf("Failed to delete secret %s in system namespace: %v", config.ServingRoutingCertName, err) + } ++ ++ if ingressNS != "" && systemNS != ingressNS { ++ t.Log("Deleting secret in ingress namespace") ++ if err := clients.KubeClient.CoreV1().Secrets(ingressNS).Delete(context.Background(), config.ServingRoutingCertName, v1.DeleteOptions{}); err != nil { ++ t.Fatalf("Failed to delete secret %s in ingress namespace: %v", config.ServingRoutingCertName, err) ++ } ++ } ++ + checkEndpointState(t, clients, url) + } + diff --git a/test/e2e/systeminternaltls/system_internal_tls_test.go b/test/e2e/systeminternaltls/system_internal_tls_test.go index 61931497c370..2bc1edb9c984 100644 --- a/test/e2e/systeminternaltls/system_internal_tls_test.go +++ b/test/e2e/systeminternaltls/system_internal_tls_test.go @@ -113,6 +113,7 @@ func TestSystemInternalTLS(t *testing.T) { // TestTLSCertificateRotation tests certificate rotation and automatic reloading of certs. func TestTLSCertificateRotation(t *testing.T) { + if !test.ServingFlags.EnableAlphaFeatures { t.Skip("Alpha features not enabled") } @@ -162,6 +163,7 @@ func TestTLSCertificateRotation(t *testing.T) { t.Log("Creating ConfigMap with old and new CA certs") systemNS := os.Getenv(system.NamespaceEnvKey) + ingressNS := os.Getenv(test.GatewayNamespaceOverride) // Create ConfigMap with networking.knative.dev/trust-bundle label in required namespaces cm := &corev1.ConfigMap{ @@ -179,7 +181,15 @@ func TestTLSCertificateRotation(t *testing.T) { _, err = clients.KubeClient.CoreV1().ConfigMaps(systemNS). Create(context.Background(), cm, v1.CreateOptions{}) if err != nil { - t.Fatal("Failed to create configmap:", err) + t.Fatal("Failed to create configmap in "+systemNS, err) + } + + if ingressNS != "" && systemNS != ingressNS { + _, err = clients.KubeClient.CoreV1().ConfigMaps(ingressNS). + Create(context.Background(), cm, v1.CreateOptions{}) + if err != nil { + t.Fatal("Failed to create configmap in "+ingressNS, err) + } } // Clean up on test failure or interrupt @@ -187,7 +197,14 @@ func TestTLSCertificateRotation(t *testing.T) { test.TearDown(clients, &names) if err := clients.KubeClient.CoreV1().ConfigMaps(systemNS). Delete(context.Background(), cm.Name, v1.DeleteOptions{}); err != nil && !errors.IsNotFound(err) { - t.Fatal("Failed to delete configmap:", err) + t.Fatal("Failed to delete configmap in "+systemNS, err) + } + + if ingressNS != "" && systemNS != ingressNS { + if err := clients.KubeClient.CoreV1().ConfigMaps(ingressNS). + Delete(context.Background(), cm.Name, v1.DeleteOptions{}); err != nil && !errors.IsNotFound(err) { + t.Fatal("Failed to delete configmap in "+ingressNS, err) + } } }) @@ -218,6 +235,14 @@ func TestTLSCertificateRotation(t *testing.T) { if err := clients.KubeClient.CoreV1().Secrets(systemNS).Delete(context.Background(), config.ServingRoutingCertName, v1.DeleteOptions{}); err != nil { t.Fatalf("Failed to delete secret %s in system namespace: %v", config.ServingRoutingCertName, err) } + + if ingressNS != "" && systemNS != ingressNS { + t.Log("Deleting secret in ingress namespace") + if err := clients.KubeClient.CoreV1().Secrets(ingressNS).Delete(context.Background(), config.ServingRoutingCertName, v1.DeleteOptions{}); err != nil { + t.Fatalf("Failed to delete secret %s in ingress namespace: %v", config.ServingRoutingCertName, err) + } + } + checkEndpointState(t, clients, url) }