From ea3535ace61007da72bc12a501699d4f32b08978 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Herv=C3=A9=20Werner?= Date: Fri, 11 Nov 2022 07:47:15 +0100 Subject: [PATCH] Upgrade ingress addon MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - Switch to a Helm-based installation. That will help to relieve the project of the maintenance burden as by default the latest chart version is picked. If case of trouble the user can override the chart version to install. - Set up the admission controller by default. This helps to detect invalid configurations early, see https://kubernetes.github.io/ingress-nginx/how-it-works/#avoiding-outage-from-wrong-configuration. It can be disabled at addon installation time to get a configuration similar to what is set up so far. - breaking change: the metrics endpoint is no longer exposed on the host (hostPort), instead it is exposed internally only through a ClusterIP service. Note that it is fairly easy to expose it externally though an Ingress. Signed-off-by: Hervé Werner --- addons.yaml | 4 +- addons/ingress/disable | 28 +--- addons/ingress/enable | 86 +++++++---- addons/ingress/ingress-class.yaml | 9 ++ addons/ingress/ingress.yaml | 238 ------------------------------ tests/utils.py | 23 ++- tests/validators.py | 20 +-- 7 files changed, 106 insertions(+), 302 deletions(-) create mode 100644 addons/ingress/ingress-class.yaml delete mode 100644 addons/ingress/ingress.yaml diff --git a/addons.yaml b/addons.yaml index 1204725e..85f6f24b 100644 --- a/addons.yaml +++ b/addons.yaml @@ -33,8 +33,8 @@ microk8s-addons: - name: "ingress" description: "Ingress controller for external access" - version: "1.2.0" - check_status: "pod/nginx-ingress-microk8s-controller" + version: "N/A" + check_status: "daemonset.apps/ingress-nginx-controller" supported_architectures: - arm64 - amd64 diff --git a/addons/ingress/disable b/addons/ingress/disable index acd8b15f..376ae15c 100755 --- a/addons/ingress/disable +++ b/addons/ingress/disable @@ -1,34 +1,16 @@ #!/usr/bin/env bash - set -e source $SNAP/actions/common/utils.sh CURRENT_DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd) source $CURRENT_DIR/../common/utils.sh -echo "Disabling Ingress" - -ARCH=$(arch) -TAG="v1.0.0-alpha.2" -DEFAULT_CERT="- ' '" # This default value is always fine when deleting resources. -EXTRA_ARGS="- --publish-status-address=127.0.0.1" - - -KUBECTL="$SNAP/kubectl --kubeconfig=${SNAP_DATA}/credentials/client.config" -# Clean up old ingress controller resources in the default namespace, in case these are still lurking around. -$KUBECTL delete deployment -n default default-http-backend > /dev/null 2>&1 || true -$KUBECTL delete service -n default default-http-backend > /dev/null 2>&1 || true -$KUBECTL delete serviceaccount -n default nginx-ingress-microk8s-serviceaccount > /dev/null 2>&1 || true -$KUBECTL delete role -n default nginx-ingress-microk8s-role > /dev/null 2>&1 || true -$KUBECTL delete rolebinding -n default nginx-ingress-microk8s > /dev/null 2>&1 || true -$KUBECTL delete configmap -n default nginx-load-balancer-microk8s-conf > /dev/null 2>&1 || true -$KUBECTL delete daemonset -n default nginx-ingress-microk8s-controller > /dev/null 2>&1 || true +NAMESPACE="ingress" +echo "Disabling Ingress" -declare -A map -map[\$TAG]="$TAG" -map[\$DEFAULT_CERT]="$DEFAULT_CERT" -map[\$EXTRA_ARGS]="$EXTRA_ARGS" -use_addon_manifest ingress/ingress delete "$(declare -p map)" +"$SNAP/microk8s-helm.wrapper" uninstall ingress-nginx -n $NAMESPACE > /dev/null 2>&1 +use_addon_manifest ingress/ingress-class delete +"$SNAP/microk8s-kubectl.wrapper" delete namespace $NAMESPACE echo "Ingress is disabled" diff --git a/addons/ingress/enable b/addons/ingress/enable index 6035436c..914e606c 100755 --- a/addons/ingress/enable +++ b/addons/ingress/enable @@ -1,43 +1,73 @@ #!/usr/bin/env bash - set -e source $SNAP/actions/common/utils.sh CURRENT_DIR=$(cd $(dirname "${BASH_SOURCE[0]}") && pwd) source $CURRENT_DIR/../common/utils.sh -read -ra ARGUMENTS <<<"$1" - -read -r key value <<<$(echo "${ARGUMENTS[@]}" | gawk -F "=" '{print $1 ,$2}') -read -ra CERT_SECRET <<< "$value" - -KEY_NAME="default-ssl-certificate" - -if [ ! -z "$key" ] && [ "$key" != $KEY_NAME ] -then - echo "Unknown argument '$key'." - echo "You can use '$KEY_NAME' to load the default TLS certificate from a secret, eg" - echo " microk8s enable ingress:$KEY_NAME=namespace/secret_name" - exit 1 -fi +NAMESPACE="ingress" echo "Enabling Ingress" -ARCH=$(arch) -TAG="v1.2.0" -EXTRA_ARGS="- --publish-status-address=127.0.0.1" -DEFAULT_CERT="- ' '" +CERT_SECRET= +DISABLE_VALIDATING_ADMISSION= +INGRESS_NGINX_VALUES= +INGRESS_NGINX_VERSION= +while [ $# -ge 1 ]; do + case $1 in + --default-ssl-certificate=*) + CERT_SECRET="${1#*=}" + echo "Setting ${CERT_SECRET} as the default ingress certificate" + shift + ;; + --disable-validating-admission-controller) + DISABLE_VALIDATING_ADMISSION=1 + shift + ;; + --values=*) + INGRESS_NGINX_VALUES="${1#*=}" + shift + ;; + --version=*) + INGRESS_NGINX_VERSION="${1#*=}" + shift + ;; + *) + echo "Unknown option ${1}" >&2 + exit 1 + ;; + esac +done -if [ ! -z "$CERT_SECRET" ] -then - DEFAULT_CERT="- --default-ssl-certificate=${CERT_SECRET}" - echo "Setting ${CERT_SECRET} as default ingress certificate" +HELM_OPTS= +if [ -n "${INGRESS_NGINX_VALUES}" ]; then + HELM_OPTS+="--values ${INGRESS_NGINX_VALUES} " fi +if [ -n "${INGRESS_NGINX_VERSION}" ]; then + HELM_OPTS+="--version ${INGRESS_NGINX_VERSION} " +fi +if [ -n "${CERT_SECRET}" ]; then + HELM_OPTS+="--set controller.extraArgs.default-ssl-certificate=${CERT_SECRET} " +fi +if [ -n "${DISABLE_VALIDATING_ADMISSION}" ]; then + HELM_OPTS+="--set controller.admissionWebhooks.enabled=false " +fi + +"${SNAP}/microk8s-helm3.wrapper" upgrade --install ingress-nginx ingress-nginx \ + --repo https://kubernetes.github.io/ingress-nginx \ + --namespace $NAMESPACE --create-namespace \ + --set controller.extraArgs.publish-status-address=127.0.0.1 \ + --set controller.hostPort.enabled=true \ + --set controller.ingressClass=public \ + --set controller.ingressClassResource.name=public \ + --set controller.ingressClassResource.default=true \ + --set controller.kind=DaemonSet \ + --set controller.metrics.enabled=true \ + --set controller.publishService.enabled=false \ + --set controller.service.type=ClusterIP \ + ${HELM_OPTS} -declare -A map -map[\$TAG]="$TAG" -map[\$DEFAULT_CERT]="$DEFAULT_CERT" -map[\$EXTRA_ARGS]="$EXTRA_ARGS" -use_addon_manifest ingress/ingress apply "$(declare -p map)" +# Creating an additional public IngressClass to stay backward-compatible +use_addon_manifest ingress/ingress-class apply echo "Ingress is enabled" diff --git a/addons/ingress/ingress-class.yaml b/addons/ingress/ingress-class.yaml new file mode 100644 index 00000000..a4d69b1b --- /dev/null +++ b/addons/ingress/ingress-class.yaml @@ -0,0 +1,9 @@ +--- +apiVersion: networking.k8s.io/v1 +kind: IngressClass +metadata: + labels: + app.kubernetes.io/managed-by: microk8s + name: nginx +spec: + controller: k8s.io/ingress-nginx diff --git a/addons/ingress/ingress.yaml b/addons/ingress/ingress.yaml deleted file mode 100644 index 80a63eee..00000000 --- a/addons/ingress/ingress.yaml +++ /dev/null @@ -1,238 +0,0 @@ -apiVersion: networking.k8s.io/v1 -kind: IngressClass -metadata: - name: public - annotations: - ingressclass.kubernetes.io/is-default-class: "true" -spec: - controller: k8s.io/ingress-nginx ---- -apiVersion: networking.k8s.io/v1 -kind: IngressClass -metadata: - name: nginx -spec: - controller: k8s.io/ingress-nginx ---- -apiVersion: v1 -kind: Namespace -metadata: - name: ingress ---- -apiVersion: v1 -kind: ServiceAccount -metadata: - name: nginx-ingress-microk8s-serviceaccount - namespace: ingress ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRole -metadata: - name: nginx-ingress-microk8s-clusterrole -rules: -- apiGroups: - - "" - resources: - - nodes - - services - verbs: - - get - - list - - watch -- apiGroups: - - "" - resources: - - configmaps - - endpoints - - nodes - - pods - - secrets - verbs: - - list - - watch -- apiGroups: - - "" - resources: - - events - verbs: - - create - - patch -- apiGroups: - - extensions - - networking.k8s.io - resources: - - ingresses - verbs: - - get - - list - - watch -- apiGroups: - - extensions - - networking.k8s.io - resources: - - ingresses/status - verbs: - - update -- apiGroups: - - networking.k8s.io - resources: - - ingressclasses - verbs: - - get - - list - - watch ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: Role -metadata: - name: nginx-ingress-microk8s-role - namespace: ingress -rules: -- apiGroups: - - "" - resources: - - configmaps - - endpoints - - pods - - secrets - verbs: - - get -- apiGroups: - - "" - resources: - - configmaps - resourceNames: - - ingress-controller-leader - verbs: - - create - - update -- apiGroups: - - "" - resources: - - configmaps - verbs: - - create ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: ClusterRoleBinding -metadata: - name: nginx-ingress-microk8s -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: ClusterRole - name: nginx-ingress-microk8s-clusterrole -subjects: -- kind: ServiceAccount - name: nginx-ingress-microk8s-serviceaccount - namespace: ingress ---- -apiVersion: rbac.authorization.k8s.io/v1 -kind: RoleBinding -metadata: - name: nginx-ingress-microk8s - namespace: ingress -roleRef: - apiGroup: rbac.authorization.k8s.io - kind: Role - name: nginx-ingress-microk8s-role -subjects: -- kind: ServiceAccount - name: nginx-ingress-microk8s-serviceaccount ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: nginx-load-balancer-microk8s-conf - namespace: ingress ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: nginx-ingress-tcp-microk8s-conf - namespace: ingress ---- -apiVersion: v1 -kind: ConfigMap -metadata: - name: nginx-ingress-udp-microk8s-conf - namespace: ingress ---- -apiVersion: apps/v1 -kind: DaemonSet -metadata: - name: nginx-ingress-microk8s-controller - namespace: ingress - labels: - microk8s-application: nginx-ingress-microk8s -spec: - selector: - matchLabels: - name: nginx-ingress-microk8s - template: - metadata: - labels: - name: nginx-ingress-microk8s - spec: - terminationGracePeriodSeconds: 60 - serviceAccountName: nginx-ingress-microk8s-serviceaccount - containers: - - image: k8s.gcr.io/ingress-nginx/controller:$TAG - name: nginx-ingress-microk8s - livenessProbe: - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - initialDelaySeconds: 10 - periodSeconds: 10 - successThreshold: 1 - failureThreshold: 3 - timeoutSeconds: 5 - readinessProbe: - httpGet: - path: /healthz - port: 10254 - scheme: HTTP - periodSeconds: 10 - successThreshold: 1 - failureThreshold: 3 - timeoutSeconds: 5 - lifecycle: - preStop: - exec: - command: - - /wait-shutdown - securityContext: - capabilities: - add: - - NET_BIND_SERVICE - drop: - - ALL - runAsUser: 101 # www-data - env: - - name: POD_NAME - valueFrom: - fieldRef: - fieldPath: metadata.name - - name: POD_NAMESPACE - valueFrom: - fieldRef: - fieldPath: metadata.namespace - ports: - - name: http - containerPort: 80 - hostPort: 80 - - name: https - containerPort: 443 - hostPort: 443 - - name: health - containerPort: 10254 - hostPort: 10254 - args: - - /nginx-ingress-controller - - --configmap=$(POD_NAMESPACE)/nginx-load-balancer-microk8s-conf - - --tcp-services-configmap=$(POD_NAMESPACE)/nginx-ingress-tcp-microk8s-conf - - --udp-services-configmap=$(POD_NAMESPACE)/nginx-ingress-udp-microk8s-conf - - --ingress-class=public - $DEFAULT_CERT - $EXTRA_ARGS diff --git a/tests/utils.py b/tests/utils.py index ed595354..b3dab98d 100644 --- a/tests/utils.py +++ b/tests/utils.py @@ -93,7 +93,7 @@ def wait_for_pod_state( pod, namespace, desired_state, desired_reason=None, label=None, timeout_insec=600 ): """ - Wait for a a pod state. If you do not specify a pod name and you set instead a label + Wait for a pod state. If you do not specify a pod name and you set instead a label only the first pod will be checked. """ deadline = datetime.datetime.now() + datetime.timedelta(seconds=timeout_insec) @@ -127,6 +127,27 @@ def wait_for_pod_state( time.sleep(3) +def wait_for_populated_endpoint(endpoint, namespace, timeout_insec=300): + """ + Wait for the endpoint to be populated. + """ + deadline = datetime.datetime.now() + datetime.timedelta(seconds=timeout_insec) + while True: + if datetime.datetime.now() > deadline: + raise TimeoutError( + "Endpoint {} still empty after {} seconds.".format( + endpoint, timeout_insec + ) + ) + cmd = "ep {} -n {}".format(endpoint, namespace) + data = kubectl_get(cmd, timeout_insec) + if "subsets" in data: + ready_addresses = data["subsets"][0].get("addresses", []) + if len(ready_addresses) > 0: + break + time.sleep(3) + + def wait_for_installation(cluster_nodes=1, timeout_insec=360): """ Wait for kubernetes service to appear. diff --git a/tests/validators.py b/tests/validators.py index 1ed6a4d4..9dc12398 100644 --- a/tests/validators.py +++ b/tests/validators.py @@ -11,6 +11,7 @@ get_arch, kubectl, wait_for_pod_state, + wait_for_populated_endpoint, kubectl_get, wait_for_installation, docker, @@ -163,16 +164,13 @@ def validate_ingress(): """ Validate ingress by creating a ingress rule. """ - daemonset = kubectl("get ds") - if "nginx-ingress-microk8s-controller" in daemonset: - wait_for_pod_state("", "default", "running", label="app=default-http-backend") - wait_for_pod_state( - "", "default", "running", label="name=nginx-ingress-microk8s" - ) - else: - wait_for_pod_state( - "", "ingress", "running", label="name=nginx-ingress-microk8s" - ) + wait_for_pod_state( + "", + "ingress", + "running", + label="app.kubernetes.io/name=ingress-nginx", + ) + wait_for_populated_endpoint("ingress-nginx-controller-admission", "ingress") manifest = TEMPLATES / "ingress.yaml" update_yaml_with_arch(manifest) @@ -391,6 +389,8 @@ def validate_cert_manager(): wait_for_pod_state( "", "cert-manager", "running", label="app.kubernetes.io/name=cert-manager" ) + wait_for_populated_endpoint("ingress-nginx-controller-admission", "ingress") + wait_for_populated_endpoint("cert-manager-webhook", "cert-manager") manifest = TEMPLATES / "cert-manager-aio-test.yaml" kubectl("apply -f {}".format(manifest))