From 2628f4022c157b60ab498624f4b3453bb07a4a54 Mon Sep 17 00:00:00 2001 From: Angelos Kolaitis Date: Tue, 12 Mar 2024 23:12:55 +0200 Subject: [PATCH] wait for apiserver instead of Ready nodes --- controllers/cloudinit/controlplane_init.go | 2 +- controllers/cloudinit/controlplane_init_test.go | 2 +- controllers/cloudinit/controlplane_join.go | 4 ++-- controllers/cloudinit/controlplane_join_test.go | 4 ++-- controllers/cloudinit/embed.go | 4 ++++ .../cloudinit/scripts/10-configure-apiserver.sh | 2 +- .../cloudinit/scripts/10-configure-calico-ipip.sh | 7 +++++++ .../cloudinit/scripts/10-configure-cert-for-lb.sh | 3 ++- controllers/cloudinit/scripts/20-microk8s-enable.sh | 2 +- controllers/cloudinit/scripts/20-microk8s-join.sh | 5 +---- controllers/cloudinit/scripts/50-wait-apiserver.sh | 13 +++++++++++++ controllers/cloudinit/worker_join.go | 2 +- controllers/cloudinit/worker_join_test.go | 2 +- 13 files changed, 37 insertions(+), 15 deletions(-) create mode 100644 controllers/cloudinit/scripts/50-wait-apiserver.sh diff --git a/controllers/cloudinit/controlplane_init.go b/controllers/cloudinit/controlplane_init.go index 817658a..4eb1123 100644 --- a/controllers/cloudinit/controlplane_init.go +++ b/controllers/cloudinit/controlplane_init.go @@ -144,7 +144,7 @@ func NewInitControlPlane(input *ControlPlaneInitInput) (*CloudConfig, error) { fmt.Sprintf("%s %q", scriptPath(installMicroK8sScript), installArgs), fmt.Sprintf("%s %q %q %q", scriptPath(configureContainerdProxyScript), input.ContainerdHTTPProxy, input.ContainerdHTTPSProxy, input.ContainerdNoProxy), scriptPath(configureKubeletScript), - "microk8s status --wait-ready", + scriptPath(waitAPIServerScript), "microk8s refresh-certs /var/tmp", fmt.Sprintf("%s %v", scriptPath(configureCalicoIPIPScript), input.IPinIP), fmt.Sprintf("%s %q", scriptPath(configureClusterAgentPortScript), input.ClusterAgentPort), diff --git a/controllers/cloudinit/controlplane_init_test.go b/controllers/cloudinit/controlplane_init_test.go index 6a9cfc5..b4cbaa1 100644 --- a/controllers/cloudinit/controlplane_init_test.go +++ b/controllers/cloudinit/controlplane_init_test.go @@ -50,7 +50,7 @@ func TestControlPlaneInit(t *testing.T) { `/capi-scripts/00-install-microk8s.sh "--channel 1.25 --classic"`, `/capi-scripts/10-configure-containerd-proxy.sh "" "" ""`, `/capi-scripts/10-configure-kubelet.sh`, - `microk8s status --wait-ready`, + `/capi-scripts/50-wait-apiserver.sh`, `microk8s refresh-certs /var/tmp`, `/capi-scripts/10-configure-calico-ipip.sh true`, `/capi-scripts/10-configure-cluster-agent-port.sh "30000"`, diff --git a/controllers/cloudinit/controlplane_join.go b/controllers/cloudinit/controlplane_join.go index 4530689..7c936dd 100644 --- a/controllers/cloudinit/controlplane_join.go +++ b/controllers/cloudinit/controlplane_join.go @@ -126,11 +126,11 @@ func NewJoinControlPlane(input *ControlPlaneJoinInput) (*CloudConfig, error) { fmt.Sprintf("%s %q", scriptPath(installMicroK8sScript), installArgs), fmt.Sprintf("%s %q %q %q", scriptPath(configureContainerdProxyScript), input.ContainerdHTTPProxy, input.ContainerdHTTPSProxy, input.ContainerdNoProxy), scriptPath(configureKubeletScript), - "microk8s status --wait-ready", + scriptPath(waitAPIServerScript), fmt.Sprintf("%s %v", scriptPath(configureCalicoIPIPScript), input.IPinIP), fmt.Sprintf("%s %q", scriptPath(configureClusterAgentPortScript), input.ClusterAgentPort), fmt.Sprintf("%s %q", scriptPath(configureDqlitePortScript), input.DqlitePort), - "microk8s status --wait-ready", + scriptPath(waitAPIServerScript), fmt.Sprintf("%s %q %q", scriptPath(configureCertLB), endpointType, input.ControlPlaneEndpoint), fmt.Sprintf("%s no %s", scriptPath(microk8sJoinScript), strings.Join(joinURLs, " ")), scriptPath(configureAPIServerScript), diff --git a/controllers/cloudinit/controlplane_join_test.go b/controllers/cloudinit/controlplane_join_test.go index b56994a..1f5115e 100644 --- a/controllers/cloudinit/controlplane_join_test.go +++ b/controllers/cloudinit/controlplane_join_test.go @@ -48,11 +48,11 @@ func TestControlPlaneJoin(t *testing.T) { `/capi-scripts/00-install-microk8s.sh "--channel 1.25 --classic"`, `/capi-scripts/10-configure-containerd-proxy.sh "" "" ""`, `/capi-scripts/10-configure-kubelet.sh`, - `microk8s status --wait-ready`, + `/capi-scripts/50-wait-apiserver.sh`, `/capi-scripts/10-configure-calico-ipip.sh true`, `/capi-scripts/10-configure-cluster-agent-port.sh "30000"`, `/capi-scripts/10-configure-dqlite-port.sh "2379"`, - `microk8s status --wait-ready`, + `/capi-scripts/50-wait-apiserver.sh`, `/capi-scripts/10-configure-cert-for-lb.sh "DNS" "k8s.my-domain.com"`, `/capi-scripts/20-microk8s-join.sh no "10.0.3.39:30000/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" "10.0.3.40:30000/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" "10.0.3.41:30000/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"`, `/capi-scripts/10-configure-apiserver.sh`, diff --git a/controllers/cloudinit/embed.go b/controllers/cloudinit/embed.go index 31adcae..39fa77e 100644 --- a/controllers/cloudinit/embed.go +++ b/controllers/cloudinit/embed.go @@ -75,6 +75,9 @@ const ( // configureTraefikScript configures the control plane endpoint in the traefik provider configuration. configureTraefikScript script = "30-configure-traefik.sh" + + // waitAPIServerScript waits for the kube-apiserver to be ready. + waitAPIServerScript script = "50-wait-apiserver.sh" ) var allScripts = []script{ @@ -92,6 +95,7 @@ var allScripts = []script{ configureKubeletScript, microk8sEnableScript, microk8sJoinScript, + waitAPIServerScript, } func mustGetScript(scriptName script) string { diff --git a/controllers/cloudinit/scripts/10-configure-apiserver.sh b/controllers/cloudinit/scripts/10-configure-apiserver.sh index 5289b36..f727147 100644 --- a/controllers/cloudinit/scripts/10-configure-apiserver.sh +++ b/controllers/cloudinit/scripts/10-configure-apiserver.sh @@ -37,7 +37,7 @@ while ! snap restart microk8s.daemon-kubelite; do done # delete kubernetes service to make sure port is updated -microk8s status --wait-ready +/capi-scripts/50-wait-apiserver.sh microk8s kubectl delete svc kubernetes # redirect port 16443 to 6443 diff --git a/controllers/cloudinit/scripts/10-configure-calico-ipip.sh b/controllers/cloudinit/scripts/10-configure-calico-ipip.sh index 60da028..6035af4 100644 --- a/controllers/cloudinit/scripts/10-configure-calico-ipip.sh +++ b/controllers/cloudinit/scripts/10-configure-calico-ipip.sh @@ -15,6 +15,13 @@ fi CNI_YAML="/var/snap/microk8s/current/args/cni-network/cni.yaml" +if [ ! -f "${CNI_YAML}" ]; then + echo "Will not configure Calico, missing cni.yaml" + exit 0 +fi + +/capi-scripts/50-wait-apiserver.sh + # Stop calico-node and delete ippools to ensure no vxlan pools are left around microk8s kubectl delete daemonset/calico-node -n kube-system || true microk8s kubectl delete ippools --all || true diff --git a/controllers/cloudinit/scripts/10-configure-cert-for-lb.sh b/controllers/cloudinit/scripts/10-configure-cert-for-lb.sh index fa3fb82..85730d3 100644 --- a/controllers/cloudinit/scripts/10-configure-cert-for-lb.sh +++ b/controllers/cloudinit/scripts/10-configure-cert-for-lb.sh @@ -22,4 +22,5 @@ sleep 10 while ! snap restart microk8s.daemon-kubelite; do sleep 5 done -microk8s status --wait-ready + +/capi-scripts/50-wait-apiserver.sh diff --git a/controllers/cloudinit/scripts/20-microk8s-enable.sh b/controllers/cloudinit/scripts/20-microk8s-enable.sh index 5e7c978..2c1066f 100644 --- a/controllers/cloudinit/scripts/20-microk8s-enable.sh +++ b/controllers/cloudinit/scripts/20-microk8s-enable.sh @@ -12,6 +12,6 @@ microk8s enable community || true while [[ "$@" != "" ]]; do microk8s enable "$1" - microk8s status --wait-ready + /capi-scripts/50-wait-apiserver.sh shift done diff --git a/controllers/cloudinit/scripts/20-microk8s-join.sh b/controllers/cloudinit/scripts/20-microk8s-join.sh index e432335..fed5141 100644 --- a/controllers/cloudinit/scripts/20-microk8s-join.sh +++ b/controllers/cloudinit/scripts/20-microk8s-join.sh @@ -55,8 +55,5 @@ done sleep 10 if [ ${1} == "no" ]; then - while ! microk8s status --wait-ready; do - echo "Waiting for the cluster to come up" - sleep 5 - done + /capi-scripts/50-wait-apiserver.sh fi diff --git a/controllers/cloudinit/scripts/50-wait-apiserver.sh b/controllers/cloudinit/scripts/50-wait-apiserver.sh new file mode 100644 index 0000000..c88ad26 --- /dev/null +++ b/controllers/cloudinit/scripts/50-wait-apiserver.sh @@ -0,0 +1,13 @@ +#!/bin/bash -xe + +# Usage: +# $0 +# +# Assumptions: +# - microk8s is installed +# - microk8s kubelite service is running + +while ! microk8s kubectl get --raw /readyz; do + echo Waiting for kube-apiserver + sleep 3 +done diff --git a/controllers/cloudinit/worker_join.go b/controllers/cloudinit/worker_join.go index 20dac3c..10efcc9 100644 --- a/controllers/cloudinit/worker_join.go +++ b/controllers/cloudinit/worker_join.go @@ -115,7 +115,7 @@ func NewJoinWorker(input *WorkerInput) (*CloudConfig, error) { fmt.Sprintf("%s %q", scriptPath(installMicroK8sScript), installArgs), fmt.Sprintf("%s %q %q %q", scriptPath(configureContainerdProxyScript), input.ContainerdHTTPProxy, input.ContainerdHTTPSProxy, input.ContainerdNoProxy), scriptPath(configureKubeletScript), - "microk8s status --wait-ready", + scriptPath(waitAPIServerScript), fmt.Sprintf("%s %q", scriptPath(configureClusterAgentPortScript), input.ClusterAgentPort), fmt.Sprintf("%s yes %s", scriptPath(microk8sJoinScript), strings.Join(joinURLs, " ")), fmt.Sprintf("%s %s 6443 %s", scriptPath(configureTraefikScript), input.ControlPlaneEndpoint, stopApiServerProxyRefreshes), diff --git a/controllers/cloudinit/worker_join_test.go b/controllers/cloudinit/worker_join_test.go index a207203..bc57e86 100644 --- a/controllers/cloudinit/worker_join_test.go +++ b/controllers/cloudinit/worker_join_test.go @@ -45,7 +45,7 @@ func TestWorkerJoin(t *testing.T) { `/capi-scripts/00-install-microk8s.sh "--channel 1.24 --classic"`, `/capi-scripts/10-configure-containerd-proxy.sh "" "" ""`, `/capi-scripts/10-configure-kubelet.sh`, - `microk8s status --wait-ready`, + `/capi-scripts/50-wait-apiserver.sh`, `/capi-scripts/10-configure-cluster-agent-port.sh "30000"`, `/capi-scripts/20-microk8s-join.sh yes "10.0.3.194:30000/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa" "10.0.3.195:30000/aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa"`, `/capi-scripts/30-configure-traefik.sh capi-aws-apiserver-1647391446.us-east-1.elb.amazonaws.com 6443 no`,