From 323cfaae0d2d863fee69c43f7ab67fd3e71a29a5 Mon Sep 17 00:00:00 2001 From: Prajwal Athreya <145620504+prajwalvathreya@users.noreply.github.com> Date: Wed, 14 Aug 2024 17:37:24 -0400 Subject: [PATCH] [test] e2e tests(konnectivity) for kubeadm-full (#447) * E2E test for Konnectivity and checking provisioned CAPI resources and child cluster resources * Added additional tests for deleting child cluster * Updated GHA workflow files to run E2E test on draft PR * Updating path to e2e kubeadm-full test * Checking if correct key is passed to the selector * Checking if correct key is passed to the selector. v2. * Updated GHA filter and added tests for etcd * Added Ephemeral key Added tests for StatefulSets Split DeamonSets tests and konnectivity tests * - Konnectivity complete tests - Etcd tests for volume mount, partial checks * - reverted build_test_ci.yml to original file available on the main branch * - removed files related to ETCD * - formatted chainsaw-test.yaml for konnectivity server and agent * - updating workflow files to run e2e tests on GHA * - combined konnectivity deployments and daemonsets into one file for improved readability * - updated test identifier name - fixed the right expected values to assert on in daemonsets - updated autoscaler version to a hardcoded one * - hygine changes in files release.yml and build_test_ci.yml - removed etcd-tests from konnectivity branch - added tests for cluster and node balancer(LinodeCluster) * - updated file names to be consistent * Update e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/check-child-cluster-and-vpc-deleted.yaml Co-authored-by: Rahul Sharma * Update e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/chainsaw-test.yaml Co-authored-by: Rahul Sharma * Update e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/assert-child-cluster-resources.yaml Co-authored-by: Rahul Sharma * - combined all parent resources into one file - seperated tests for deployment, daemonsets, statefulsets - added meaningful test names for better understanding while going through logs - added env variables to set konnectivity agents to 1 during testing * - added test to get logs from a pod * - fixed kubeconfig issue for getting pod logs * - updated environment variable for cluster-autoscaler-version --------- Co-authored-by: Rahul Sharma --- .github/filters.yml | 49 ++-- .github/workflows/build_test_ci.yml | 150 +++++----- .github/workflows/e2e-test.yaml | 1 + Makefile | 2 +- .../assert-capi-resources.yaml | 79 ++++++ .../assert-child-cluster-daemonsets.yaml | 35 +++ .../assert-child-cluster-deployments.yaml | 35 +++ .../assert-child-cluster-resources.yaml | 76 +++++ .../assert-child-cluster-statefulsets.yaml | 7 + .../assert-konnectivity-resources.yaml | 20 ++ .../chainsaw-test.yaml | 266 ++++++++++++++++++ .../check-child-cluster-and-vpc-deleted.yaml | 10 + .../cluster-autoscaler.yaml | 2 +- .../addons/konnectivity/konnectivity.yaml | 1 + 14 files changed, 632 insertions(+), 101 deletions(-) create mode 100644 e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/assert-capi-resources.yaml create mode 100644 e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/assert-child-cluster-daemonsets.yaml create mode 100644 e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/assert-child-cluster-deployments.yaml create mode 100644 e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/assert-child-cluster-resources.yaml create mode 100644 e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/assert-child-cluster-statefulsets.yaml create mode 100644 e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/assert-konnectivity-resources.yaml create mode 100644 e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/chainsaw-test.yaml create mode 100644 e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/check-child-cluster-and-vpc-deleted.yaml diff --git a/.github/filters.yml b/.github/filters.yml index 51b1bd681..591555cd0 100644 --- a/.github/filters.yml +++ b/.github/filters.yml @@ -4,59 +4,60 @@ src: kubeadm: - templates/flavors/kubeadm/default/* - e2e/capl-cluster-flavors/kubeadm-capl-cluster/* -kubeadm_cluster-autoscaler: +kubeadm-cluster-autoscaler: - templates/flavors/kubeadm/cluster-autoscaler/* -kubeadm_dual-stack: +kubeadm-dual-stack: - templates/flavors/kubeadm/dual-stack/* -kubeadm_etcd-backup-restore: +kubeadm-etcd-backup-restore: - templates/flavors/kubeadm/etcd-backup-restore/* -kubeadm_etcd-disk: +kubeadm-etcd-disk: - templates/flavors/kubeadm/etcd-disk/* -kubeadm_full: +kubeadm-full: - templates/flavors/kubeadm/full/* -kubeadm_full-vpcless: + - e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/* +kubeadm-full-vpcless: - templates/flavors/kubeadm/full-vpcless/* -kubeadm_self-healing: +kubeadm-self-healing: - templates/flavors/kubeadm/self-healing/* -kubeadm_vpcless: +kubeadm-vpcless: - templates/flavors/kubeadm/vpcless/* -kubeadm_cilium_bgp_lb: +kubeadm-cilium-bgp-lb: - templates/flavors/kubeadm/cilium-bgp-lb/* -kubeadm_konnectivity: +kubeadm-konnectivity: - templates/flavors/kubeadm/konnectivity/* k3s: - templates/flavors/k3s/default/* - e2e/capl-cluster-flavors/k3s-capl-cluster/* -k3s_cluster-autoscaler: +k3s-cluster-autoscaler: - templates/flavors/k3s/cluster-autoscaler/* -k3s_dual-stack: +k3s-dual-stack: - templates/flavors/k3s/dual-stack/* -k3s_etcd-backup-restore: +k3s-etcd-backup-restore: - templates/flavors/k3s/etcd-backup-restore/* -k3s_full: +k3s-full: - templates/flavors/k3s/full/* -k3s_full-vpcless: +k3s-full-vpcless: - templates/flavors/k3s/full-vpcless/* -k3s_self-healing: +k3s-self-healing: - templates/flavors/k3s/self-healing/* -k3s_vpcless: +k3s-vpcless: - templates/flavors/k3s/vpcless/* rke2: - templates/flavors/rke2/default/* - e2e/capl-cluster-flavors/rke2-capl-cluster/* -rke2_cluster-autoscaler: +rke2-cluster-autoscaler: - templates/flavors/rke2/cluster-autoscaler/* -rke2_etcd-backup-restore: +rke2-etcd-backup-restore: - templates/flavors/rke2/etcd-backup-restore/* -rke2_etcd-disk: +rke2-etcd-disk: - templates/flavors/rke2/etcd-disk/* -rke2_full: +rke2-full: - templates/flavors/rke2/full/* -rke2_full-vpcless: +rke2-full-vpcless: - templates/flavors/rke2/full-vpcless/* -rke2_self-healing: +rke2-self-healing: - templates/flavors/rke2/self-healing/* -rke2_vpcless: +rke2-vpcless: - templates/flavors/rke2/vpcless/* diff --git a/.github/workflows/build_test_ci.yml b/.github/workflows/build_test_ci.yml index 8c1173fd0..2973458fa 100644 --- a/.github/workflows/build_test_ci.yml +++ b/.github/workflows/build_test_ci.yml @@ -44,57 +44,57 @@ jobs: needs: changes if: ${{ contains(fromJSON(needs.changes.outputs.paths), 'src') }} steps: - - uses: actions/checkout@v4 - - name: Validate YAML file - run: yamllint templates + - uses: actions/checkout@v4 + - name: Validate YAML file + run: yamllint templates go-build-test: runs-on: ubuntu-latest needs: changes if: ${{ contains(fromJSON(needs.changes.outputs.paths), 'src') }} steps: - - name: Harden Runner - uses: step-security/harden-runner@v2 - with: - disable-sudo: true - egress-policy: block - allowed-endpoints: > - api.github.com:443 - github.com:443 - golang.org:443 - proxy.golang.org:443 - sum.golang.org:443 - objects.githubusercontent.com:443 - storage.googleapis.com:443 - cli.codecov.io:443 - api.codecov.io:443 - raw.githubusercontent.com:443 - - - uses: actions/checkout@v4 - - - name: Set up Go - uses: actions/setup-go@v5 - with: - go-version-file: 'go.mod' - check-latest: true - - - name: Build - run: make build - - - name: Check for generated diff - run: make check-gen-diff - - - name: Test - run: make test - - - name: Upload coverage reports to Codecov - uses: codecov/codecov-action@v4 - with: - files: ./coverage.out - fail_ci_if_error: true - verbose: true - token: ${{ secrets.CODECOV_TOKEN }} - slug: linode/cluster-api-provider-linode + - name: Harden Runner + uses: step-security/harden-runner@v2 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + api.github.com:443 + github.com:443 + golang.org:443 + proxy.golang.org:443 + sum.golang.org:443 + objects.githubusercontent.com:443 + storage.googleapis.com:443 + cli.codecov.io:443 + api.codecov.io:443 + raw.githubusercontent.com:443 + + - uses: actions/checkout@v4 + + - name: Set up Go + uses: actions/setup-go@v5 + with: + go-version-file: 'go.mod' + check-latest: true + + - name: Build + run: make build + + - name: Check for generated diff + run: make check-gen-diff + + - name: Test + run: make test + + - name: Upload coverage reports to Codecov + uses: codecov/codecov-action@v4 + with: + files: ./coverage.out + fail_ci_if_error: true + verbose: true + token: ${{ secrets.CODECOV_TOKEN }} + slug: linode/cluster-api-provider-linode e2e-test: needs: changes @@ -107,7 +107,7 @@ jobs: include: - flavor: ${{ github.ref == 'refs/heads/main' && 'all' || 'quick' }} uses: ./.github/workflows/e2e-test.yaml - if: ${{ github.event.pull_request.draft == false && contains(fromJSON(needs.changes.outputs.paths), 'src') }} + if: ${{contains(fromJSON(needs.changes.outputs.paths), 'src')}} secrets: inherit with: e2e-selector: ${{ matrix.flavor }} @@ -119,32 +119,32 @@ jobs: needs: changes if: ${{ contains(fromJSON(needs.changes.outputs.paths), 'src') }} steps: - - name: Harden Runner - uses: step-security/harden-runner@v2 - with: - disable-sudo: true - egress-policy: block - allowed-endpoints: > - api.github.com:443 - github.com:443 - proxy.golang.org:443 - sum.golang.org:443 - go.dev:443 - dl.google.com:443 - golang.org:443 - objects.githubusercontent.com:443 - registry-1.docker.io:443 - auth.docker.io:443 - production.cloudflare.docker.com:443 - gcr.io:443 - storage.googleapis.com:443 - - - uses: actions/checkout@v4 - - - name: Docker cache - uses: ScribeMD/docker-cache@0.5.0 - with: - key: docker-${{ runner.os }}-${{ hashFiles('go.sum') }} - - - name: Build the Docker image - run: make docker-build + - name: Harden Runner + uses: step-security/harden-runner@v2 + with: + disable-sudo: true + egress-policy: block + allowed-endpoints: > + api.github.com:443 + github.com:443 + proxy.golang.org:443 + sum.golang.org:443 + go.dev:443 + dl.google.com:443 + golang.org:443 + objects.githubusercontent.com:443 + registry-1.docker.io:443 + auth.docker.io:443 + production.cloudflare.docker.com:443 + gcr.io:443 + storage.googleapis.com:443 + + - uses: actions/checkout@v4 + + - name: Docker cache + uses: ScribeMD/docker-cache@0.5.0 + with: + key: docker-${{ runner.os }}-${{ hashFiles('go.sum') }} + + - name: Build the Docker image + run: make docker-build diff --git a/.github/workflows/e2e-test.yaml b/.github/workflows/e2e-test.yaml index 50c52c69e..e5b250e7a 100644 --- a/.github/workflows/e2e-test.yaml +++ b/.github/workflows/e2e-test.yaml @@ -24,6 +24,7 @@ on: - k3s - rke2 - default-cluster + - kubeadm-full - linodecluster - linodemachine - linodeobj diff --git a/Makefile b/Makefile index e167a8e4c..732f15168 100644 --- a/Makefile +++ b/Makefile @@ -153,7 +153,7 @@ test: generate fmt vet envtest ## Run tests. .PHONY: e2etest e2etest: generate local-release local-deploy chainsaw - GIT_REF=$(GIT_REF) $(CHAINSAW) test ./e2e --selector $(E2E_SELECTOR) $(E2E_FLAGS) + GIT_REF=$(GIT_REF) SSE_KEY=$$(openssl rand -base64 32) $(CHAINSAW) test ./e2e --selector $(E2E_SELECTOR) $(E2E_FLAGS) local-deploy: kind ctlptl tilt kustomize clusterctl $(CTLPTL) apply -f .tilt/ctlptl-config.yaml diff --git a/e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/assert-capi-resources.yaml b/e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/assert-capi-resources.yaml new file mode 100644 index 000000000..48008146c --- /dev/null +++ b/e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/assert-capi-resources.yaml @@ -0,0 +1,79 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: caaph-controller-manager + namespace: caaph-system +status: + availableReplicas: 1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: capi-operator-cluster-api-operator + namespace: capi-operator-system +status: + availableReplicas: 1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: capi-controller-manager + namespace: capi-system +status: + availableReplicas: 1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: capl-controller-manager + namespace: capl-system +status: + availableReplicas: 1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cert-manager + namespace: cert-manager +status: + availableReplicas: 1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cert-manager-cainjector + namespace: cert-manager +status: + availableReplicas: 1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cert-manager-webhook + namespace: cert-manager +status: + availableReplicas: 1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: coredns + namespace: kube-system +status: + availableReplicas: 2 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: capi-kubeadm-bootstrap-controller-manager + namespace: kubeadm-bootstrap-system +status: + availableReplicas: 1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: capi-kubeadm-control-plane-controller-manager + namespace: kubeadm-control-plane-system +status: + availableReplicas: 1 diff --git a/e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/assert-child-cluster-daemonsets.yaml b/e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/assert-child-cluster-daemonsets.yaml new file mode 100644 index 000000000..fa12ba942 --- /dev/null +++ b/e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/assert-child-cluster-daemonsets.yaml @@ -0,0 +1,35 @@ +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: ccm-linode + namespace: kube-system +status: + currentNumberScheduled: 1 + desiredNumberScheduled: 1 + numberAvailable: 1 + numberMisscheduled: 0 + numberReady: 1 +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: cilium + namespace: kube-system +status: + currentNumberScheduled: 2 + desiredNumberScheduled: 2 + numberAvailable: 2 + numberMisscheduled: 0 + numberReady: 2 +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: csi-linode-node + namespace: kube-system +status: + currentNumberScheduled: 2 + desiredNumberScheduled: 2 + numberAvailable: 2 + numberMisscheduled: 0 + numberReady: 2 diff --git a/e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/assert-child-cluster-deployments.yaml b/e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/assert-child-cluster-deployments.yaml new file mode 100644 index 000000000..b33bfc474 --- /dev/null +++ b/e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/assert-child-cluster-deployments.yaml @@ -0,0 +1,35 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cilium-operator + namespace: kube-system +status: + availableReplicas: 2 + readyReplicas: 2 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: coredns + namespace: kube-system +status: + availableReplicas: 2 + readyReplicas: 2 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hubble-relay + namespace: kube-system +status: + availableReplicas: 1 + readyReplicas: 1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: hubble-ui + namespace: kube-system +status: + availableReplicas: 1 + readyReplicas: 1 diff --git a/e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/assert-child-cluster-resources.yaml b/e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/assert-child-cluster-resources.yaml new file mode 100644 index 000000000..f3b7d83b6 --- /dev/null +++ b/e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/assert-child-cluster-resources.yaml @@ -0,0 +1,76 @@ +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 +kind: LinodeMachine +metadata: + labels: + cluster.x-k8s.io/cluster-name: ($cluster) +spec: + region: (env('LINODE_REGION')) + type: g6-standard-2 +status: + ready: true + instanceState: running +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Machine +metadata: + labels: + cluster.x-k8s.io/cluster-name: ($cluster) +spec: + clusterName: ($cluster) +status: + bootstrapReady: true + infrastructureReady: true +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: MachineDeployment +metadata: + labels: + cluster.x-k8s.io/cluster-name: ($cluster) +spec: + clusterName: ($cluster) + replicas: 1 +status: + readyReplicas: 1 + unavailableReplicas: 0 + availableReplicas: 1 +--- +apiVersion: controlplane.cluster.x-k8s.io/v1beta1 +kind: KubeadmControlPlane +metadata: + labels: + cluster.x-k8s.io/cluster-name: ($cluster) +status: + readyReplicas: 1 + unavailableReplicas: 0 + ready: true +--- +apiVersion: addons.cluster.x-k8s.io/v1alpha1 +kind: HelmReleaseProxy +metadata: + labels: + cluster.x-k8s.io/cluster-name: ($cluster) +status: + conditions: + - type: Ready + status: "True" + - type: ClusterAvailable + status: "True" + - type: HelmReleaseReady + status: "True" + status: deployed +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 +kind: LinodeCluster +metadata: + name: ($cluster) +status: + ready: true +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + name: ($cluster) +status: + infrastructureReady: true + phase: Provisioned diff --git a/e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/assert-child-cluster-statefulsets.yaml b/e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/assert-child-cluster-statefulsets.yaml new file mode 100644 index 000000000..7648fbccb --- /dev/null +++ b/e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/assert-child-cluster-statefulsets.yaml @@ -0,0 +1,7 @@ +apiVersion: apps/v1 +kind: StatefulSet +metadata: + name: csi-linode-controller + namespace: kube-system +status: + replicas: 1 diff --git a/e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/assert-konnectivity-resources.yaml b/e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/assert-konnectivity-resources.yaml new file mode 100644 index 000000000..bd2066846 --- /dev/null +++ b/e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/assert-konnectivity-resources.yaml @@ -0,0 +1,20 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: konnectivity-agent + namespace: kube-system +status: + availableReplicas: 1 + readyReplicas: 1 +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: konnectivity-server + namespace: kube-system +status: + currentNumberScheduled: 1 + desiredNumberScheduled: 1 + numberAvailable: 1 + numberMisscheduled: 0 + numberReady: 1 diff --git a/e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/chainsaw-test.yaml b/e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/chainsaw-test.yaml new file mode 100644 index 000000000..4af7d154c --- /dev/null +++ b/e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/chainsaw-test.yaml @@ -0,0 +1,266 @@ +# yaml-language-server: $schema=https://raw.githubusercontent.com/kyverno/chainsaw/main/.schemas/json/test-chainsaw-v1alpha1.json +apiVersion: chainsaw.kyverno.io/v1alpha1 +kind: Test +metadata: + creationTimestamp: null + name: kubeadm-full-capl-cluster + # Labels to allow the test to be triggered based on selector flag + labels: + all: + kubeadm-full: + flavors: +spec: + bindings: + # Identifier for the E2E test run + - name: run + value: (join('-', ['e2e', 'kdm-ft', env('GIT_REF')])) + - name: cluster + # Format the cluster name + value: (trim((truncate(($run), `32`)), '-')) + template: true + steps: + # Test for initial resources allocation + - name: Testing all CAPI provider resources + try: + - assert: + file: assert-capi-resources.yaml + + # Test for generating cluster using clusterctl + - name: Generate cluster using clusterctl + try: + - script: + env: + - name: CLUSTER + value: ($cluster) + - name: NAMESPACE + value: ($namespace) + - name: CLUSTERCTL_CONFIG + value: (env('CLUSTERCTL_CONFIG')) + - name: SSE_KEY + value: (env('SSE_KEY')) + - name: KONNECTIVITY_AGENT_REPLICAS + value: '1' # Here, 1 is set for testing purposes. Default is 3. + - name: CLUSTER_AUTOSCALER_VERSION + value: 'v1.29.4' + content: | + set -e + if [ -z "$SSE_KEY" ]; then + echo "SSE_KEY not set" >&2 + exit 1 + else + clusterctl generate cluster $CLUSTER -n $NAMESPACE \ + --flavor kubeadm-full --kubernetes-version v1.29.1 \ + --infrastructure local-linode:v0.0.0 \ + --control-plane-machine-count 1 --worker-machine-count 1 \ + --config ${CLUSTERCTL_CONFIG:=${HOME}/.cluster-api/clusterctl.yaml} > kubeadm-full-cluster.yaml + fi + check: + ($error == null): true + + # Test for applying created yaml + - name: Testing all available resources + try: + - apply: + file: kubeadm-full-cluster.yaml + - assert: + file: assert-child-cluster-resources.yaml + catch: + - describe: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 + kind: LinodeMachine + - describe: + apiVersion: cluster.x-k8s.io/v1beta1 + kind: Machine + - describe: + apiVersion: cluster.x-k8s.io/v1beta1 + kind: MachineDeployment + - describe: + apiVersion: controlplane.cluster.x-k8s.io/v1beta1 + kind: KubeadmControlPlane + - describe: + apiVersion: addons.cluster.x-k8s.io/v1alpha1 + kind: HelmReleaseProxy + - describe: + cluster: ($cluster) + apiVersion: cluster.x-k8s.io/v1beta1 + kind: Cluster + namespace: ($namespace) + - describe: + cluster: ($cluster) + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 + kind: LinodeCluster + namespace: ($namespace) + + # Test to check if linodes are created + - name: Testing to see if the Linodes are created + try: + - script: + env: + - name: TARGET_API + value: api.linode.com + - name: TARGET_API_VERSION + value: v4beta + - name: URI + value: linode/instances + - name: FILTER + value: (to_string({"tags":($cluster)})) + content: | + set -e + curl -s \ + -H "Authorization: Bearer $LINODE_TOKEN" \ + -H "X-Filter: $FILTER" \ + -H "Content-Type: application/json" \ + "https://$TARGET_API/$TARGET_API_VERSION/$URI" + check: + ($error): ~ + (json_parse($stdout)): + results: 2 + + # Get the KUBECONFIG of the child cluster for later use + - name: Get child cluster kubeconfig + try: + - script: + env: + - name: CLUSTER + value: ($cluster) + - name: NAMESPACE + value: ($namespace) + - name: CLUSTERCTL_CONFIG + value: (env('CLUSTERCTL_CONFIG')) + content: | + set -e + clusterctl get kubeconfig $CLUSTER -n $NAMESPACE > kubeadm-full-cluster-kubeconfig.yaml + check: + ($error == null): true + + # Test child cluster deployments + - clusters: + kubeadm-full-cluster: + kubeconfig: ./kubeadm-full-cluster-kubeconfig.yaml + name: Testing child cluster deployments + try: + - assert: + cluster: kubeadm-full-cluster + file: assert-child-cluster-deployments.yaml + catch: + - describe: + cluster: kubeadm-full-cluster + apiVersion: apps/v1 + kind: Deployment + namespace: kube-system + + # Test for child cluster DaemonSets + - clusters: + kubeadm-full-cluster: + kubeconfig: ./kubeadm-full-cluster-kubeconfig.yaml + name: Testing child cluster DaemonSets + try: + - assert: + cluster: kubeadm-full-cluster + file: assert-child-cluster-daemonsets.yaml + catch: + - describe: + cluster: kubeadm-full-cluster + apiVersion: apps/v1 + kind: DaemonSet + namespace: kube-system + + # Test for child cluster StatefulSets + - clusters: + kubeadm-full-cluster: + kubeconfig: ./kubeadm-full-cluster-kubeconfig.yaml + name: Testing child cluster StatefulSets + try: + - assert: + cluster: kubeadm-full-cluster + file: assert-child-cluster-statefulsets.yaml + catch: + - describe: + cluster: kubeadm-full-cluster + apiVersion: apps/v1 + kind: StatefulSet + namespace: kube-system + + # Test Konnectivity server and agent + - clusters: + kubeadm-full-cluster: + kubeconfig: ./kubeadm-full-cluster-kubeconfig.yaml + name: Testing konnectivity resources + try: + - assert: + cluster: kubeadm-full-cluster + file: assert-konnectivity-resources.yaml + catch: + - describe: + cluster: kubeadm-full-cluster + apiVersion: apps/v1 + kind: Deployment + namespace: kube-system + - describe: + cluster: kubeadm-full-cluster + apiVersion: apps/v1 + kind: DaemonSet + namespace: kube-system + + # Test to check if konnectivity is working + - clusters: + kubeadm-full-cluster: + kubeconfig: ./kubeadm-full-cluster-kubeconfig.yaml + name: Testing to check if logs are retrievable + try: + - script: + content: | + KUBECONFIG=./kubeadm-full-cluster-kubeconfig.yaml kubectl logs csi-linode-controller-0 -n kube-system + check: + ($error == null): true + + # Test to check if child cluster is deleted + - name: Testing to see if child cluster is deleted + try: + - delete: + ref: + apiVersion: cluster.x-k8s.io/v1beta1 + kind: Cluster + name: ($cluster) + - delete: + ref: + apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 + kind: LinodeVPC + name: ($cluster) + - error: + file: check-child-cluster-and-vpc-deleted.yaml + + # Test to check if linodes are deleted + - name: Testing to check if the linodes are deleted + try: + - script: + env: + - name: TARGET_API + value: api.linode.com + - name: TARGET_API_VERSION + value: v4beta + - name: URI + value: linode/instances + - name: FILTER + value: (to_string({"tags":($cluster)})) + content: | + set -e + curl -s \ + -H "Authorization: Bearer $LINODE_TOKEN" \ + -H "X-Filter: $FILTER" \ + -H "Content-Type: application/json" \ + "https://$TARGET_API/$TARGET_API_VERSION/$URI" + check: + ($error): ~ + (json_parse($stdout)): + results: 0 + + # Delete generated manifests + - name: Delete generated child cluster manifest yaml + try: + - script: + content: | + rm -f kubeadm-full-cluster.yaml + rm -f kubeadm-full-cluster-kubeconfig.yaml + check: + ($error == null): true diff --git a/e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/check-child-cluster-and-vpc-deleted.yaml b/e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/check-child-cluster-and-vpc-deleted.yaml new file mode 100644 index 000000000..26a6d0e44 --- /dev/null +++ b/e2e/capl-cluster-flavors/kubeadm-full-capl-cluster/check-child-cluster-and-vpc-deleted.yaml @@ -0,0 +1,10 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 +kind: LinodeMachine +metadata: + labels: + cluster.x-k8s.io/cluster-name: ($cluster) +--- +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha2 +kind: LinodeVPC +metadata: + name: ($cluster) diff --git a/templates/addons/cluster-autoscaler/cluster-autoscaler.yaml b/templates/addons/cluster-autoscaler/cluster-autoscaler.yaml index c55e5f637..0205b8e84 100644 --- a/templates/addons/cluster-autoscaler/cluster-autoscaler.yaml +++ b/templates/addons/cluster-autoscaler/cluster-autoscaler.yaml @@ -16,7 +16,7 @@ spec: app: ${CLUSTER_NAME}-cluster-autoscaler spec: containers: - - image: registry.k8s.io/autoscaling/cluster-autoscaler:${CLUSTER_AUTOSCALER_VERSION:=${KUBERNETES_VERSION%.*}.0} + - image: registry.k8s.io/autoscaling/cluster-autoscaler:${CLUSTER_AUTOSCALER_VERSION:=v1.29.4} name: cluster-autoscaler command: [/cluster-autoscaler] args: diff --git a/templates/addons/konnectivity/konnectivity.yaml b/templates/addons/konnectivity/konnectivity.yaml index d0bbfd005..659028c80 100644 --- a/templates/addons/konnectivity/konnectivity.yaml +++ b/templates/addons/konnectivity/konnectivity.yaml @@ -18,3 +18,4 @@ spec: proxyServerHost: {{ .InfraCluster.spec.controlPlaneEndpoint.host }} proxyServerPort: ${KONNECTIVITY_PORT:=8132} serverCount: ${CONTROL_PLANE_MACHINE_COUNT} + agentReplicas: ${KONNECTIVITY_AGENT_REPLICAS:=3}