Add cluster-upgrade e2e tests

.github/workflows/pr.yaml

@@ -6,7 +6,6 @@ on:
 
 env:
   GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
-  GO_VERSION: "1.20"
 
 jobs:
   validate:
@@ -26,7 +25,6 @@ jobs:
       - uses: actions/setup-go@v4
         with:
           go-version-file: "./go.mod"
-          go-version: "${{ env.GO_VERSION }}"
           cache: true
 
       - name: ${{ matrix.target }}

.github/workflows/release.yaml

@@ -6,7 +6,6 @@ on:
       - "v*"
 env:
   REGISTRY: ghcr.io/${{ github.repository }}
-  GO_VERSION: "1.20"
 
 permissions:
   packages: write
@@ -26,7 +25,7 @@ jobs:
 
       - uses: actions/setup-go@v4
         with:
-          go-version: "${{ env.GO_VERSION }}"
+          go-version-file: "./go.mod"
           cache: true
 
       - name: Set up QEMU

test/e2e/cluster_upgrade.go

@@ -0,0 +1,185 @@
+//go:build e2e
+// +build e2e
+
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	"context"
+	"fmt"
+	"os"
+	"path/filepath"
+
+	. "github.com/onsi/ginkgo/v2"
+	. "github.com/onsi/gomega"
+
+	corev1 "k8s.io/api/core/v1"
+	"k8s.io/utils/ptr"
+
+	"sigs.k8s.io/cluster-api/test/framework"
+	"sigs.k8s.io/cluster-api/test/framework/clusterctl"
+	"sigs.k8s.io/cluster-api/util"
+)
+
+// ClusterUpgradeSpecInput is the input for ClusterUpgradeConformanceSpec.
+type ClusterUpgradeSpecInput struct {
+	E2EConfig             *clusterctl.E2EConfig
+	ClusterctlConfigPath  string
+	BootstrapClusterProxy framework.ClusterProxy
+	ArtifactFolder        string
+	SkipCleanup           bool
+	ControlPlaneWaiters   ControlPlaneWaiters
+
+	// InfrastructureProviders specifies the infrastructure to use for clusterctl
+	// operations (Example: get cluster templates).
+	// Note: In most cases this need not be specified. It only needs to be specified when
+	// multiple infrastructure providers (ex: CAPD + in-memory) are installed on the cluster as clusterctl will not be
+	// able to identify the default.
+	InfrastructureProvider *string
+
+	// ControlPlaneMachineCount is used in `config cluster` to configure the count of the control plane machines used in the test.
+	// Default is 1.
+	ControlPlaneMachineCount *int64
+
+	// WorkerMachineCount is used in `config cluster` to configure the count of the worker machines used in the test.
+	// NOTE: If the WORKER_MACHINE_COUNT var is used multiple times in the cluster template, the absolute count of
+	// worker machines is a multiple of WorkerMachineCount.
+	// Default is 2.
+	WorkerMachineCount *int64
+
+	// Flavor to use when creating the cluster for testing, "upgrades" is used if not specified.
+	Flavor *string
+}
+
+func ClusterUpgradeSpec(ctx context.Context, inputGetter func() ClusterUpgradeSpecInput) {
+	const (
+		specName = "workload-cluster-upgrade"
+	)
+
+	var (
+		input         ClusterUpgradeSpecInput
+		namespace     *corev1.Namespace
+		cancelWatches context.CancelFunc
+
+		controlPlaneMachineCount int64
+		workerMachineCount       int64
+
+		result              *ApplyClusterTemplateAndWaitResult
+		clusterName         string
+		clusterctlLogFolder string
+	)
+
+	BeforeEach(func() {
+		Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName)
+		input = inputGetter()
+		Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName)
+		Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling %s spec", specName)
+		Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName)
+		Expect(os.MkdirAll(input.ArtifactFolder, 0750)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName)
+
+		Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion))
+		Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersionUpgradeTo))
+
+		clusterName = fmt.Sprintf("capik3s-cluster-upgrade-%s", util.RandomString(6))
+
+		if input.ControlPlaneMachineCount == nil {
+			controlPlaneMachineCount = 1
+		} else {
+			controlPlaneMachineCount = *input.ControlPlaneMachineCount
+		}
+
+		if input.WorkerMachineCount == nil {
+			workerMachineCount = 2
+		} else {
+			workerMachineCount = *input.WorkerMachineCount
+		}
+
+		// Setup a Namespace where to host objects for this spec and create a watcher for the namespace events.
+		namespace, cancelWatches = setupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder)
+
+		result = new(ApplyClusterTemplateAndWaitResult)
+
+		clusterctlLogFolder = filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName())
+	})
+
+	AfterEach(func() {
+		cleanInput := cleanupInput{
+			SpecName:        specName,
+			Cluster:         result.Cluster,
+			ClusterProxy:    input.BootstrapClusterProxy,
+			Namespace:       namespace,
+			CancelWatches:   cancelWatches,
+			IntervalsGetter: input.E2EConfig.GetIntervals,
+			SkipCleanup:     input.SkipCleanup,
+			ArtifactFolder:  input.ArtifactFolder,
+		}
+
+		dumpSpecResourcesAndCleanup(ctx, cleanInput)
+	})
+
+	It("Should create and upgrade a workload cluster", func() {
+		By("Creating a workload cluster")
+		ApplyClusterTemplateAndWait(ctx, ApplyClusterTemplateAndWaitInput{
+			ClusterProxy: input.BootstrapClusterProxy,
+			ConfigCluster: clusterctl.ConfigClusterInput{
+				LogFolder:                clusterctlLogFolder,
+				ClusterctlConfigPath:     input.ClusterctlConfigPath,
+				KubeconfigPath:           input.BootstrapClusterProxy.GetKubeconfigPath(),
+				InfrastructureProvider:   *input.InfrastructureProvider,
+				Flavor:                   ptr.Deref(input.Flavor, ""),
+				Namespace:                namespace.Name,
+				ClusterName:              clusterName,
+				KubernetesVersion:        input.E2EConfig.GetVariable(KubernetesVersion),
+				ControlPlaneMachineCount: &controlPlaneMachineCount,
+				WorkerMachineCount:       &workerMachineCount,
+			},
+			ControlPlaneWaiters:          input.ControlPlaneWaiters,
+			WaitForClusterIntervals:      input.E2EConfig.GetIntervals(specName, "wait-cluster"),
+			WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"),
+			WaitForMachineDeployments:    input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
+		}, result)
+
+		By("Upgrading the Kubernetes control-plane")
+		UpgradeControlPlaneAndWaitForUpgrade(ctx, UpgradeControlPlaneAndWaitForUpgradeInput{
+			ClusterProxy:                input.BootstrapClusterProxy,
+			Cluster:                     result.Cluster,
+			ControlPlane:                result.ControlPlane,
+			KubernetesUpgradeVersion:    input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo),
+			WaitForMachinesToBeUpgraded: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"),
+		})
+
+		By("Upgrading the machine deployment")
+		framework.UpgradeMachineDeploymentsAndWait(ctx, framework.UpgradeMachineDeploymentsAndWaitInput{
+			ClusterProxy:                input.BootstrapClusterProxy,
+			Cluster:                     result.Cluster,
+			UpgradeVersion:              input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo),
+			MachineDeployments:          result.MachineDeployments,
+			WaitForMachinesToBeUpgraded: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"),
+		})
+
+		By("Waiting until nodes are ready")
+		workloadProxy := input.BootstrapClusterProxy.GetWorkloadCluster(ctx, namespace.Name, result.Cluster.Name)
+		workloadClient := workloadProxy.GetClient()
+		framework.WaitForNodesReady(ctx, framework.WaitForNodesReadyInput{
+			Lister:            workloadClient,
+			KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo),
+			Count:             int(result.ExpectedTotalNodes()),
+			WaitForNodesReady: input.E2EConfig.GetIntervals(specName, "wait-nodes-ready"),
+		})
+	})
+}

test/e2e/cluster_upgrade_test.go

@@ -0,0 +1,57 @@
+//go:build e2e
+// +build e2e
+
+/*
+Copyright 2021 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package e2e
+
+import (
+	. "github.com/onsi/ginkgo/v2"
+	"k8s.io/utils/ptr"
+)
+
+var _ = Describe("Workload cluster upgrade [K3s-Upgrade]", func() {
+	Context("Upgrading a cluster with 1 control plane", func() {
+		ClusterUpgradeSpec(ctx, func() ClusterUpgradeSpecInput {
+			return ClusterUpgradeSpecInput{
+				E2EConfig:                e2eConfig,
+				ClusterctlConfigPath:     clusterctlConfigPath,
+				BootstrapClusterProxy:    bootstrapClusterProxy,
+				ArtifactFolder:           artifactFolder,
+				SkipCleanup:              skipCleanup,
+				InfrastructureProvider:   ptr.To("docker"),
+				ControlPlaneMachineCount: ptr.To[int64](1),
+				WorkerMachineCount:       ptr.To[int64](2),
+			}
+		})
+	})
+
+	Context("Upgrading a cluster with HA control plane", func() {
+		ClusterUpgradeSpec(ctx, func() ClusterUpgradeSpecInput {
+			return ClusterUpgradeSpecInput{
+				E2EConfig:                e2eConfig,
+				ClusterctlConfigPath:     clusterctlConfigPath,
+				BootstrapClusterProxy:    bootstrapClusterProxy,
+				ArtifactFolder:           artifactFolder,
+				SkipCleanup:              skipCleanup,
+				InfrastructureProvider:   ptr.To("docker"),
+				ControlPlaneMachineCount: ptr.To[int64](3),
+				WorkerMachineCount:       ptr.To[int64](1),
+			}
+		})
+	})
+})

test/e2e/config/k3s-docker.yaml

@@ -75,20 +75,22 @@ providers:
         targetName: "metadata.yaml"
 
 variables:
-  KUBERNETES_VERSION: "v1.28.6"
-  KUBERNETES_VERSION_MANAGEMENT: "v1.28.6"
+  KUBERNETES_VERSION_MANAGEMENT: "v1.28.0"
+  KUBERNETES_VERSION: "v1.28.6+k3s2"
+  KUBERNETES_VERSION_UPGRADE_TO: "v1.28.7+k3s1"
   IP_FAMILY: "IPv4"
+  KIND_IMAGE_VERSION: "v1.28.0"
 
 intervals:
   # The array is defined as [timeout, polling interval]
   # copied from https://github.com/kubernetes-sigs/cluster-api/blob/main/test/e2e/config/docker.yaml
   default/wait-controllers: ["3m", "10s"]
   default/wait-cluster: ["5m", "10s"]
   default/wait-control-plane: ["10m", "10s"]
-  default/wait-worker-nodes: ["5m", "10s"]
+  default/wait-worker-nodes: ["10m", "10s"]
   default/wait-machine-pool-nodes: ["5m", "10s"]
   default/wait-delete-cluster: ["3m", "10s"]
-  default/wait-machine-upgrade: ["20m", "10s"]
+  default/wait-machine-upgrade: ["30m", "10s"]
   default/wait-machine-pool-upgrade: ["5m", "10s"]
   default/wait-nodes-ready: ["10m", "10s"]
   default/wait-machine-remediation: ["5m", "10s"]

test/e2e/data/infrastructure-docker/cluster-template.yaml

@@ -1,4 +1,4 @@
-# TODO: copied from https://github.com/k3s-io/cluster-api-k3s/pull/93/files#diff-c4a336ec56832a2ff7aed26c94d0d67ae3a0e6139d30701cc53c0f0962fe8cca
+# TODO: copied and modified from https://github.com/k3s-io/cluster-api-k3s/pull/93/files#diff-c4a336ec56832a2ff7aed26c94d0d67ae3a0e6139d30701cc53c0f0962fe8cca
 # should be the same as samples/docker/quickstart.yaml in the future
 # for testing the quickstart scenario
 apiVersion: cluster.x-k8s.io/v1beta1
@@ -40,7 +40,7 @@ spec:
     kind: DockerMachineTemplate
     name: ${CLUSTER_NAME}-control-plane
   replicas: ${CONTROL_PLANE_MACHINE_COUNT}
-  version: ${KUBERNETES_VERSION}+k3s2
+  version: ${KUBERNETES_VERSION}
   kthreesConfigSpec:
     serverConfig:
       tlsSan:
@@ -53,7 +53,7 @@ metadata:
 spec:
   template:
     spec:
-      customImage: kindest/node:${KUBERNETES_VERSION}
+      customImage: kindest/node:${KIND_IMAGE_VERSION}
 ---
 apiVersion: cluster.x-k8s.io/v1beta1
 kind: MachineDeployment
@@ -65,7 +65,15 @@ spec:
   selector:
     matchLabels:
       cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME}
+
+      # This label will be needed for upgrade test
+      # it will be used as a selector for only selecting
+      # machines belonging to this machine deployment
+      cluster.x-k8s.io/deployment-name: worker-md-0
   template:
+    metadata:
+      labels:
+        cluster.x-k8s.io/deployment-name: worker-md-0
     spec:
       version: ${KUBERNETES_VERSION}
       clusterName: ${CLUSTER_NAME}
@@ -86,7 +94,7 @@ metadata:
 spec:
   template:
     spec:
-      customImage: kindest/node:${KUBERNETES_VERSION}
+      customImage: kindest/node:${KIND_IMAGE_VERSION}
 ---
 apiVersion: bootstrap.cluster.x-k8s.io/v1beta1
 kind: KThreesConfigTemplate
@@ -95,4 +103,3 @@ metadata:
 spec:
   template:
     spec:
-      version: ${KUBERNETES_VERSION}+k3s2