diff --git a/test/e2e/cluster_upgrade.go b/test/e2e/cluster_upgrade.go new file mode 100644 index 00000000..d53d9fa8 --- /dev/null +++ b/test/e2e/cluster_upgrade.go @@ -0,0 +1,185 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + "context" + "fmt" + "os" + "path/filepath" + + . "github.com/onsi/ginkgo/v2" + . "github.com/onsi/gomega" + + corev1 "k8s.io/api/core/v1" + "k8s.io/utils/ptr" + + "sigs.k8s.io/cluster-api/test/framework" + "sigs.k8s.io/cluster-api/test/framework/clusterctl" + "sigs.k8s.io/cluster-api/util" +) + +// ClusterUpgradeSpecInput is the input for ClusterUpgradeConformanceSpec. +type ClusterUpgradeSpecInput struct { + E2EConfig *clusterctl.E2EConfig + ClusterctlConfigPath string + BootstrapClusterProxy framework.ClusterProxy + ArtifactFolder string + SkipCleanup bool + ControlPlaneWaiters ControlPlaneWaiters + + // InfrastructureProviders specifies the infrastructure to use for clusterctl + // operations (Example: get cluster templates). + // Note: In most cases this need not be specified. It only needs to be specified when + // multiple infrastructure providers (ex: CAPD + in-memory) are installed on the cluster as clusterctl will not be + // able to identify the default. + InfrastructureProvider *string + + // ControlPlaneMachineCount is used in `config cluster` to configure the count of the control plane machines used in the test. + // Default is 1. + ControlPlaneMachineCount *int64 + + // WorkerMachineCount is used in `config cluster` to configure the count of the worker machines used in the test. + // NOTE: If the WORKER_MACHINE_COUNT var is used multiple times in the cluster template, the absolute count of + // worker machines is a multiple of WorkerMachineCount. + // Default is 2. + WorkerMachineCount *int64 + + // Flavor to use when creating the cluster for testing, "upgrades" is used if not specified. + Flavor *string +} + +func ClusterUpgradeSpec(ctx context.Context, inputGetter func() ClusterUpgradeSpecInput) { + const ( + specName = "workload-cluster-upgrade" + ) + + var ( + input ClusterUpgradeSpecInput + namespace *corev1.Namespace + cancelWatches context.CancelFunc + + controlPlaneMachineCount int64 + workerMachineCount int64 + + result *ApplyClusterTemplateAndWaitResult + clusterName string + clusterctlLogFolder string + ) + + BeforeEach(func() { + Expect(ctx).NotTo(BeNil(), "ctx is required for %s spec", specName) + input = inputGetter() + Expect(input.E2EConfig).ToNot(BeNil(), "Invalid argument. input.E2EConfig can't be nil when calling %s spec", specName) + Expect(input.ClusterctlConfigPath).To(BeAnExistingFile(), "Invalid argument. input.ClusterctlConfigPath must be an existing file when calling %s spec", specName) + Expect(input.BootstrapClusterProxy).ToNot(BeNil(), "Invalid argument. input.BootstrapClusterProxy can't be nil when calling %s spec", specName) + Expect(os.MkdirAll(input.ArtifactFolder, 0750)).To(Succeed(), "Invalid argument. input.ArtifactFolder can't be created for %s spec", specName) + + Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersion)) + Expect(input.E2EConfig.Variables).To(HaveKey(KubernetesVersionUpgradeTo)) + + clusterName = fmt.Sprintf("capik3s-cluster-upgrade-%s", util.RandomString(6)) + + if input.ControlPlaneMachineCount == nil { + controlPlaneMachineCount = 1 + } else { + controlPlaneMachineCount = *input.ControlPlaneMachineCount + } + + if input.WorkerMachineCount == nil { + workerMachineCount = 2 + } else { + workerMachineCount = *input.WorkerMachineCount + } + + // Setup a Namespace where to host objects for this spec and create a watcher for the namespace events. + namespace, cancelWatches = setupSpecNamespace(ctx, specName, input.BootstrapClusterProxy, input.ArtifactFolder) + + result = new(ApplyClusterTemplateAndWaitResult) + + clusterctlLogFolder = filepath.Join(input.ArtifactFolder, "clusters", input.BootstrapClusterProxy.GetName()) + }) + + AfterEach(func() { + cleanInput := cleanupInput{ + SpecName: specName, + Cluster: result.Cluster, + ClusterProxy: input.BootstrapClusterProxy, + Namespace: namespace, + CancelWatches: cancelWatches, + IntervalsGetter: input.E2EConfig.GetIntervals, + SkipCleanup: input.SkipCleanup, + ArtifactFolder: input.ArtifactFolder, + } + + dumpSpecResourcesAndCleanup(ctx, cleanInput) + }) + + It("Should create and upgrade a workload cluster", func() { + By("Creating a workload cluster") + ApplyClusterTemplateAndWait(ctx, ApplyClusterTemplateAndWaitInput{ + ClusterProxy: input.BootstrapClusterProxy, + ConfigCluster: clusterctl.ConfigClusterInput{ + LogFolder: clusterctlLogFolder, + ClusterctlConfigPath: input.ClusterctlConfigPath, + KubeconfigPath: input.BootstrapClusterProxy.GetKubeconfigPath(), + InfrastructureProvider: *input.InfrastructureProvider, + Flavor: ptr.Deref(input.Flavor, ""), + Namespace: namespace.Name, + ClusterName: clusterName, + KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersion), + ControlPlaneMachineCount: &controlPlaneMachineCount, + WorkerMachineCount: &workerMachineCount, + }, + ControlPlaneWaiters: input.ControlPlaneWaiters, + WaitForClusterIntervals: input.E2EConfig.GetIntervals(specName, "wait-cluster"), + WaitForControlPlaneIntervals: input.E2EConfig.GetIntervals(specName, "wait-control-plane"), + WaitForMachineDeployments: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"), + }, result) + + By("Upgrading the Kubernetes control-plane") + UpgradeControlPlaneAndWaitForUpgrade(ctx, UpgradeControlPlaneAndWaitForUpgradeInput{ + ClusterProxy: input.BootstrapClusterProxy, + Cluster: result.Cluster, + ControlPlane: result.ControlPlane, + KubernetesUpgradeVersion: input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo), + WaitForMachinesToBeUpgraded: input.E2EConfig.GetIntervals(specName, "wait-machine-upgrade"), + }) + + By("Upgrading the machine deployment") + framework.UpgradeMachineDeploymentsAndWait(ctx, framework.UpgradeMachineDeploymentsAndWaitInput{ + ClusterProxy: input.BootstrapClusterProxy, + Cluster: result.Cluster, + UpgradeVersion: input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo), + MachineDeployments: result.MachineDeployments, + WaitForMachinesToBeUpgraded: input.E2EConfig.GetIntervals(specName, "wait-worker-nodes"), + }) + + By("Waiting until nodes are ready") + workloadProxy := input.BootstrapClusterProxy.GetWorkloadCluster(ctx, namespace.Name, result.Cluster.Name) + workloadClient := workloadProxy.GetClient() + framework.WaitForNodesReady(ctx, framework.WaitForNodesReadyInput{ + Lister: workloadClient, + KubernetesVersion: input.E2EConfig.GetVariable(KubernetesVersionUpgradeTo), + Count: int(result.ExpectedTotalNodes()), + WaitForNodesReady: input.E2EConfig.GetIntervals(specName, "wait-nodes-ready"), + }) + }) +} diff --git a/test/e2e/cluster_upgrade_test.go b/test/e2e/cluster_upgrade_test.go new file mode 100644 index 00000000..4185e8d8 --- /dev/null +++ b/test/e2e/cluster_upgrade_test.go @@ -0,0 +1,57 @@ +//go:build e2e +// +build e2e + +/* +Copyright 2021 The Kubernetes Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package e2e + +import ( + . "github.com/onsi/ginkgo/v2" + "k8s.io/utils/ptr" +) + +var _ = Describe("Workload cluster upgrade [K3s-Upgrade]", func() { + Context("Upgrading a cluster with 1 control plane", func() { + ClusterUpgradeSpec(ctx, func() ClusterUpgradeSpecInput { + return ClusterUpgradeSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + InfrastructureProvider: ptr.To("docker"), + ControlPlaneMachineCount: ptr.To[int64](1), + WorkerMachineCount: ptr.To[int64](2), + } + }) + }) + + Context("Upgrading a cluster with HA control plane", func() { + ClusterUpgradeSpec(ctx, func() ClusterUpgradeSpecInput { + return ClusterUpgradeSpecInput{ + E2EConfig: e2eConfig, + ClusterctlConfigPath: clusterctlConfigPath, + BootstrapClusterProxy: bootstrapClusterProxy, + ArtifactFolder: artifactFolder, + SkipCleanup: skipCleanup, + InfrastructureProvider: ptr.To("docker"), + ControlPlaneMachineCount: ptr.To[int64](3), + WorkerMachineCount: ptr.To[int64](1), + } + }) + }) +}) diff --git a/test/e2e/config/k3s-docker.yaml b/test/e2e/config/k3s-docker.yaml index 7b5fe99b..33204722 100644 --- a/test/e2e/config/k3s-docker.yaml +++ b/test/e2e/config/k3s-docker.yaml @@ -75,9 +75,11 @@ providers: targetName: "metadata.yaml" variables: - KUBERNETES_VERSION: "v1.28.6" - KUBERNETES_VERSION_MANAGEMENT: "v1.28.6" + KUBERNETES_VERSION_MANAGEMENT: "v1.28.0" + KUBERNETES_VERSION: "v1.28.6+k3s2" + KUBERNETES_VERSION_UPGRADE_TO: "v1.28.7+k3s1" IP_FAMILY: "IPv4" + KIND_IMAGE_VERSION: "v1.28.0" intervals: # The array is defined as [timeout, polling interval] @@ -85,10 +87,10 @@ intervals: default/wait-controllers: ["3m", "10s"] default/wait-cluster: ["5m", "10s"] default/wait-control-plane: ["10m", "10s"] - default/wait-worker-nodes: ["5m", "10s"] + default/wait-worker-nodes: ["10m", "10s"] default/wait-machine-pool-nodes: ["5m", "10s"] default/wait-delete-cluster: ["3m", "10s"] - default/wait-machine-upgrade: ["20m", "10s"] + default/wait-machine-upgrade: ["30m", "10s"] default/wait-machine-pool-upgrade: ["5m", "10s"] default/wait-nodes-ready: ["10m", "10s"] default/wait-machine-remediation: ["5m", "10s"] diff --git a/test/e2e/data/infrastructure-docker/cluster-template.yaml b/test/e2e/data/infrastructure-docker/cluster-template.yaml index a6e796a2..d555a71c 100644 --- a/test/e2e/data/infrastructure-docker/cluster-template.yaml +++ b/test/e2e/data/infrastructure-docker/cluster-template.yaml @@ -1,4 +1,4 @@ -# TODO: copied from https://github.com/k3s-io/cluster-api-k3s/pull/93/files#diff-c4a336ec56832a2ff7aed26c94d0d67ae3a0e6139d30701cc53c0f0962fe8cca +# TODO: copied and modified from https://github.com/k3s-io/cluster-api-k3s/pull/93/files#diff-c4a336ec56832a2ff7aed26c94d0d67ae3a0e6139d30701cc53c0f0962fe8cca # should be the same as samples/docker/quickstart.yaml in the future # for testing the quickstart scenario apiVersion: cluster.x-k8s.io/v1beta1 @@ -40,7 +40,7 @@ spec: kind: DockerMachineTemplate name: ${CLUSTER_NAME}-control-plane replicas: ${CONTROL_PLANE_MACHINE_COUNT} - version: ${KUBERNETES_VERSION}+k3s2 + version: ${KUBERNETES_VERSION} kthreesConfigSpec: serverConfig: tlsSan: @@ -53,7 +53,7 @@ metadata: spec: template: spec: - customImage: kindest/node:${KUBERNETES_VERSION} + customImage: kindest/node:${KIND_IMAGE_VERSION} --- apiVersion: cluster.x-k8s.io/v1beta1 kind: MachineDeployment @@ -65,7 +65,15 @@ spec: selector: matchLabels: cluster.x-k8s.io/cluster-name: ${CLUSTER_NAME} + + # This label will be needed for upgrade test + # it will be used as a selector for only selecting + # machines belonging to this machine deployment + cluster.x-k8s.io/deployment-name: worker-md-0 template: + metadata: + labels: + cluster.x-k8s.io/deployment-name: worker-md-0 spec: version: ${KUBERNETES_VERSION} clusterName: ${CLUSTER_NAME} @@ -86,7 +94,7 @@ metadata: spec: template: spec: - customImage: kindest/node:${KUBERNETES_VERSION} + customImage: kindest/node:${KIND_IMAGE_VERSION} --- apiVersion: bootstrap.cluster.x-k8s.io/v1beta1 kind: KThreesConfigTemplate @@ -95,4 +103,3 @@ metadata: spec: template: spec: - version: ${KUBERNETES_VERSION}+k3s2 \ No newline at end of file diff --git a/test/e2e/helpers.go b/test/e2e/helpers.go index c44abb47..0fb6b693 100644 --- a/test/e2e/helpers.go +++ b/test/e2e/helpers.go @@ -31,6 +31,7 @@ import ( "sigs.k8s.io/cluster-api/test/framework" "sigs.k8s.io/cluster-api/test/framework/clusterctl" + "sigs.k8s.io/cluster-api/util/patch" "sigs.k8s.io/controller-runtime/pkg/client" controlplanev1 "github.com/k3s-io/cluster-api-k3s/controlplane/api/v1beta1" @@ -549,6 +550,44 @@ func WaitForControlPlaneAndMachinesReady(ctx context.Context, input WaitForContr }) } +// UpgradeControlPlaneAndWaitForUpgradeInput is the input type for UpgradeControlPlaneAndWaitForUpgrade. +type UpgradeControlPlaneAndWaitForUpgradeInput struct { + ClusterProxy framework.ClusterProxy + Cluster *clusterv1.Cluster + ControlPlane *controlplanev1.KThreesControlPlane + KubernetesUpgradeVersion string + WaitForMachinesToBeUpgraded []interface{} +} + +// UpgradeControlPlaneAndWaitForUpgrade upgrades a KubeadmControlPlane and waits for it to be upgraded. +func UpgradeControlPlaneAndWaitForUpgrade(ctx context.Context, input UpgradeControlPlaneAndWaitForUpgradeInput) { + Expect(ctx).NotTo(BeNil(), "ctx is required for UpgradeControlPlaneAndWaitForUpgrade") + Expect(input.ClusterProxy).ToNot(BeNil(), "Invalid argument. input.ClusterProxy can't be nil when calling UpgradeControlPlaneAndWaitForUpgrade") + Expect(input.Cluster).ToNot(BeNil(), "Invalid argument. input.Cluster can't be nil when calling UpgradeControlPlaneAndWaitForUpgrade") + Expect(input.ControlPlane).ToNot(BeNil(), "Invalid argument. input.ControlPlane can't be nil when calling UpgradeControlPlaneAndWaitForUpgrade") + Expect(input.KubernetesUpgradeVersion).ToNot(BeNil(), "Invalid argument. input.KubernetesUpgradeVersion can't be empty when calling UpgradeControlPlaneAndWaitForUpgrade") + + mgmtClient := input.ClusterProxy.GetClient() + + Byf("Patching the new kubernetes version to KCP") + patchHelper, err := patch.NewHelper(input.ControlPlane, mgmtClient) + Expect(err).ToNot(HaveOccurred()) + + input.ControlPlane.Spec.Version = input.KubernetesUpgradeVersion + + Eventually(func() error { + return patchHelper.Patch(ctx, input.ControlPlane) + }, retryableOperationTimeout, retryableOperationInterval).Should(Succeed(), "Failed to patch the new kubernetes version to KCP %s", klog.KObj(input.ControlPlane)) + + Byf("Waiting for control-plane machines to have the upgraded kubernetes version") + framework.WaitForControlPlaneMachinesToBeUpgraded(ctx, framework.WaitForControlPlaneMachinesToBeUpgradedInput{ + Lister: mgmtClient, + Cluster: input.Cluster, + MachineCount: int(*input.ControlPlane.Spec.Replicas), + KubernetesUpgradeVersion: input.KubernetesUpgradeVersion, + }, input.WaitForMachinesToBeUpgraded...) +} + // byClusterOptions returns a set of ListOptions that allows to identify all the objects belonging to a Cluster. func byClusterOptions(name, namespace string) []client.ListOption { return []client.ListOption{