From 4a2acdaf9f6d0b59b7d4d6356fa73195c3beecf5 Mon Sep 17 00:00:00 2001 From: Richard Kovacs Date: Tue, 6 Feb 2024 22:20:43 +0100 Subject: [PATCH] Machine e2e test with bring your own VPC --- Makefile | 4 +- controller/linodemachine_controller.go | 5 +- .../linodemachine_controller_helpers.go | 10 +++- controller/linodevpc_controller.go | 3 +- e2e/Makefile | 40 +++++++++++++--- e2e/README.MD | 11 ++++- .../byovpc/00-assert.yaml | 15 ++++++ .../byovpc/01-assert.yaml | 8 ++++ .../byovpc/01-create-cluster.tpl.yml | 46 +++++++++++++++++++ .../byovpc/01-create-cluster.yaml | 11 +++++ .../byovpc/01-create-vpc.tpl.yml | 10 ++++ .../byovpc/01-vpc-create-options.json | 10 ++++ .../byovpc/02-assert.yaml | 10 ++++ .../byovpc/02-create-linodemachine.tpl.yml | 12 +++++ .../byovpc/02-create-linodemachine.yaml | 6 +++ .../byovpc/03-delete-linodemachine.yaml | 6 +++ .../byovpc/03-error.yaml | 4 ++ .../byovpc/04-verify-linode-instance.yaml | 5 ++ e2e/linodemachine-controller/byovpc/Makefile | 11 +++++ .../minimal/01-assert.yaml | 17 ------- .../minimal/01-create-cluster.yaml | 2 + .../minimal/02-create-linodemachine.yaml | 2 +- 22 files changed, 217 insertions(+), 31 deletions(-) create mode 100644 e2e/linodemachine-controller/byovpc/00-assert.yaml create mode 100644 e2e/linodemachine-controller/byovpc/01-assert.yaml create mode 100644 e2e/linodemachine-controller/byovpc/01-create-cluster.tpl.yml create mode 100644 e2e/linodemachine-controller/byovpc/01-create-cluster.yaml create mode 100644 e2e/linodemachine-controller/byovpc/01-create-vpc.tpl.yml create mode 100644 e2e/linodemachine-controller/byovpc/01-vpc-create-options.json create mode 100644 e2e/linodemachine-controller/byovpc/02-assert.yaml create mode 100644 e2e/linodemachine-controller/byovpc/02-create-linodemachine.tpl.yml create mode 100644 e2e/linodemachine-controller/byovpc/02-create-linodemachine.yaml create mode 100644 e2e/linodemachine-controller/byovpc/03-delete-linodemachine.yaml create mode 100644 e2e/linodemachine-controller/byovpc/03-error.yaml create mode 100644 e2e/linodemachine-controller/byovpc/04-verify-linode-instance.yaml create mode 100644 e2e/linodemachine-controller/byovpc/Makefile delete mode 100644 e2e/linodemachine-controller/minimal/01-assert.yaml diff --git a/Makefile b/Makefile index 96a3be68d..ceb9b0140 100644 --- a/Makefile +++ b/Makefile @@ -92,10 +92,12 @@ test: manifests generate fmt vet envtest ## Run tests. e2etest: make --no-print-directory _e2etest # Workaround to force the flag on Github Action -_e2etest: kind ctlptl tilt kuttl kustomize clusterctl envsubst manifests generate +_e2etest-infra: kind ctlptl tilt kuttl kustomize clusterctl @echo -n "LINODE_TOKEN=$(LINODE_TOKEN)" > config/default/.env.linode $(CTLPTL) apply -f .tilt/ctlptl-config.yaml $(TILT) ci --timeout 240s -f Tiltfile + +_e2etest: manifests generate envsubst _e2etest-infra ROOT_DIR="$(PWD)" $(KUTTL) test --config e2e/kuttl-config.yaml ##@ Build diff --git a/controller/linodemachine_controller.go b/controller/linodemachine_controller.go index 04d6a0531..a3a4003b2 100644 --- a/controller/linodemachine_controller.go +++ b/controller/linodemachine_controller.go @@ -29,6 +29,7 @@ import ( "github.com/linode/cluster-api-provider-linode/util/reconciler" "github.com/linode/linodego" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/client-go/tools/record" @@ -233,7 +234,7 @@ func (r *LinodeMachineReconciler) reconcile( } // Always close the scope when exiting this function so we can persist any LinodeMachine changes. - if patchErr := machineScope.Close(ctx); patchErr != nil && utilerrors.FilterOut(patchErr) != nil { + if patchErr := machineScope.Close(ctx); patchErr != nil && utilerrors.FilterOut(patchErr, apierrors.IsNotFound) != nil { logger.Error(patchErr, "failed to patch LinodeMachine") err = errors.Join(err, patchErr) @@ -324,7 +325,7 @@ func (r *LinodeMachineReconciler) reconcileCreate( if machineScope.LinodeCluster.Spec.VPCRef != nil { iface, err := r.getVPCInterfaceConfig(ctx, machineScope, createConfig.Interfaces, logger) if err != nil { - logger.Error(err, "Failed to get VPC interface confiog") + logger.Error(err, "Failed to get VPC interface config") return nil, err } diff --git a/controller/linodemachine_controller_helpers.go b/controller/linodemachine_controller_helpers.go index ecaa44e64..bf3c3ab2c 100644 --- a/controller/linodemachine_controller_helpers.go +++ b/controller/linodemachine_controller_helpers.go @@ -173,9 +173,14 @@ func (r *LinodeMachineReconciler) requestsForCluster(ctx context.Context, namesp continue } + infraNs := item.Spec.InfrastructureRef.Namespace + if infraNs == "" { + infraNs = item.Namespace + } + result = append(result, ctrl.Request{ NamespacedName: client.ObjectKey{ - Namespace: item.Namespace, + Namespace: infraNs, Name: item.Spec.InfrastructureRef.Name, }, }) @@ -187,6 +192,9 @@ func (r *LinodeMachineReconciler) requestsForCluster(ctx context.Context, namesp func (r *LinodeMachineReconciler) getVPCInterfaceConfig(ctx context.Context, machineScope *scope.MachineScope, existingIfaces []linodego.InstanceConfigInterfaceCreateOptions, logger logr.Logger) (*linodego.InstanceConfigInterfaceCreateOptions, error) { name := machineScope.LinodeCluster.Spec.VPCRef.Name namespace := machineScope.LinodeCluster.Spec.VPCRef.Namespace + if namespace == "" { + namespace = machineScope.LinodeCluster.Namespace + } logger = logger.WithValues("vpcName", name, "vpcNamespace", namespace) diff --git a/controller/linodevpc_controller.go b/controller/linodevpc_controller.go index 8beb96abc..12dac4dec 100644 --- a/controller/linodevpc_controller.go +++ b/controller/linodevpc_controller.go @@ -24,6 +24,7 @@ import ( "time" corev1 "k8s.io/api/core/v1" + apierrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/runtime" utilerrors "k8s.io/apimachinery/pkg/util/errors" "k8s.io/client-go/tools/record" @@ -121,7 +122,7 @@ func (r *LinodeVPCReconciler) reconcile( } // Always close the scope when exiting this function so we can persist any LinodeMachine changes. - if patchErr := vpcScope.Close(ctx); patchErr != nil && utilerrors.FilterOut(patchErr) != nil { + if patchErr := vpcScope.Close(ctx); patchErr != nil && utilerrors.FilterOut(patchErr, apierrors.IsNotFound) != nil { logger.Error(patchErr, "failed to patch LinodeVPC") err = errors.Join(err, patchErr) diff --git a/e2e/Makefile b/e2e/Makefile index 83d77f710..a2cbf5a0c 100644 --- a/e2e/Makefile +++ b/e2e/Makefile @@ -1,5 +1,21 @@ -runTestCase: - @T="$$(KUBECONFIG="$$ROOT_DIR/kubeconfig" kuttl test --timeout 300 --skip-delete --namespace "$$NAMESPACE" "$$TC" 2>&1)" ;\ +ROOT_DIR ?= $(shell git rev-parse --show-toplevel) +BIN_DIR ?= $(ROOT_DIR)/bin +TARGET_API ?= api.linode.com +TARGET_API_VERSION ?= v4beta + +runThisTest: + @echo "PLease execute make -C ($ROOT_DIR)/Makefile _e2etest-infra to spin up infrastructure" + make --no-print-directory _runThisTest + +_runThisTest: + @D="$$(mktemp -d)" ;\ + cp $(ROOT_DIR)/e2e/Makefile $$D ;\ + mkdir $$D/suit ;\ + cp -r $(PWD) $$D/suit ;\ + ROOT_DIR=$(ROOT_DIR) KUBECONFIG="$(ROOT_DIR)/kubeconfig" $(BIN_DIR)/kuttl test --timeout 300 --skip-delete "$$D/suit" + +runTestSuit: + @T="$$(KUBECONFIG="$$ROOT_DIR/kubeconfig" $(BIN_DIR)/kuttl test --timeout 300 --skip-delete --namespace "$$NAMESPACE" "$$TS" 2>&1)" ;\ echo "$$T" |\ grep -v harness.go |\ grep -v ^=== |\ @@ -9,16 +25,28 @@ runTestCase: grep -v "kutt-test config testdirs is overridden" ;\ echo "$$T" | \ grep '^PASS' || \ - (echo "$$T" | grep harness.go && cat "$$TC/step/00-step.yaml" && exit 1) + (echo "$$T" | grep harness.go && find $$TS -name *.yaml -exec cat {} \; && exit 1) renderTestCase: @D="$$(mktemp -d)" ;\ - mkdir -p "$$D/step" ;\ - envsubst -i "$$TPL" -o "$$D/step/00-step.yaml" ;\ + mkdir -p "$$D/case" ;\ + $(BIN_DIR)/envsubst -i "$$TPL" -o "$$D/case/00-case.yaml" ;\ echo -n "$$D" +renderManifest: + @echo $(shell make --no-print-directory renderTestCase)/case/00-case.yaml + getKubeUid: @kubectl get -o jsonpath='{.metadata.uid}' -n "$$NAMESPACE" "$$OBJ" +patchKubeObj: + @kubectl patch --type=merge -p "$$PATCH" -n "$$NAMESPACE" "$$OBJ" + +patchKubeObjStatus: + @kubectl patch --type=merge --subresource status -p "$$PATCH" -n "$$NAMESPACE" "$$OBJ" + callLinodeApiGet: - @curl -s -H "Authorization: Bearer $$LINODE_TOKEN" -H "X-Filter: $$FILTER" "https://api.linode.com/v4beta/$$URI" + @curl -s -H "Authorization: Bearer $$LINODE_TOKEN" -H "X-Filter: $$FILTER" -H "Content-Type: application/json" "https://$(TARGET_API)/$(TARGET_API_VERSION)/$$URI" + +callLinodeApiPost: + @curl -s -H "Authorization: Bearer $$LINODE_TOKEN" -H "Content-Type: application/json" -d "$$BODY" -X POST "https://$(TARGET_API)/$(TARGET_API_VERSION)/$$URI" diff --git a/e2e/README.MD b/e2e/README.MD index e660dc72d..e86283df6 100644 --- a/e2e/README.MD +++ b/e2e/README.MD @@ -47,7 +47,7 @@ kind: TestStep commands: - script: |- MACHINE_UID="$(OBJ=machines/machine-sample make getKubeUid)" \ - TC="$(TPL="$PWD/02-create-linodemachine.tpl.yml" make renderTestCase)" make runTestCase + TS="$(TPL="$PWD/02-create-linodemachine.tpl.yml" make renderTestCase)" make runTestSuit ``` - You can run testsuits (which can contain template rendering). @@ -57,7 +57,7 @@ commands: apiVersion: kuttl.dev/v1beta1 kind: TestStep commands: - - script: TC="$PWD/testsuit" make runTestCase + - script: TS="$PWD/testsuit" make runTestSuit ``` - Test execution order is not guaranteed. If a test depends on another resource it is strongly suggested to create `assert` files to wait for an event. Almost every test depends on CAPI providers, so there is a good chance, you have to create a `00-assert.yaml` in your test folder. @@ -80,3 +80,10 @@ metadata: status: availableReplicas: 1 ``` + +## Executing individual test suit + +```bash +make _e2etest-infra # Only once per cluster! +(cd e2e/linodemachine-controller/minimal ; make runThisTest) +``` diff --git a/e2e/linodemachine-controller/byovpc/00-assert.yaml b/e2e/linodemachine-controller/byovpc/00-assert.yaml new file mode 100644 index 000000000..4107fd8f0 --- /dev/null +++ b/e2e/linodemachine-controller/byovpc/00-assert.yaml @@ -0,0 +1,15 @@ +apiVersion: apps/v1 +kind: Deployment +metadata: + name: capi-controller-manager + namespace: capi-system +status: + availableReplicas: 1 +--- +apiVersion: apps/v1 +kind: Deployment +metadata: + name: cluster-api-provider-linode-controller-manager + namespace: cluster-api-provider-linode-system +status: + availableReplicas: 1 diff --git a/e2e/linodemachine-controller/byovpc/01-assert.yaml b/e2e/linodemachine-controller/byovpc/01-assert.yaml new file mode 100644 index 000000000..04de738aa --- /dev/null +++ b/e2e/linodemachine-controller/byovpc/01-assert.yaml @@ -0,0 +1,8 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: LinodeVPC +metadata: + annotations: + cluster.x-k8s.io/paused: "true" + name: linodevpc-sample +status: + ready: true diff --git a/e2e/linodemachine-controller/byovpc/01-create-cluster.tpl.yml b/e2e/linodemachine-controller/byovpc/01-create-cluster.tpl.yml new file mode 100644 index 000000000..8d5767e13 --- /dev/null +++ b/e2e/linodemachine-controller/byovpc/01-create-cluster.tpl.yml @@ -0,0 +1,46 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: LinodeCluster +metadata: + annotations: + cluster.x-k8s.io/paused: "true" + name: linodecluster-sample +spec: + region: us-sea + vpcRef: + kind: LinodeVPC + name: linodevpc-sample + namespace: ${NAMESPACE} + uid: ${VPC_UID} +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Cluster +metadata: + annotations: + cluster.x-k8s.io/paused: "true" + name: cluster-sample +spec: + paused: true + infrastructureRef: + name: linodecluster-sample +--- +apiVersion: cluster.x-k8s.io/v1beta1 +kind: Machine +metadata: + annotations: + cluster.x-k8s.io/paused: "true" + name: machine-sample +spec: + clusterName: cluster-sample + bootstrap: + configRef: + apiVersion: v1 + kind: "ConfigMap" + name: "boostrap-sample" + dataSecretName: bootstrap-data-sample +--- +apiVersion: v1 +kind: Secret +metadata: + name: bootstrap-data-sample +data: + value: dG91Y2ggL29rCg== diff --git a/e2e/linodemachine-controller/byovpc/01-create-cluster.yaml b/e2e/linodemachine-controller/byovpc/01-create-cluster.yaml new file mode 100644 index 000000000..dc2cf4ccc --- /dev/null +++ b/e2e/linodemachine-controller/byovpc/01-create-cluster.yaml @@ -0,0 +1,11 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - command: make createVPC + - script: |- + VPC_ID=$(make fetchVPCID) \ + TS="$(TPL="$PWD/01-create-vpc.tpl.yml" make renderTestCase)" make runTestSuit + - command: make enableVPC + - script: |- + VPC_UID="$(OBJ=linodevpcs/linodevpc-sample make getKubeUid)" \ + TS="$(TPL="$PWD/01-create-cluster.tpl.yml" make renderTestCase)" make runTestSuit diff --git a/e2e/linodemachine-controller/byovpc/01-create-vpc.tpl.yml b/e2e/linodemachine-controller/byovpc/01-create-vpc.tpl.yml new file mode 100644 index 000000000..cce91d414 --- /dev/null +++ b/e2e/linodemachine-controller/byovpc/01-create-vpc.tpl.yml @@ -0,0 +1,10 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: LinodeVPC +metadata: + annotations: + cluster.x-k8s.io/paused: "true" + name: linodevpc-sample +spec: + label: capli-e2e-byovpc-sample + region: us-sea + vpcID: ${VPC_ID} diff --git a/e2e/linodemachine-controller/byovpc/01-vpc-create-options.json b/e2e/linodemachine-controller/byovpc/01-vpc-create-options.json new file mode 100644 index 000000000..61051291a --- /dev/null +++ b/e2e/linodemachine-controller/byovpc/01-vpc-create-options.json @@ -0,0 +1,10 @@ +{ + "label": "capli-e2e-byovpc-sample", + "region": "us-sea", + "subnets": [ + { + "ipv4": "10.0.0.0/24", + "label": "capli-e2e-byovpc-sample" + } + ] +} diff --git a/e2e/linodemachine-controller/byovpc/02-assert.yaml b/e2e/linodemachine-controller/byovpc/02-assert.yaml new file mode 100644 index 000000000..1cb149503 --- /dev/null +++ b/e2e/linodemachine-controller/byovpc/02-assert.yaml @@ -0,0 +1,10 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: LinodeMachine +metadata: + name: linodemachine-sample +spec: + region: us-sea + type: g5-nanode-1 +status: + ready: true + instanceState: running diff --git a/e2e/linodemachine-controller/byovpc/02-create-linodemachine.tpl.yml b/e2e/linodemachine-controller/byovpc/02-create-linodemachine.tpl.yml new file mode 100644 index 000000000..0a4b23b4e --- /dev/null +++ b/e2e/linodemachine-controller/byovpc/02-create-linodemachine.tpl.yml @@ -0,0 +1,12 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: LinodeMachine +metadata: + ownerReferences: + - apiVersion: cluster.x-k8s.io/v1beta1 + kind: Machine + name: machine-sample + uid: ${MACHINE_UID} + name: linodemachine-sample +spec: + region: us-sea + type: g5-nanode-1 diff --git a/e2e/linodemachine-controller/byovpc/02-create-linodemachine.yaml b/e2e/linodemachine-controller/byovpc/02-create-linodemachine.yaml new file mode 100644 index 000000000..ad15f30bb --- /dev/null +++ b/e2e/linodemachine-controller/byovpc/02-create-linodemachine.yaml @@ -0,0 +1,6 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + MACHINE_UID="$(OBJ=machines/machine-sample make getKubeUid)" \ + TS="$(TPL="$PWD/02-create-linodemachine.tpl.yml" make renderTestCase)" make runTestSuit diff --git a/e2e/linodemachine-controller/byovpc/03-delete-linodemachine.yaml b/e2e/linodemachine-controller/byovpc/03-delete-linodemachine.yaml new file mode 100644 index 000000000..3bf5e64ad --- /dev/null +++ b/e2e/linodemachine-controller/byovpc/03-delete-linodemachine.yaml @@ -0,0 +1,6 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +delete: +- apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 + kind: LinodeMachine + name: linodemachine-sample diff --git a/e2e/linodemachine-controller/byovpc/03-error.yaml b/e2e/linodemachine-controller/byovpc/03-error.yaml new file mode 100644 index 000000000..4ff1b308e --- /dev/null +++ b/e2e/linodemachine-controller/byovpc/03-error.yaml @@ -0,0 +1,4 @@ +apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1 +kind: LinodeMachine +metadata: + name: linodemachine-sample diff --git a/e2e/linodemachine-controller/byovpc/04-verify-linode-instance.yaml b/e2e/linodemachine-controller/byovpc/04-verify-linode-instance.yaml new file mode 100644 index 000000000..4515f8566 --- /dev/null +++ b/e2e/linodemachine-controller/byovpc/04-verify-linode-instance.yaml @@ -0,0 +1,5 @@ +apiVersion: kuttl.dev/v1beta1 +kind: TestStep +commands: + - script: |- + URI="linode/instances" FILTER="{\"tags\":\"$(OBJ=linodemachines/linodemachine-sample make getKubeUid)\"}" make callLinodeApiGet | grep 'results": 0' diff --git a/e2e/linodemachine-controller/byovpc/Makefile b/e2e/linodemachine-controller/byovpc/Makefile new file mode 100644 index 000000000..8c8d2f0e5 --- /dev/null +++ b/e2e/linodemachine-controller/byovpc/Makefile @@ -0,0 +1,11 @@ +include ../../Makefile + +createVPC: + @URI=vpcs FILTER='{"label":"capli-e2e-byovpc-sample"}' make --no-print-directory callLinodeApiGet | grep -v 'results": 0' ||\ + URI=vpcs BODY='@01-vpc-create-options.json' make --no-print-directory callLinodeApiPost + +enableVPC: + PATCH='{"status":{"ready":true}}' OBJ="linodevpcs/linodevpc-sample" make --no-print-directory patchKubeObjStatus + +fetchVPCID: + @URI=vpcs FILTER='{"label":"capli-e2e-byovpc-sample"}' make --no-print-directory callLinodeApiGet | jq -r .data[0].id diff --git a/e2e/linodemachine-controller/minimal/01-assert.yaml b/e2e/linodemachine-controller/minimal/01-assert.yaml deleted file mode 100644 index ca3986171..000000000 --- a/e2e/linodemachine-controller/minimal/01-assert.yaml +++ /dev/null @@ -1,17 +0,0 @@ -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Cluster -metadata: - annotations: - cluster.x-k8s.io/paused: "true" - name: cluster-sample -spec: - paused: true ---- -apiVersion: cluster.x-k8s.io/v1beta1 -kind: Machine -metadata: - annotations: - cluster.x-k8s.io/paused: "true" - name: machine-sample -spec: - clusterName: cluster-sample diff --git a/e2e/linodemachine-controller/minimal/01-create-cluster.yaml b/e2e/linodemachine-controller/minimal/01-create-cluster.yaml index c77670b2f..21ff6f163 100644 --- a/e2e/linodemachine-controller/minimal/01-create-cluster.yaml +++ b/e2e/linodemachine-controller/minimal/01-create-cluster.yaml @@ -4,6 +4,8 @@ metadata: annotations: cluster.x-k8s.io/paused: "true" name: linodecluster-sample +spec: + region: us-sea --- apiVersion: cluster.x-k8s.io/v1beta1 kind: Cluster diff --git a/e2e/linodemachine-controller/minimal/02-create-linodemachine.yaml b/e2e/linodemachine-controller/minimal/02-create-linodemachine.yaml index de40c5987..ad15f30bb 100644 --- a/e2e/linodemachine-controller/minimal/02-create-linodemachine.yaml +++ b/e2e/linodemachine-controller/minimal/02-create-linodemachine.yaml @@ -3,4 +3,4 @@ kind: TestStep commands: - script: |- MACHINE_UID="$(OBJ=machines/machine-sample make getKubeUid)" \ - TC="$(TPL="$PWD/02-create-linodemachine.tpl.yml" make renderTestCase)" make runTestCase + TS="$(TPL="$PWD/02-create-linodemachine.tpl.yml" make renderTestCase)" make runTestSuit