Machine e2e test with bring your own VPC (#83)

Co-authored-by: Richard Kovacs <[email protected]>

Makefile

@@ -92,10 +92,12 @@ test: manifests generate fmt vet envtest ## Run tests.
 e2etest:
 	make --no-print-directory _e2etest # Workaround to force the flag on Github Action
 
-_e2etest: kind ctlptl tilt kuttl kustomize clusterctl envsubst manifests generate
+_e2etest-infra: kind ctlptl tilt kuttl kustomize clusterctl
 	@echo -n "LINODE_TOKEN=$(LINODE_TOKEN)" > config/default/.env.linode
 	$(CTLPTL) apply -f .tilt/ctlptl-config.yaml
 	$(TILT) ci --timeout 240s -f Tiltfile
+
+_e2etest: manifests generate envsubst _e2etest-infra
 	ROOT_DIR="$(PWD)" $(KUTTL) test --config e2e/kuttl-config.yaml
 
 ##@ Build

controller/linodemachine_controller.go

@@ -29,6 +29,7 @@ import (
 	"github.com/linode/cluster-api-provider-linode/util/reconciler"
 	"github.com/linode/linodego"
 	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/client-go/tools/record"
@@ -233,7 +234,7 @@ func (r *LinodeMachineReconciler) reconcile(
 		}
 
 		// Always close the scope when exiting this function so we can persist any LinodeMachine changes.
-		if patchErr := machineScope.Close(ctx); patchErr != nil && utilerrors.FilterOut(patchErr) != nil {
+		if patchErr := machineScope.Close(ctx); patchErr != nil && utilerrors.FilterOut(patchErr, apierrors.IsNotFound) != nil {
 			logger.Error(patchErr, "failed to patch LinodeMachine")
 
 			err = errors.Join(err, patchErr)
@@ -324,7 +325,7 @@ func (r *LinodeMachineReconciler) reconcileCreate(
 		if machineScope.LinodeCluster.Spec.VPCRef != nil {
 			iface, err := r.getVPCInterfaceConfig(ctx, machineScope, createConfig.Interfaces, logger)
 			if err != nil {
-				logger.Error(err, "Failed to get VPC interface confiog")
+				logger.Error(err, "Failed to get VPC interface config")
 
 				return nil, err
 			}

controller/linodemachine_controller_helpers.go

@@ -173,9 +173,14 @@ func (r *LinodeMachineReconciler) requestsForCluster(ctx context.Context, namesp
 			continue
 		}
 
+		infraNs := item.Spec.InfrastructureRef.Namespace
+		if infraNs == "" {
+			infraNs = item.Namespace
+		}
+
 		result = append(result, ctrl.Request{
 			NamespacedName: client.ObjectKey{
-				Namespace: item.Namespace,
+				Namespace: infraNs,
 				Name:      item.Spec.InfrastructureRef.Name,
 			},
 		})
@@ -187,6 +192,9 @@ func (r *LinodeMachineReconciler) requestsForCluster(ctx context.Context, namesp
 func (r *LinodeMachineReconciler) getVPCInterfaceConfig(ctx context.Context, machineScope *scope.MachineScope, existingIfaces []linodego.InstanceConfigInterfaceCreateOptions, logger logr.Logger) (*linodego.InstanceConfigInterfaceCreateOptions, error) {
 	name := machineScope.LinodeCluster.Spec.VPCRef.Name
 	namespace := machineScope.LinodeCluster.Spec.VPCRef.Namespace
+	if namespace == "" {
+		namespace = machineScope.LinodeCluster.Namespace
+	}
 
 	logger = logger.WithValues("vpcName", name, "vpcNamespace", namespace)
 

controller/linodevpc_controller.go

@@ -24,6 +24,7 @@ import (
 	"time"
 
 	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilerrors "k8s.io/apimachinery/pkg/util/errors"
 	"k8s.io/client-go/tools/record"
@@ -121,7 +122,7 @@ func (r *LinodeVPCReconciler) reconcile(
 		}
 
 		// Always close the scope when exiting this function so we can persist any LinodeMachine changes.
-		if patchErr := vpcScope.Close(ctx); patchErr != nil && utilerrors.FilterOut(patchErr) != nil {
+		if patchErr := vpcScope.Close(ctx); patchErr != nil && utilerrors.FilterOut(patchErr, apierrors.IsNotFound) != nil {
 			logger.Error(patchErr, "failed to patch LinodeVPC")
 
 			err = errors.Join(err, patchErr)

e2e/Makefile

@@ -1,5 +1,21 @@
-runTestCase:
-	@T="$$(KUBECONFIG="$$ROOT_DIR/kubeconfig" kuttl test --timeout 300 --skip-delete --namespace "$$NAMESPACE" "$$TC" 2>&1)" ;\
+ROOT_DIR ?= $(shell git rev-parse --show-toplevel)
+BIN_DIR ?= $(ROOT_DIR)/bin
+TARGET_API ?= api.linode.com
+TARGET_API_VERSION ?= v4beta
+
+runThisTest:
+	@echo "PLease execute make -C ($ROOT_DIR)/Makefile _e2etest-infra to spin up infrastructure"
+	make --no-print-directory _runThisTest
+
+_runThisTest:
+	@D="$$(mktemp -d)" ;\
+	cp $(ROOT_DIR)/e2e/Makefile $$D ;\
+	mkdir $$D/suit ;\
+	cp -r $(PWD) $$D/suit ;\
+	ROOT_DIR=$(ROOT_DIR) KUBECONFIG="$(ROOT_DIR)/kubeconfig" $(BIN_DIR)/kuttl test --timeout 300 --skip-delete "$$D/suit"
+
+runTestSuit:
+	@T="$$(KUBECONFIG="$$ROOT_DIR/kubeconfig" $(BIN_DIR)/kuttl test --timeout 300 --skip-delete --namespace "$$NAMESPACE" "$$TS" 2>&1)" ;\
 	echo "$$T" |\
 		grep -v harness.go |\
 		grep -v ^=== |\
@@ -9,16 +25,28 @@ runTestCase:
 		grep -v "kutt-test config testdirs is overridden" ;\
 	echo "$$T" | \
 		grep '^PASS' || \
-			(echo "$$T" | grep harness.go && cat "$$TC/step/00-step.yaml" && exit 1)
+			(echo "$$T" | grep harness.go && find $$TS -name *.yaml -exec cat {} \; && exit 1)
 
 renderTestCase:
 	@D="$$(mktemp -d)" ;\
-	mkdir -p "$$D/step" ;\
-	envsubst -i "$$TPL" -o "$$D/step/00-step.yaml" ;\
+	mkdir -p "$$D/case" ;\
+	$(BIN_DIR)/envsubst -i "$$TPL" -o "$$D/case/00-case.yaml" ;\
 	echo -n "$$D"
 
+renderManifest:
+	@echo $(shell make --no-print-directory renderTestCase)/case/00-case.yaml
+
 getKubeUid:
 	@kubectl get -o jsonpath='{.metadata.uid}' -n "$$NAMESPACE" "$$OBJ"
 
+patchKubeObj:
+	@kubectl patch --type=merge -p "$$PATCH" -n "$$NAMESPACE" "$$OBJ"
+
+patchKubeObjStatus:
+	@kubectl patch --type=merge --subresource status -p "$$PATCH" -n "$$NAMESPACE" "$$OBJ"
+
 callLinodeApiGet:
-	@curl -s -H "Authorization: Bearer $$LINODE_TOKEN" -H "X-Filter: $$FILTER" "https://api.linode.com/v4beta/$$URI"
+	@curl -s -H "Authorization: Bearer $$LINODE_TOKEN" -H "X-Filter: $$FILTER" -H "Content-Type: application/json" "https://$(TARGET_API)/$(TARGET_API_VERSION)/$$URI"
+
+callLinodeApiPost:
+	@curl -s -H "Authorization: Bearer $$LINODE_TOKEN" -H "Content-Type: application/json" -d "$$BODY" -X POST "https://$(TARGET_API)/$(TARGET_API_VERSION)/$$URI"

e2e/README.MD

@@ -47,7 +47,7 @@ kind: TestStep
 commands:
   - script: |-
       MACHINE_UID="$(OBJ=machines/machine-sample make getKubeUid)" \
-      TC="$(TPL="$PWD/02-create-linodemachine.tpl.yml" make renderTestCase)" make runTestCase
+      TS="$(TPL="$PWD/02-create-linodemachine.tpl.yml" make renderTestCase)" make runTestSuit
 ```
 
   - You can run testsuits (which can contain template rendering).
@@ -57,7 +57,7 @@ commands:
 apiVersion: kuttl.dev/v1beta1
 kind: TestStep
 commands:
-  - script: TC="$PWD/testsuit" make runTestCase
+  - script: TS="$PWD/testsuit" make runTestSuit
 ```
 
   - Test execution order is not guaranteed. If a test depends on another resource it is strongly suggested to create `assert` files to wait for an event. Almost every test depends on CAPI providers, so there is a good chance, you have to create a `00-assert.yaml` in your test folder.
@@ -80,3 +80,10 @@ metadata:
 status:
   availableReplicas: 1
 ```
+
+## Executing individual test suit
+
+```bash
+make _e2etest-infra # Only once per cluster!
+(cd e2e/linodemachine-controller/minimal ; make runThisTest)
+``` 

e2e/linodemachine-controller/byovpc/00-assert.yaml

@@ -0,0 +1,15 @@
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: capi-controller-manager
+  namespace: capi-system
+status:
+  availableReplicas: 1
+---
+apiVersion: apps/v1
+kind: Deployment
+metadata:
+  name: cluster-api-provider-linode-controller-manager
+  namespace: cluster-api-provider-linode-system
+status:
+  availableReplicas: 1

e2e/linodemachine-controller/byovpc/01-assert.yaml

@@ -0,0 +1,8 @@
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+kind: LinodeVPC
+metadata:
+  annotations:
+    cluster.x-k8s.io/paused: "true"
+  name: linodevpc-sample
+status:
+  ready: true

e2e/linodemachine-controller/byovpc/01-create-cluster.tpl.yml

@@ -0,0 +1,46 @@
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+kind: LinodeCluster
+metadata:
+  annotations:
+    cluster.x-k8s.io/paused: "true"
+  name: linodecluster-sample
+spec:
+  region: us-sea
+  vpcRef:
+    kind: LinodeVPC
+    name: linodevpc-sample
+    namespace: ${NAMESPACE}
+    uid: ${VPC_UID}
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Cluster
+metadata:
+  annotations:
+    cluster.x-k8s.io/paused: "true"
+  name: cluster-sample
+spec:
+  paused: true
+  infrastructureRef:
+    name: linodecluster-sample
+---
+apiVersion: cluster.x-k8s.io/v1beta1
+kind: Machine
+metadata:
+  annotations:
+    cluster.x-k8s.io/paused: "true"
+  name: machine-sample
+spec:
+  clusterName: cluster-sample
+  bootstrap:
+    configRef:
+      apiVersion: v1
+      kind: "ConfigMap"
+      name: "boostrap-sample"
+    dataSecretName: bootstrap-data-sample
+---
+apiVersion: v1
+kind: Secret
+metadata:
+  name: bootstrap-data-sample
+data:
+  value: dG91Y2ggL29rCg==

e2e/linodemachine-controller/byovpc/01-create-cluster.yaml

@@ -0,0 +1,11 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - command: make createVPC
+  - script: |-
+      VPC_ID=$(make fetchVPCID) \
+      TS="$(TPL="$PWD/01-create-vpc.tpl.yml" make renderTestCase)" make runTestSuit
+  - command: make enableVPC
+  - script: |-
+      VPC_UID="$(OBJ=linodevpcs/linodevpc-sample make getKubeUid)" \
+      TS="$(TPL="$PWD/01-create-cluster.tpl.yml" make renderTestCase)" make runTestSuit

e2e/linodemachine-controller/byovpc/01-create-vpc.tpl.yml

@@ -0,0 +1,10 @@
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+kind: LinodeVPC
+metadata:
+  annotations:
+    cluster.x-k8s.io/paused: "true"
+  name: linodevpc-sample
+spec:
+  label: capli-e2e-byovpc-sample
+  region: us-sea
+  vpcID: ${VPC_ID}

e2e/linodemachine-controller/byovpc/01-vpc-create-options.json

@@ -0,0 +1,10 @@
+{
+    "label": "capli-e2e-byovpc-sample",
+    "region": "us-sea",
+    "subnets": [
+        {
+            "ipv4": "10.0.0.0/24",
+            "label": "capli-e2e-byovpc-sample"
+        }
+    ]
+}

e2e/linodemachine-controller/byovpc/02-assert.yaml

@@ -0,0 +1,10 @@
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+kind: LinodeMachine
+metadata:
+  name: linodemachine-sample
+spec:
+  region: us-sea
+  type: g5-nanode-1
+status:
+  ready: true
+  instanceState: running

e2e/linodemachine-controller/byovpc/02-create-linodemachine.tpl.yml

@@ -0,0 +1,12 @@
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+kind: LinodeMachine
+metadata:
+  ownerReferences:
+  - apiVersion: cluster.x-k8s.io/v1beta1
+    kind: Machine
+    name: machine-sample
+    uid: ${MACHINE_UID}
+  name: linodemachine-sample
+spec:
+  region: us-sea
+  type: g5-nanode-1

e2e/linodemachine-controller/byovpc/02-create-linodemachine.yaml

@@ -0,0 +1,6 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |-
+      MACHINE_UID="$(OBJ=machines/machine-sample make getKubeUid)" \
+      TS="$(TPL="$PWD/02-create-linodemachine.tpl.yml" make renderTestCase)" make runTestSuit

e2e/linodemachine-controller/byovpc/03-delete-linodemachine.yaml

@@ -0,0 +1,6 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+delete:
+- apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+  kind: LinodeMachine
+  name: linodemachine-sample

e2e/linodemachine-controller/byovpc/03-error.yaml

@@ -0,0 +1,4 @@
+apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+kind: LinodeMachine
+metadata:
+  name: linodemachine-sample

e2e/linodemachine-controller/byovpc/04-verify-linode-instance.yaml

@@ -0,0 +1,5 @@
+apiVersion: kuttl.dev/v1beta1
+kind: TestStep
+commands:
+  - script: |-
+      URI="linode/instances" FILTER="{\"tags\":\"$(OBJ=linodemachines/linodemachine-sample make getKubeUid)\"}" make callLinodeApiGet | grep 'results": 0'

e2e/linodemachine-controller/byovpc/Makefile

@@ -0,0 +1,11 @@
+include ../../Makefile
+
+createVPC:
+	@URI=vpcs FILTER='{"label":"capli-e2e-byovpc-sample"}' make --no-print-directory callLinodeApiGet | grep -v 'results": 0' ||\
+	URI=vpcs BODY='@01-vpc-create-options.json' make --no-print-directory callLinodeApiPost
+
+enableVPC:
+	PATCH='{"status":{"ready":true}}' OBJ="linodevpcs/linodevpc-sample" make --no-print-directory patchKubeObjStatus
+
+fetchVPCID:
+	@URI=vpcs FILTER='{"label":"capli-e2e-byovpc-sample"}' make --no-print-directory callLinodeApiGet | jq -r .data[0].id

e2e/linodemachine-controller/minimal/01-create-cluster.yaml

@@ -4,6 +4,8 @@ metadata:
   annotations:
     cluster.x-k8s.io/paused: "true"
   name: linodecluster-sample
+spec:
+  region: us-sea
 ---
 apiVersion: cluster.x-k8s.io/v1beta1
 kind: Cluster

e2e/linodemachine-controller/minimal/02-create-linodemachine.yaml

@@ -3,4 +3,4 @@ kind: TestStep
 commands:
   - script: |-
       MACHINE_UID="$(OBJ=machines/machine-sample make getKubeUid)" \
-      TC="$(TPL="$PWD/02-create-linodemachine.tpl.yml" make renderTestCase)" make runTestCase
+      TS="$(TPL="$PWD/02-create-linodemachine.tpl.yml" make renderTestCase)" make runTestSuit