e2e: Replace Kuttl-in-Kuttl

Makefile

@@ -93,12 +93,12 @@ test: manifests generate fmt vet envtest ## Run tests.
 e2etest:
 	make --no-print-directory _e2etest # Workaround to force the flag on Github Action
 
-_e2etest-infra: kind ctlptl tilt kuttl kustomize clusterctl envsubst
+_e2etest-infra: kind ctlptl tilt kuttl kustomize clusterctl
 	@echo -n "LINODE_TOKEN=$(LINODE_TOKEN)" > config/default/.env.linode
 	$(CTLPTL) apply -f .tilt/ctlptl-config.yaml
 	$(TILT) ci --timeout 240s -f Tiltfile
 
-_e2etest: manifests generate envsubst _e2etest-infra
+_e2etest: manifests generate _e2etest-infra
 	ROOT_DIR="$(PWD)" $(KUTTL) test --config e2e/kuttl-config.yaml
 
 ##@ Build
@@ -199,7 +199,6 @@ KIND ?= $(LOCALBIN)/kind
 KUTTL ?= $(LOCALBIN)/kubectl-kuttl
 # setup-envtest does not have devbox support so always use CACHE_BIN
 ENVTEST ?= $(CACHE_BIN)/setup-envtest
-ENVSUBST ?= $(LOCALBIN)/envsubst
 HUSKY ?= $(LOCALBIN)/husky
 NILAWAY ?= $(LOCALBIN)/nilaway
 GOVULNC ?= $(LOCALBIN)/govulncheck
@@ -212,13 +211,12 @@ CONTROLLER_TOOLS_VERSION ?= v0.14.0
 TILT_VERSION ?= 0.33.6
 KIND_VERSION ?= 0.20.0
 KUTTL_VERSION ?= 0.15.0
-ENVSUBST_VERSION ?= v1.4.2
 HUSKY_VERSION ?= v0.2.16
 NILAWAY_VERSION ?= latest
 GOVULNC_VERSION ?= v1.0.1
 
 .PHONY: tools
-tools: $(KUSTOMIZE) $(CTLPTL) $(CLUSTERCTL) $(CONTROLLER_GEN) $(TILT) $(KIND) $(KUTTL) $(ENVTEST) $(ENVSUBST) $(HUSKY) $(NILAWAY) $(GOVULNC)
+tools: $(KUSTOMIZE) $(CTLPTL) $(CLUSTERCTL) $(CONTROLLER_GEN) $(TILT) $(KIND) $(KUTTL) $(ENVTEST) $(HUSKY) $(NILAWAY) $(GOVULNC)
 
 .PHONY: kustomize
 kustomize: $(KUSTOMIZE) ## Download kustomize locally if necessary.
@@ -268,12 +266,6 @@ envtest: $(ENVTEST) ## Download setup-envtest locally if necessary.
 $(ENVTEST): $(LOCALBIN)
 	GOBIN=$(CACHE_BIN) go install sigs.k8s.io/controller-runtime/tools/setup-envtest@latest
 
-.PHONY: envsubst
-envsubst: $(ENVSUBST) ## Download envsubst locally if necessary.
-$(ENVSUBST): $(LOCALBIN)
-	curl -Lso $(ENVSUBST) https://github.com/a8m/envsubst/releases/download/$(ENVSUBST_VERSION)/envsubst-$(shell uname -s)-$(ARCH)
-	chmod +x $(ENVSUBST)
-
 .PHONY: husky
 husky: $(HUSKY) ## Download husky locally if necessary.
 	@echo Execute install command to enable git hooks: ./bin/husky install

devbox.json

@@ -13,8 +13,7 @@
     "kustomize@latest",
     "kuttl@latest",
     "nilaway@latest",
-    "tilt@latest",
-    "envsubst@latest"
+    "tilt@latest"
   ],
   "shell": {
     "init_hook": [],

e2e/Makefile

@@ -14,28 +14,6 @@ _runThisTest:
 	cp -r $(PWD) $$D/suit ;\
 	ROOT_DIR=$(ROOT_DIR) KUBECONFIG="$(ROOT_DIR)/kubeconfig" kubectl-kuttl test --timeout 300 --skip-delete "$$D/suit"
 
-runTestSuit:
-	@T="$$(KUBECONFIG="$$ROOT_DIR/kubeconfig" kubectl-kuttl test --timeout 300 --skip-delete --namespace "$$NAMESPACE" "$$TS" 2>&1)" ;\
-	echo "$$T" |\
-		grep -v harness.go |\
-		grep -v ^=== |\
-		grep -v "running without a 'kuttl-test.yaml" |\
-		grep -v "creation of user-supplied namespace" |\
-		grep -v "not match file name regexp" |\
-		grep -v "kutt-test config testdirs is overridden" ;\
-	echo "$$T" | \
-		grep '^PASS' || \
-			(echo "$$T" | grep harness.go && find $$TS -name *.yaml -exec cat {} \; && exit 1)
-
-renderTestCase:
-	@D="$$(mktemp -d)" ;\
-	mkdir -p "$$D/case" ;\
-	envsubst -i "$$TPL" -o "$$D/case/00-case.yaml" ;\
-	printf "$$D\n"
-
-renderManifest:
-	@echo $(shell make --no-print-directory renderTestCase)/case/00-case.yaml
-
 getKubeUid:
 	@kubectl get -o jsonpath='{.metadata.uid}' -n "$$NAMESPACE" "$$OBJ"
 

e2e/README.MD

@@ -1,68 +1,38 @@
 # E2E Framework
 
-The E2E framework uses Kuttl under the hood. Kuttl is a simple and most importantly static tool.
-That means there is no way to dynamically generate manifests.
-The solution is KIK (Kuttl-In-Kuttl)! 
+The E2E framework uses Kuttl under the hood. Kuttl is a simple and most importantly static tool that handles applying manifests for running assertions in a namespace it generates for each test.
 
-## Kuttl-In-Kuttl
+In order to dynamically generate manifests, Kuttl supports declaring a `TestStep` manifest which includes an option for running scripts. This allows us apply manifests with dynamic values from stdin.
 
-## How does it work
+Additionally, helper functions can be invoked in the scripts by creating a Makefile in each test folder that references `../../Makefile` (i.e. `e2e/Makefile`).
 
- - Instead of creating a static Kuttl manifest, you can create template. Template extension must be `.yml` otherwise Kuttl starts executing it.
- - Template is rendered via `envsubst`, so you can use Bash variables for dynamic data injection.
-  - Variables always present
-    - `${KUBECONFIG}`
-    - `${NAMESPACE}`
-    - `${ROOT_DIR}`
+## Example TestStep
 
-    Example template
-```yaml
-apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
-kind: LinodeMachine
-metadata:
-  ownerReferences:
-  - apiVersion: cluster.x-k8s.io/v1beta1
-    kind: Machine
-    name: machine-sample
-    uid: ${MACHINE_UID}
-  name: linodemachine-sample
-spec:
-  region: us-sea
-  type: g5-nanode-1
-```
-
- - Helper tools are available, please create a Makefile in your test folder. In this way you can use built in helpers without including any script in the test, plus this file is a good place for test specific helpers.
-
-    Example Makefile
-```makefile
-include ../../Makefile
-```
-
- - Template is nothing without a runner, please use helpers provided in Makefile to render and run template.
-
-    Example template runner
 ```yaml
 apiVersion: kuttl.dev/v1beta1
 kind: TestStep
 commands:
   - script: |-
-      MACHINE_UID="$(OBJ=machines/machine-sample make getKubeUid)" \
-      TS="$(TPL="$PWD/02-create-linodemachine.tpl.yml" make renderTestCase)" make runTestSuit
+      cat <<EOF | kubectl apply -n $NAMESPACE -f -
+      apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+      kind: LinodeMachine
+      metadata:
+        ownerReferences:
+        - apiVersion: cluster.x-k8s.io/v1beta1
+          kind: Machine
+          name: machine-sample
+          uid: $(OBJ=machines/machine-sample make getKubeUid)
+        name: linodemachine-sample
+      spec:
+        region: us-sea
+        type: g5-nanode-1
+      EOF
 ```
 
-  - You can run testsuits (which can contain template rendering).
-
-    Example testsuit runner:
-```yaml
-apiVersion: kuttl.dev/v1beta1
-kind: TestStep
-commands:
-  - script: TS="$PWD/testsuit" make runTestSuit
-```
+## Test Execution
 
-  - Test execution order is not guaranteed. If a test depends on another resource it is strongly suggested to create `assert` files to wait for an event. Almost every test depends on CAPI providers, so there is a good chance, you have to create a `00-assert.yaml` in your test folder.
+Note that test execution order is not guaranteed. If a test depends on another resource it is strongly suggested to create `assert` files to wait for an event. Almost every test depends on CAPI providers, so there is a good chance you have to create a `00-assert.yaml` in your test folder that asserts the following:
 
-    Example `00-assert.yaml`:
 ```yaml
 apiVersion: apps/v1
 kind: Deployment
@@ -81,7 +51,7 @@ status:
   availableReplicas: 1
 ```
 
-## Executing individual test suit
+## Executing individual tests
 
 ```bash
 make _e2etest-infra # Only once per cluster!

e2e/linodemachine-controller/byovpc/01-create-cluster.yaml

@@ -3,9 +3,65 @@ kind: TestStep
 commands:
   - command: make createVPC
   - script: |-
-      VPC_ID=$(make fetchVPCID) \
-      TS="$(TPL="$PWD/01-create-vpc.tpl.yml" make renderTestCase)" make runTestSuit
+      cat <<EOF | kubectl apply -n $NAMESPACE -f -
+      apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+      kind: LinodeVPC
+      metadata:
+        annotations:
+            cluster.x-k8s.io/paused: "true"
+        name: linodevpc-sample
+      spec:
+        label: capli-e2e-byovpc-for-all-tests
+        region: us-sea
+        vpcID: $(make fetchVPCID)
+      EOF
   - command: make enableVPC
   - script: |-
-      VPC_UID="$(OBJ=linodevpcs/linodevpc-sample make getKubeUid)" \
-      TS="$(TPL="$PWD/01-create-cluster.tpl.yml" make renderTestCase)" make runTestSuit
+      cat <<EOF | kubectl apply -n $NAMESPACE -f -
+      apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+      kind: LinodeCluster
+      metadata:
+        annotations:
+          cluster.x-k8s.io/paused: "true"
+        name: linodecluster-sample
+      spec:
+        region: us-sea
+        vpcRef:
+          kind: LinodeVPC
+          name: linodevpc-sample
+          namespace: $NAMESPACE
+          uid: $(OBJ=linodevpcs/linodevpc-sample make getKubeUid)
+      ---
+      apiVersion: cluster.x-k8s.io/v1beta1
+      kind: Cluster
+      metadata:
+        annotations:
+          cluster.x-k8s.io/paused: "true"
+        name: cluster-sample
+      spec:
+        paused: true
+        infrastructureRef:
+          name: linodecluster-sample
+      ---
+      apiVersion: cluster.x-k8s.io/v1beta1
+      kind: Machine
+      metadata:
+        annotations:
+          cluster.x-k8s.io/paused: "true"
+        name: machine-sample
+      spec:
+        clusterName: cluster-sample
+        bootstrap:
+          configRef:
+            apiVersion: v1
+            kind: "ConfigMap"
+            name: "boostrap-sample"
+          dataSecretName: bootstrap-data-sample
+      ---
+      apiVersion: v1
+      kind: Secret
+      metadata:
+        name: bootstrap-data-sample
+      data:
+        value: dG91Y2ggL29rCg==
+      EOF

e2e/linodemachine-controller/byovpc/02-create-linodemachine.yaml

@@ -2,5 +2,17 @@ apiVersion: kuttl.dev/v1beta1
 kind: TestStep
 commands:
   - script: |-
-      MACHINE_UID="$(OBJ=machines/machine-sample make getKubeUid)" \
-      TS="$(TPL="$PWD/02-create-linodemachine.tpl.yml" make renderTestCase)" make runTestSuit
+      cat <<EOF | kubectl apply -n $NAMESPACE -f -
+      apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+      kind: LinodeMachine
+      metadata:
+        ownerReferences:
+        - apiVersion: cluster.x-k8s.io/v1beta1
+          kind: Machine
+          name: machine-sample
+          uid: $(OBJ=machines/machine-sample make getKubeUid)
+        name: linodemachine-sample
+      spec:
+        region: us-sea
+        type: g5-nanode-1
+      EOF

e2e/linodemachine-controller/minimal/02-create-linodemachine.yaml

@@ -2,5 +2,17 @@ apiVersion: kuttl.dev/v1beta1
 kind: TestStep
 commands:
   - script: |-
-      MACHINE_UID="$(OBJ=machines/machine-sample make getKubeUid)" \
-      TS="$(TPL="$PWD/02-create-linodemachine.tpl.yml" make renderTestCase)" make runTestSuit
+      cat <<EOF | kubectl apply -n $NAMESPACE -f -
+      apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+      kind: LinodeMachine
+      metadata:
+        ownerReferences:
+        - apiVersion: cluster.x-k8s.io/v1beta1
+          kind: Machine
+          name: machine-sample
+          uid: $(OBJ=machines/machine-sample make getKubeUid)
+        name: linodemachine-sample
+      spec:
+        region: us-sea
+        type: g5-nanode-1
+      EOF