From 2f93bfc077843faf52de6274b7665a4f26b798a6 Mon Sep 17 00:00:00 2001
From: bcm820 <bmendoza@akamai.com>
Date: Thu, 22 Feb 2024 06:38:15 -0500
Subject: [PATCH] e2e: Replace Kuttl-in-Kuttl (#131)

---
 e2e/Makefile                                  | 22 ------
 e2e/README.MD                                 | 72 ++++++-------------
 .../byovpc/01-create-cluster.tpl.yml          | 48 -------------
 .../byovpc/01-create-cluster.yaml             | 66 +++++++++++++++--
 .../byovpc/01-create-vpc.tpl.yml              | 10 ---
 .../byovpc/02-create-linodemachine.tpl.yml    | 12 ----
 .../byovpc/02-create-linodemachine.yaml       | 16 ++++-
 .../minimal/02-create-linodemachine.tpl.yml   | 12 ----
 .../minimal/02-create-linodemachine.yaml      | 16 ++++-
 9 files changed, 111 insertions(+), 163 deletions(-)
 delete mode 100644 e2e/linodemachine-controller/byovpc/01-create-cluster.tpl.yml
 delete mode 100644 e2e/linodemachine-controller/byovpc/01-create-vpc.tpl.yml
 delete mode 100644 e2e/linodemachine-controller/byovpc/02-create-linodemachine.tpl.yml
 delete mode 100644 e2e/linodemachine-controller/minimal/02-create-linodemachine.tpl.yml

diff --git a/e2e/Makefile b/e2e/Makefile
index 40bc73aa1..f7237cd25 100644
--- a/e2e/Makefile
+++ b/e2e/Makefile
@@ -14,28 +14,6 @@ _runThisTest:
 	cp -r $(PWD) $$D/suit ;\
 	ROOT_DIR=$(ROOT_DIR) KUBECONFIG="$(ROOT_DIR)/kubeconfig" kubectl-kuttl test --timeout 300 --skip-delete "$$D/suit"
 
-runTestSuit:
-	@T="$$(KUBECONFIG="$$ROOT_DIR/kubeconfig" kubectl-kuttl test --timeout 300 --skip-delete --namespace "$$NAMESPACE" "$$TS" 2>&1)" ;\
-	echo "$$T" |\
-		grep -v harness.go |\
-		grep -v ^=== |\
-		grep -v "running without a 'kuttl-test.yaml" |\
-		grep -v "creation of user-supplied namespace" |\
-		grep -v "not match file name regexp" |\
-		grep -v "kutt-test config testdirs is overridden" ;\
-	echo "$$T" | \
-		grep '^PASS' || \
-			(echo "$$T" | grep harness.go && find $$TS -name *.yaml -exec cat {} \; && exit 1)
-
-renderTestCase:
-	@D="$$(mktemp -d)" ;\
-	mkdir -p "$$D/case" ;\
-	envsubst -i "$$TPL" -o "$$D/case/00-case.yaml" ;\
-	printf "$$D\n"
-
-renderManifest:
-	@echo $(shell make --no-print-directory renderTestCase)/case/00-case.yaml
-
 getKubeUid:
 	@kubectl get -o jsonpath='{.metadata.uid}' -n "$$NAMESPACE" "$$OBJ"
 
diff --git a/e2e/README.MD b/e2e/README.MD
index 078b94ab8..055a2c48e 100644
--- a/e2e/README.MD
+++ b/e2e/README.MD
@@ -1,68 +1,38 @@
 # E2E Framework
 
-The E2E framework uses Kuttl under the hood. Kuttl is a simple and most importantly static tool.
-That means there is no way to dynamically generate manifests.
-The solution is KIK (Kuttl-In-Kuttl)! 
+The E2E framework uses Kuttl under the hood. Kuttl is a simple and most importantly static tool that handles applying manifests for running assertions in a namespace it generates for each test.
 
-## Kuttl-In-Kuttl
+In order to dynamically generate manifests, Kuttl supports declaring a `TestStep` manifest which includes an option for running scripts. This allows us apply manifests with dynamic values from stdin.
 
-## How does it work
+Additionally, helper functions can be invoked in the scripts by creating a Makefile in each test folder that references `../../Makefile` (i.e. `e2e/Makefile`).
 
- - Instead of creating a static Kuttl manifest, you can create template. Template extension must be `.yml` otherwise Kuttl starts executing it.
- - Template is rendered via `envsubst`, so you can use Bash variables for dynamic data injection.
-  - Variables always present
-    - `${KUBECONFIG}`
-    - `${NAMESPACE}`
-    - `${ROOT_DIR}`
+## Example TestStep
 
-    Example template
-```yaml
-apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
-kind: LinodeMachine
-metadata:
-  ownerReferences:
-  - apiVersion: cluster.x-k8s.io/v1beta1
-    kind: Machine
-    name: machine-sample
-    uid: ${MACHINE_UID}
-  name: linodemachine-sample
-spec:
-  region: us-sea
-  type: g5-nanode-1
-```
-
- - Helper tools are available, please create a Makefile in your test folder. In this way you can use built in helpers without including any script in the test, plus this file is a good place for test specific helpers.
-
-    Example Makefile
-```makefile
-include ../../Makefile
-```
-
- - Template is nothing without a runner, please use helpers provided in Makefile to render and run template.
-
-    Example template runner
 ```yaml
 apiVersion: kuttl.dev/v1beta1
 kind: TestStep
 commands:
   - script: |-
-      MACHINE_UID="$(OBJ=machines/machine-sample make getKubeUid)" \
-      TS="$(TPL="$PWD/02-create-linodemachine.tpl.yml" make renderTestCase)" make runTestSuit
+      cat <<EOF | kubectl apply -n $NAMESPACE -f -
+      apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+      kind: LinodeMachine
+      metadata:
+        ownerReferences:
+        - apiVersion: cluster.x-k8s.io/v1beta1
+          kind: Machine
+          name: machine-sample
+          uid: $(OBJ=machines/machine-sample make getKubeUid)
+        name: linodemachine-sample
+      spec:
+        region: us-sea
+        type: g5-nanode-1
+      EOF
 ```
 
-  - You can run testsuits (which can contain template rendering).
-
-    Example testsuit runner:
-```yaml
-apiVersion: kuttl.dev/v1beta1
-kind: TestStep
-commands:
-  - script: TS="$PWD/testsuit" make runTestSuit
-```
+## Test Execution
 
-  - Test execution order is not guaranteed. If a test depends on another resource it is strongly suggested to create `assert` files to wait for an event. Almost every test depends on CAPI providers, so there is a good chance, you have to create a `00-assert.yaml` in your test folder.
+Note that test execution order is not guaranteed. If a test depends on another resource it is strongly suggested to create `assert` files to wait for an event. Almost every test depends on CAPI providers, so there is a good chance you have to create a `00-assert.yaml` in your test folder that asserts the following:
 
-    Example `00-assert.yaml`:
 ```yaml
 apiVersion: apps/v1
 kind: Deployment
@@ -81,7 +51,7 @@ status:
   availableReplicas: 1
 ```
 
-## Executing individual test suit
+## Executing individual tests
 
 ```bash
 make _e2etest-infra # Only once per cluster!
diff --git a/e2e/linodemachine-controller/byovpc/01-create-cluster.tpl.yml b/e2e/linodemachine-controller/byovpc/01-create-cluster.tpl.yml
deleted file mode 100644
index 2c96e8a3a..000000000
--- a/e2e/linodemachine-controller/byovpc/01-create-cluster.tpl.yml
+++ /dev/null
@@ -1,48 +0,0 @@
-apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
-kind: LinodeCluster
-metadata:
-  annotations:
-    cluster.x-k8s.io/paused: "true"
-  name: linodecluster-sample
-spec:
-  region: us-sea
-  vpcRef:
-    kind: LinodeVPC
-    name: linodevpc-sample
-    namespace: ${NAMESPACE}
-    uid: ${VPC_UID}
----
-apiVersion: cluster.x-k8s.io/v1beta1
-kind: Cluster
-metadata:
-  annotations:
-    cluster.x-k8s.io/paused: "true"
-  name: cluster-sample
-spec:
-  paused: true
-  infrastructureRef:
-    apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
-    kind: LinodeCluster
-    name: linodecluster-sample
----
-apiVersion: cluster.x-k8s.io/v1beta1
-kind: Machine
-metadata:
-  annotations:
-    cluster.x-k8s.io/paused: "true"
-  name: machine-sample
-spec:
-  clusterName: cluster-sample
-  bootstrap:
-    configRef:
-      apiVersion: v1
-      kind: "ConfigMap"
-      name: "boostrap-sample"
-    dataSecretName: bootstrap-data-sample
----
-apiVersion: v1
-kind: Secret
-metadata:
-  name: bootstrap-data-sample
-data:
-  value: dG91Y2ggL29rCg==
diff --git a/e2e/linodemachine-controller/byovpc/01-create-cluster.yaml b/e2e/linodemachine-controller/byovpc/01-create-cluster.yaml
index dc2cf4ccc..846a889e8 100644
--- a/e2e/linodemachine-controller/byovpc/01-create-cluster.yaml
+++ b/e2e/linodemachine-controller/byovpc/01-create-cluster.yaml
@@ -3,9 +3,67 @@ kind: TestStep
 commands:
   - command: make createVPC
   - script: |-
-      VPC_ID=$(make fetchVPCID) \
-      TS="$(TPL="$PWD/01-create-vpc.tpl.yml" make renderTestCase)" make runTestSuit
+      cat <<EOF | kubectl apply -n $NAMESPACE -f -
+      apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+      kind: LinodeVPC
+      metadata:
+        annotations:
+            cluster.x-k8s.io/paused: "true"
+        name: linodevpc-sample
+      spec:
+        label: capli-e2e-byovpc-for-all-tests
+        region: us-sea
+        vpcID: $(make fetchVPCID)
+      EOF
   - command: make enableVPC
   - script: |-
-      VPC_UID="$(OBJ=linodevpcs/linodevpc-sample make getKubeUid)" \
-      TS="$(TPL="$PWD/01-create-cluster.tpl.yml" make renderTestCase)" make runTestSuit
+      cat <<EOF | kubectl apply -n $NAMESPACE -f -
+      apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+      kind: LinodeCluster
+      metadata:
+        annotations:
+          cluster.x-k8s.io/paused: "true"
+        name: linodecluster-sample
+      spec:
+        region: us-sea
+        vpcRef:
+          kind: LinodeVPC
+          name: linodevpc-sample
+          namespace: $NAMESPACE
+          uid: $(OBJ=linodevpcs/linodevpc-sample make getKubeUid)
+      ---
+      apiVersion: cluster.x-k8s.io/v1beta1
+      kind: Cluster
+      metadata:
+        annotations:
+          cluster.x-k8s.io/paused: "true"
+        name: cluster-sample
+      spec:
+        paused: true
+        infrastructureRef:
+          apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+          kind: LinodeCluster
+          name: linodecluster-sample
+      ---
+      apiVersion: cluster.x-k8s.io/v1beta1
+      kind: Machine
+      metadata:
+        annotations:
+          cluster.x-k8s.io/paused: "true"
+        name: machine-sample
+      spec:
+        clusterName: cluster-sample
+        bootstrap:
+          configRef:
+            apiVersion: v1
+            kind: "ConfigMap"
+            name: "boostrap-sample"
+          dataSecretName: bootstrap-data-sample
+      ---
+      apiVersion: v1
+      kind: Secret
+      metadata:
+        name: bootstrap-data-sample
+      data:
+        value: dG91Y2ggL29rCg==
+      EOF
diff --git a/e2e/linodemachine-controller/byovpc/01-create-vpc.tpl.yml b/e2e/linodemachine-controller/byovpc/01-create-vpc.tpl.yml
deleted file mode 100644
index 6474b04b7..000000000
--- a/e2e/linodemachine-controller/byovpc/01-create-vpc.tpl.yml
+++ /dev/null
@@ -1,10 +0,0 @@
-apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
-kind: LinodeVPC
-metadata:
-  annotations:
-    cluster.x-k8s.io/paused: "true"
-  name: linodevpc-sample
-spec:
-  label: capli-e2e-byovpc-for-all-tests
-  region: us-sea
-  vpcID: ${VPC_ID}
diff --git a/e2e/linodemachine-controller/byovpc/02-create-linodemachine.tpl.yml b/e2e/linodemachine-controller/byovpc/02-create-linodemachine.tpl.yml
deleted file mode 100644
index 0a4b23b4e..000000000
--- a/e2e/linodemachine-controller/byovpc/02-create-linodemachine.tpl.yml
+++ /dev/null
@@ -1,12 +0,0 @@
-apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
-kind: LinodeMachine
-metadata:
-  ownerReferences:
-  - apiVersion: cluster.x-k8s.io/v1beta1
-    kind: Machine
-    name: machine-sample
-    uid: ${MACHINE_UID}
-  name: linodemachine-sample
-spec:
-  region: us-sea
-  type: g5-nanode-1
diff --git a/e2e/linodemachine-controller/byovpc/02-create-linodemachine.yaml b/e2e/linodemachine-controller/byovpc/02-create-linodemachine.yaml
index ad15f30bb..aeb131b4c 100644
--- a/e2e/linodemachine-controller/byovpc/02-create-linodemachine.yaml
+++ b/e2e/linodemachine-controller/byovpc/02-create-linodemachine.yaml
@@ -2,5 +2,17 @@ apiVersion: kuttl.dev/v1beta1
 kind: TestStep
 commands:
   - script: |-
-      MACHINE_UID="$(OBJ=machines/machine-sample make getKubeUid)" \
-      TS="$(TPL="$PWD/02-create-linodemachine.tpl.yml" make renderTestCase)" make runTestSuit
+      cat <<EOF | kubectl apply -n $NAMESPACE -f -
+      apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+      kind: LinodeMachine
+      metadata:
+        ownerReferences:
+        - apiVersion: cluster.x-k8s.io/v1beta1
+          kind: Machine
+          name: machine-sample
+          uid: $(OBJ=machines/machine-sample make getKubeUid)
+        name: linodemachine-sample
+      spec:
+        region: us-sea
+        type: g5-nanode-1
+      EOF
diff --git a/e2e/linodemachine-controller/minimal/02-create-linodemachine.tpl.yml b/e2e/linodemachine-controller/minimal/02-create-linodemachine.tpl.yml
deleted file mode 100644
index 0a4b23b4e..000000000
--- a/e2e/linodemachine-controller/minimal/02-create-linodemachine.tpl.yml
+++ /dev/null
@@ -1,12 +0,0 @@
-apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
-kind: LinodeMachine
-metadata:
-  ownerReferences:
-  - apiVersion: cluster.x-k8s.io/v1beta1
-    kind: Machine
-    name: machine-sample
-    uid: ${MACHINE_UID}
-  name: linodemachine-sample
-spec:
-  region: us-sea
-  type: g5-nanode-1
diff --git a/e2e/linodemachine-controller/minimal/02-create-linodemachine.yaml b/e2e/linodemachine-controller/minimal/02-create-linodemachine.yaml
index ad15f30bb..aeb131b4c 100644
--- a/e2e/linodemachine-controller/minimal/02-create-linodemachine.yaml
+++ b/e2e/linodemachine-controller/minimal/02-create-linodemachine.yaml
@@ -2,5 +2,17 @@ apiVersion: kuttl.dev/v1beta1
 kind: TestStep
 commands:
   - script: |-
-      MACHINE_UID="$(OBJ=machines/machine-sample make getKubeUid)" \
-      TS="$(TPL="$PWD/02-create-linodemachine.tpl.yml" make renderTestCase)" make runTestSuit
+      cat <<EOF | kubectl apply -n $NAMESPACE -f -
+      apiVersion: infrastructure.cluster.x-k8s.io/v1alpha1
+      kind: LinodeMachine
+      metadata:
+        ownerReferences:
+        - apiVersion: cluster.x-k8s.io/v1beta1
+          kind: Machine
+          name: machine-sample
+          uid: $(OBJ=machines/machine-sample make getKubeUid)
+        name: linodemachine-sample
+      spec:
+        region: us-sea
+        type: g5-nanode-1
+      EOF