Add automated testing of AWS templates, rework existing e2e testing to use kubeclient

.github/workflows/build.yml

@@ -0,0 +1,52 @@
+name: Build and Unit Test
+
+concurrency:
+  group: test-${{ github.head_ref || github.run_id }}
+  cancel-in-progress: true
+
+on:
+  push:
+    branches:
+      - main
+      - release-*
+    tags:
+      - '*'
+    paths-ignore:
+      - '**.md'
+  pull_request:
+    branches:
+      - main
+      - release-*
+    paths-ignore:
+      - 'config/**'
+      - '**.md'
+
+env:
+  GO_VERSION: '1.22'
+
+jobs:
+  build:
+    name: Build and Unit Test
+    runs-on: ubuntu-latest
+    steps:
+      - name: Checkout repository
+        uses: actions/checkout@v4
+      - name: Setup Go
+        uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION }}
+      - name: Lint
+        uses: golangci/golangci-lint-action@v6
+        with:
+          args: --timeout 10m0s
+      - name: Verify all generated pieces are up-to-date
+        run: make generate-all && git add -N . && git diff --exit-code
+      - name: Unit tests
+        run: |
+          make test
+      - name: Build
+        run: |
+          make build
+      - name: Image build
+        run: |
+          make docker-build

.github/workflows/test.yml

@@ -1,54 +1,45 @@
-name: Build and Test
+name: E2E Tests
 
 concurrency:
   group: test-${{ github.head_ref || github.run_id }}
   cancel-in-progress: true
 
 on:
-  push:
+  pull_request_target:
+    types: [labeled]
     branches:
       - main
       - release-*
-    tags:
-      - "*"
-    paths-ignore:
-      - '**.md'
-  pull_request:
-    branches:
-      - main
-      - release-*
-    tags:
-      - "*"
     paths-ignore:
       - 'config/**'
       - '**.md'
-
 env:
   GO_VERSION: '1.22'
 
 jobs:
-  build:
-    name: Build
+  e2etest:
+    name: E2E Tests
     runs-on: ubuntu-latest
+    if: contains(github.event.pull_request.labels.*.name, 'test-e2e')
+    env:
+      AWS_REGION: us-west-2
+      AWS_ACCESS_KEY_ID: ${{ secrets.CI_AWS_ACCESS_KEY_ID }}
+      AWS_SECRET_ACCESS_KEY: ${{ secrets.CI_AWS_SECRET_ACCESS_KEY }}
     steps:
       - name: Checkout repository
         uses: actions/checkout@v4
-      - name: Set up Go
+      - name: Setup Go
         uses: actions/setup-go@v5
         with:
-          go-version: ${{ env.GO_VERSION }}
-      - name: Lint
-        uses: golangci/golangci-lint-action@v6
-        with:
-          args: --timeout 10m0s
-      - name: Verify all generated pieces are up-to-date
-        run: make generate-all && git add -N . && git diff --exit-code
-      - name: Unit tests
+        go-version: ${{ env.GO_VERSION }}
+      - name: Setup kubectl
+        uses: azure/setup-kubectl@v4
+      - name: Run E2E tests
         run: |
-          make test
-      - name: Build
-        run: | 
-          make build
-      - name: Image build
-        run: |
-          make docker-build
+          make test-e2e
+      - name: Archive test results
+        uses: actions/upload-artifact@v4
+        with:
+        name: test-logs
+        path: |
+          test/e2e/*.log

.gitignore

@@ -14,7 +14,13 @@ dist
 go.work
 go.work.sum
 
-# editors
+# cloud-nuke config
+*cloud_nuke.yaml
+
+# Test artifacts
+test/e2e/*.log
+
+# ditors
 .idea
 *.swp
 *.swo

.golangci.yml

@@ -9,10 +9,10 @@ issues:
   # restore some of the defaults
   # (fill in the rest as needed)
   exclude-rules:
-    - path: "api/*"
+    - path: 'api/*'
       linters:
         - lll
-    - path: "internal/*"
+    - path: 'internal/*'
       linters:
         - dupl
         - lll
@@ -21,7 +21,7 @@ linters:
   enable:
     - dupl
     - errcheck
-    - exportloopref
+    - copyloopvar
     - goconst
     - gocyclo
     - gofmt

Makefile

@@ -104,9 +104,9 @@ test: generate-all fmt vet envtest tidy external-crd ## Run tests.
 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out
 
 # Utilize Kind or modify the e2e tests to load the image locally, enabling compatibility with other vendors.
-.PHONY: test-e2e  # Run the e2e tests against a Kind k8s instance that is spun up.
-test-e2e:
-	go test ./test/e2e/ -v -ginkgo.v
+.PHONY: test-e2e # Run the e2e tests against a Kind k8s instance that is spun up.
+test-e2e: cli-install
+	KIND_CLUSTER_NAME="hmc-test" KIND_VERSION=$(KIND_VERSION) go test ./test/e2e/ -v -ginkgo.v -timeout=2h
 
 .PHONY: lint
 lint: golangci-lint ## Run golangci-lint linter & yamllint
@@ -192,6 +192,8 @@ REGISTRY_NAME ?= hmc-local-registry
 REGISTRY_PORT ?= 5001
 REGISTRY_REPO ?= oci://127.0.0.1:$(REGISTRY_PORT)/charts
 DEV_PROVIDER ?= aws
+REGISTRY_IS_OCI = $(shell echo $(REGISTRY_REPO) | grep -q oci && echo true || echo false)
+CLUSTER_NAME ?= $(shell $(YQ) '.metadata.name' ./config/dev/deployment.yaml)
 
 AWS_CREDENTIALS=${AWS_B64ENCODED_CREDENTIALS}
 
@@ -243,17 +245,31 @@ dev-undeploy: ## Undeploy controller from the K8s cluster specified in ~/.kube/c
 
 .PHONY: helm-push
 helm-push: helm-package
-	@for chart in $(CHARTS_PACKAGE_DIR)/*.tgz; do \
+	@if [ ! $(REGISTRY_IS_OCI) ]; then \
+	    repo_flag="--repo"; \
+	fi; \
+	for chart in $(CHARTS_PACKAGE_DIR)/*.tgz; do \
 		base=$$(basename $$chart .tgz); \
 		chart_version=$$(echo $$base | grep -o "v\{0,1\}[0-9]\+\.[0-9]\+\.[0-9].*"); \
 		chart_name="$${base%-"$$chart_version"}"; \
 		echo "Verifying if chart $$chart_name, version $$chart_version already exists in $(REGISTRY_REPO)"; \
-		chart_exists=$$($(HELM) pull $(REGISTRY_REPO)/$$chart_name --version $$chart_version --destination /tmp 2>&1 | grep "not found" || true); \
+		chart_exists=$$($(HELM) pull $$repo_flag $(REGISTRY_REPO) $$chart_name --version $$chart_version --destination /tmp 2>&1 | grep "not found" || true); \
 		if [ -z "$$chart_exists" ]; then \
 			echo "Chart $$chart_name version $$chart_version already exists in the repository."; \
 		else \
-			echo "Pushing $$chart to $(REGISTRY_REPO)"; \
-			$(HELM) push "$$chart" $(REGISTRY_REPO); \
+			if $(REGISTRY_IS_OCI); then \
+				echo "Pushing $$chart to $(REGISTRY_REPO)"; \
+				$(HELM) push "$$chart" $(REGISTRY_REPO); \
+			else \
+				if [ ! $$REGISTRY_USERNAME ] && [ ! $$REGISTRY_PASSWORD ]; then \
+					echo "REGISTRY_USERNAME and REGISTRY_PASSWORD must be populated to push the chart to an HTTPS repository"; \
+					exit 1; \
+				else \
+					$(HELM) repo add hmc $(REGISTRY_REPO); \
+					echo "Pushing $$chart to $(REGISTRY_REPO)"; \
+					$(HELM) cm-push "$$chart" $(REGISTRY_REPO) --username $$REGISTRY_USERNAME --password $$REGISTRY_PASSWORD; \
+				fi; \
+			fi; \
 		fi; \
 	done
 
@@ -277,21 +293,37 @@ dev-azure-creds: envsubst
 dev-apply: kind-deploy registry-deploy dev-push dev-deploy dev-templates
 
 .PHONY: dev-destroy
-dev-destroy: kind-undeploy registry-undeploy
-
-.PHONY: dev-creds-apply
-dev-creds-apply: dev-$(DEV_PROVIDER)-creds
+dev-destroy: kind-undeploy registry-undeploy ## Destroy the development environment by deleting the kind cluster and local registry.
 
 .PHONY: dev-provider-apply
 dev-provider-apply: envsubst
-	@NAMESPACE=$(NAMESPACE) $(ENVSUBST) -no-unset -i config/dev/$(DEV_PROVIDER)-managedcluster.yaml | $(KUBECTL) apply -f -
+	@if [ $(DEV_PROVIDER) = "aws" ]; then \
+		$(MAKE) dev-aws-creds; \
+	fi
+	@NAMESPACE=$(NAMESPACE) $(ENVSUBST) -no-unset -i config/dev/$(DEV_PROVIDER)-deployment.yaml | $(KUBECTL) apply -f -
 
 .PHONY: dev-provider-delete
 dev-provider-delete: envsubst
 	@NAMESPACE=$(NAMESPACE) $(ENVSUBST) -no-unset -i config/dev/$(DEV_PROVIDER)-managedcluster.yaml | $(KUBECTL) delete -f -
 
+.PHONY: dev-creds-apply
+dev-creds-apply: dev-$(DEV_PROVIDER)-creds
+
+.PHONY: envsubst awscli dev-aws-nuke
+dev-aws-nuke: ## Warning: Destructive! Nuke all AWS resources deployed by 'DEV_PROVIDER=aws dev-provider-apply', prefix with CLUSTER_NAME to nuke a specific cluster.
+	@CLUSTER_NAME=$(CLUSTER_NAME) $(ENVSUBST) < config/dev/cloud_nuke.yaml.tpl > config/dev/cloud_nuke.yaml
+	DISABLE_TELEMETRY=true $(CLOUDNUKE) aws --region $$AWS_REGION --force --config config/dev/cloud_nuke.yaml --resource-type vpc,eip,nat-gateway,ec2-subnet,elb,elbv2,internet-gateway,network-interface,security-group
+	@rm config/dev/cloud_nuke.yaml
+	@CLUSTER_NAME=$(CLUSTER_NAME) YQ=$(YQ) AWSCLI=$(AWSCLI) bash -c ./scripts/aws-nuke-ccm.sh
+
+.PHONY: test-apply
+test-apply: kind-deploy registry-deploy dev-push dev-deploy dev-templates
+
+.PHONY: test-destroy
+test-destroy: kind-undeploy registry-undeploy
+
 .PHONY: cli-install
-cli-install: clusterawsadm clusterctl
+cli-install: clusterawsadm clusterctl cloud-nuke yq awscli ## Install the necessary CLI tools for deployment, development and testing.
 
 ##@ Dependencies
 
@@ -320,8 +352,10 @@ KIND ?= $(LOCALBIN)/kind-$(KIND_VERSION)
 YQ ?= $(LOCALBIN)/yq-$(YQ_VERSION)
 CLUSTERAWSADM ?= $(LOCALBIN)/clusterawsadm
 CLUSTERCTL ?= $(LOCALBIN)/clusterctl
+CLOUDNUKE ?= $(LOCALBIN)/cloud-nuke
 ADDLICENSE ?= $(LOCALBIN)/addlicense-$(ADDLICENSE_VERSION)
 ENVSUBST ?= $(LOCALBIN)/envsubst-$(ENVSUBST_VERSION)
+AWSCLI ?= $(LOCALBIN)/aws
 
 ## Tool Versions
 CONTROLLER_TOOLS_VERSION ?= v0.14.0
@@ -330,10 +364,12 @@ GOLANGCI_LINT_VERSION ?= v1.60.1
 HELM_VERSION ?= v3.15.1
 KIND_VERSION ?= v0.23.0
 YQ_VERSION ?= v4.44.2
+CLOUDNUKE_VERSION = v0.37.1
 CLUSTERAWSADM_VERSION ?= v2.5.2
 CLUSTERCTL_VERSION ?= v1.7.3
 ADDLICENSE_VERSION ?= v1.1.1
 ENVSUBST_VERSION ?= v1.4.2
+AWSCLI_VERSION ?= 2.17.42
 
 .PHONY: controller-gen
 controller-gen: $(CONTROLLER_GEN) ## Download controller-gen locally if necessary.
@@ -382,6 +418,12 @@ yq: $(YQ) ## Download yq locally if necessary.
 $(YQ): | $(LOCALBIN)
 	$(call go-install-tool,$(YQ),github.com/mikefarah/yq/v4,${YQ_VERSION})
 
+.PHONY: cloud-nuke
+cloud-nuke: $(CLOUDNUKE) ## Download cloud-nuke locally if necessary.
+$(CLOUDNUKE): | $(LOCALBIN)
+	curl -sL https://github.com/gruntwork-io/cloud-nuke/releases/download/$(CLOUDNUKE_VERSION)/cloud-nuke_$(OS)_$(ARCH) -o $(CLOUDNUKE)
+	chmod +x $(CLOUDNUKE)
+
 .PHONY: clusterawsadm
 clusterawsadm: $(CLUSTERAWSADM) ## Download clusterawsadm locally if necessary.
 $(CLUSTERAWSADM): | $(LOCALBIN)
@@ -403,6 +445,13 @@ envsubst: $(ENVSUBST)
 $(ENVSUBST): | $(LOCALBIN)
 	$(call go-install-tool,$(ENVSUBST),github.com/a8m/envsubst/cmd/envsubst,${ENVSUBST_VERSION})
 
+.PHONY: awscli
+awscli: $(AWSCLI)
+$(AWSCLI): | $(LOCALBIN)
+	curl "https://awscli.amazonaws.com/awscli-exe-$(OS)-$(shell uname -m)-$(AWSCLI_VERSION).zip" -o "/tmp/awscliv2.zip"
+	unzip /tmp/awscliv2.zip -d /tmp
+	/tmp/aws/install -i $(LOCALBIN)/aws-cli -b $(LOCALBIN) --update
+
 # go-install-tool will 'go install' any package with custom target and name of binary, if it doesn't exist
 # $1 - target path with name of binary (ideally with version)
 # $2 - package url which can be installed
@@ -413,6 +462,6 @@ set -e; \
 package=$(2)@$(3) ;\
 echo "Downloading $${package}" ;\
 GOBIN=$(LOCALBIN) go install $${package} ;\
-mv "$$(echo "$(1)" | sed "s/-$(3)$$//")" $(1) ;\
+if [ ! -f $(1) ]; then mv -f "$$(echo "$(1)" | sed "s/-$(3)$$//")" $(1); fi ;\
 }
 endef

config/dev/aws-managedcluster.yaml

@@ -4,13 +4,13 @@ metadata:
   name: aws-dev
   namespace: ${NAMESPACE}
 spec:
-  template: aws-standalone-cp
   config:
-    region: us-east-2
-    publicIP: true
-    controlPlaneNumber: 1
-    workersNumber: 1
     controlPlane:
       instanceType: t3.small
+    controlPlaneNumber: 1
+    publicIP: true
+    region: us-west-2
     worker:
       instanceType: t3.small
+    workersNumber: 1
+  template: aws-standalone-cp