Add hosted template, improve GetUnstructuredDeployment, break out tes…

…t contexts

Signed-off-by: Kyle Squizzato <[email protected]>

.github/workflows/test.yml

@@ -72,3 +72,13 @@ jobs:
       - name: Run E2E tests
         run: |
           make test-e2e
+      - name: Get test logs
+        run: |
+          kubectl logs -n hmc-system -l app=e2e-test > test/e2e/e2e-test.log
+          ./bin/clusterctl describe cluster --show-conditions=all > test/e2e/clusterctl.log
+      - name: Archive test results
+        uses: actions/upload-artifact@v4
+        with:
+          name: test-logs
+          path: |
+            test/e2e/*.log

.gitignore

@@ -14,10 +14,13 @@ dist
 go.work
 go.work.sum
 
-# cloud nuke config
+# cloud-nuke config
 *cloud_nuke.yaml
 
-# editors
+# Test artifacts
+test/e2e/*.log
+
+# ditors
 .idea
 *.swp
 *.swo

config/dev/deployment.yaml

@@ -5,13 +5,13 @@ metadata:
 spec:
   template: aws-standalone-cp
   config:
-    region: us-east-2
+    region: us-west-2
     publicIP: true
     controlPlaneNumber: 1
     workersNumber: 1
     controlPlane:
-      amiID: ami-02f3416038bdb17fb
+      amiID: ami-0989c067ff3da4b27
       instanceType: t3.small
     worker:
-      amiID: ami-02f3416038bdb17fb
+      amiID: ami-0989c067ff3da4b27
       instanceType: t3.small

go.mod

@@ -23,6 +23,7 @@ require (
 	k8s.io/apiextensions-apiserver v0.31.0
 	k8s.io/apimachinery v0.31.0
 	k8s.io/client-go v0.31.0
+	k8s.io/utils v0.0.0-20240711033017-18e509b52bc8
 	sigs.k8s.io/controller-runtime v0.19.0
 )
 
@@ -163,7 +164,6 @@ require (
 	k8s.io/klog/v2 v2.130.1 // indirect
 	k8s.io/kube-openapi v0.0.0-20240430033511-f0e62f92d13f // indirect
 	k8s.io/kubectl v0.31.0 // indirect
-	k8s.io/utils v0.0.0-20240711033017-18e509b52bc8 // indirect
 	oras.land/oras-go v1.2.5 // indirect
 	sigs.k8s.io/gateway-api v1.1.0 // indirect
 	sigs.k8s.io/json v0.0.0-20221116044647-bc3834ca7abd // indirect

test/deployment/deployment.go

@@ -43,25 +43,36 @@ const (
 	TemplateAWSHostedCP     Template = "aws-hosted-cp"
 )
 
-//go:embed resources/deployment.yaml.tpl
-var deploymentTemplateBytes []byte
+//go:embed resources/aws-standalone-cp.yaml.tpl
+var awsStandaloneCPDeploymentTemplateBytes []byte
+
+//go:embed resources/aws-hosted-cp.yaml.tpl
+var awsHostedCPDeploymentTemplateBytes []byte
 
 // GetUnstructuredDeployment returns an unstructured deployment object based on
 // the provider and template.
 func GetUnstructuredDeployment(provider ProviderType, templateName Template) *unstructured.Unstructured {
 	GinkgoHelper()
 
 	generatedName := uuid.New().String()[:8] + "-e2e-test"
-	_, _ = fmt.Fprintf(GinkgoWriter, "Generated AWS cluster name: %q\n", generatedName)
+	_, _ = fmt.Fprintf(GinkgoWriter, "Generated cluster name: %q\n", generatedName)
 
 	switch provider {
 	case ProviderAWS:
 		// XXX: Maybe we should just use automatic AMI selection here.
 		amiID := getAWSAMI()
-
-		Expect(os.Setenv("AMI_ID", amiID)).NotTo(HaveOccurred())
+		Expect(os.Setenv("AWS_AMI_ID", amiID)).NotTo(HaveOccurred())
 		Expect(os.Setenv("DEPLOYMENT_NAME", generatedName)).NotTo(HaveOccurred())
-		Expect(os.Setenv("TEMPLATE_NAME", string(templateName))).NotTo(HaveOccurred())
+
+		var deploymentTemplateBytes []byte
+		switch templateName {
+		case TemplateAWSStandaloneCP:
+			deploymentTemplateBytes = awsStandaloneCPDeploymentTemplateBytes
+		case TemplateAWSHostedCP:
+			deploymentTemplateBytes = awsHostedCPDeploymentTemplateBytes
+		default:
+			Fail(fmt.Sprintf("unsupported AWS template: %s", templateName))
+		}
 
 		deploymentConfigBytes, err := envsubst.Bytes(deploymentTemplateBytes)
 		Expect(err).NotTo(HaveOccurred(), "failed to substitute environment variables")

test/deployment/resources/aws-hosted-cp.yaml.tpl

@@ -0,0 +1,17 @@
+apiVersion: hmc.mirantis.com/v1alpha1
+kind: Deployment
+metadata:
+  name: ${DEPLOYMENT_NAME}
+spec:
+  template: aws-hosted-cp
+  config:
+    vpcID: ${AWS_VPC_ID}
+    region: ${AWS_REGION}
+    publicIP: ${PUBLIC_IP:=true}
+    subnets:
+      - id: ${AWS_SUBNET_ID}
+        availabilityZone: ${AWS_SUBNET_AVAILABILITY_ZONE}
+    amiID: ${AWS_AMI_ID}
+    instanceType: ${INSTANCE_TYPE:=t3.medium}
+    securityGroupIDs:
+      - ${AWS_SG_ID}

test/deployment/resources/aws-standalone-cp.yaml.tpl

@@ -0,0 +1,19 @@
+apiVersion: hmc.mirantis.com/v1alpha1
+kind: Deployment
+metadata:
+  name: ${DEPLOYMENT_NAME}
+spec:
+  template: aws-standalone-cp
+  config:
+    region: ${AWS_REGION}
+    publicIP: ${PUBLIC_IP:=true}
+    controlPlaneNumber: ${CONTROL_PLANE_NUMBER:=1}
+    workersNumber: ${WORKERS_NUMBER:=1}
+    controlPlane:
+      amiID: ${AWS_AMI_ID}
+      instanceType: ${INSTANCE_TYPE:=t3.small}
+    worker:
+      amiID: ${AWS_AMI_ID}
+      instanceType: ${INSTANCE_TYPE:=t3.small}
+
+

test/deployment/validate_deployed.go

@@ -103,10 +103,6 @@ func validateMachines(ctx context.Context, kc *kubeclient.KubeClient, clusterNam
 		return fmt.Errorf("failed to list machines: %w", err)
 	}
 
-	if err != nil {
-		return fmt.Errorf("failed to list Machines: %w", err)
-	}
-
 	for _, machine := range machines {
 		if err := utils.ValidateObjectNamePrefix(&machine, clusterName); err != nil {
 			Fail(err.Error())

test/e2e/e2e_test.go

@@ -15,16 +15,20 @@
 package e2e
 
 import (
+	"bufio"
 	"context"
 	"fmt"
 	"os"
 	"os/exec"
+	"path/filepath"
 	"strings"
 	"time"
 
 	. "github.com/onsi/ginkgo/v2"
 	. "github.com/onsi/gomega"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/utils/ptr"
 
 	"github.com/Mirantis/hmc/test/deployment"
 	"github.com/Mirantis/hmc/test/kubeclient"
@@ -98,10 +102,10 @@ var _ = Describe("controller", Ordered, func() {
 
 	Context("AWS Templates", func() {
 		var (
-			kc                   *kubeclient.KubeClient
-			deleteDeploymentFunc func() error
-			clusterName          string
-			err                  error
+			kc          *kubeclient.KubeClient
+			deleteFunc  func() error
+			clusterName string
+			err         error
 		)
 
 		BeforeAll(func() {
@@ -111,10 +115,17 @@ var _ = Describe("controller", Ordered, func() {
 			ExpectWithOffset(2, kc.CreateAWSCredentialsKubeSecret(context.Background())).To(Succeed())
 		})
 
-		AfterAll(func() {
-			// Delete the deployment if it was created.
-			if deleteDeploymentFunc != nil {
-				err = deleteDeploymentFunc()
+		AfterEach(func() {
+			// If we failed collect logs from each of the affiliated controllers
+			// as well as the output of clusterctl to store as artifacts.
+			if CurrentSpecReport().Failed() {
+				By("collecting failure logs from controllers")
+				collectLogArtifacts(kc, clusterName, deployment.ProviderAWS)
+			}
+
+			// Delete the deployments if they were created.
+			if deleteFunc != nil {
+				err = deleteFunc()
 				Expect(err).NotTo(HaveOccurred())
 			}
 
@@ -127,29 +138,79 @@ var _ = Describe("controller", Ordered, func() {
 			ExpectWithOffset(2, err).NotTo(HaveOccurred())
 		})
 
-		It("should work with an AWS provider", func() {
-			By("creating a Deployment with aws-standalone-cp template")
-			d := deployment.GetUnstructuredDeployment(deployment.ProviderAWS, deployment.TemplateAWSStandaloneCP)
-			clusterName = d.GetName()
+		for _, template := range []deployment.Template{deployment.TemplateAWSStandaloneCP, deployment.TemplateAWSHostedCP} {
+			It(fmt.Sprintf("should work with an AWS provider and %s template", template), func() {
+				if template == deployment.TemplateAWSHostedCP {
+					// TODO: Create AWS resources for hosted control plane.
+					Skip("AWS hosted control plane not yet implemented")
+				}
 
-			deleteDeploymentFunc, err = kc.CreateDeployment(context.Background(), d)
-			Expect(err).NotTo(HaveOccurred())
+				By("creating a Deployment")
+				d := deployment.GetUnstructuredDeployment(deployment.ProviderAWS, template)
+				clusterName = d.GetName()
 
-			By("waiting for infrastructure providers to deploy successfully")
-			Eventually(func() error {
-				return deployment.VerifyProviderDeployed(context.Background(), kc, clusterName)
-			}).WithTimeout(30 * time.Minute).WithPolling(10 * time.Second).Should(Succeed())
+				deleteFunc, err = kc.CreateDeployment(context.Background(), d)
+				Expect(err).NotTo(HaveOccurred())
 
-			By("verifying the deployment deletes successfully")
-			err = deleteDeploymentFunc()
-			Expect(err).NotTo(HaveOccurred())
-			Eventually(func() error {
-				return deployment.VerifyProviderDeleted(context.Background(), kc, clusterName)
-			}).WithTimeout(10 * time.Minute).WithPolling(10 * time.Second).Should(Succeed())
+				By("waiting for infrastructure providers to deploy successfully")
+				Eventually(func() error {
+					return deployment.VerifyProviderDeployed(context.Background(), kc, clusterName)
+				}).WithTimeout(30 * time.Minute).WithPolling(10 * time.Second).Should(Succeed())
 
-			By("creating a Deployment with aws-hosted-cp template")
-			// TODO: Use the standalone control plane resources to craft a
-			// hosted control plane and test it.
-		})
+				By("verify the deployment deletes successfully")
+				err = deleteFunc()
+				Expect(err).NotTo(HaveOccurred())
+				Eventually(func() error {
+					return deployment.VerifyProviderDeleted(context.Background(), kc, clusterName)
+				}).WithTimeout(10 * time.Minute).WithPolling(10 * time.Second).Should(Succeed())
+			})
+		}
 	})
 })
+
+// collectLogArtfiacts collects log output from each the HMC controller,
+// CAPI controller and the provider controller as well as output from clusterctl
+// and stores them in the test/e2e directory as artifacts.
+// We could do this at the end or we could use Kubernetes' CopyPodLogs from
+// https://github.com/kubernetes/kubernetes/blob/v1.31.0/test/e2e/storage/podlogs/podlogs.go#L88
+// to stream the logs to GinkgoWriter during the test.
+func collectLogArtifacts(kc *kubeclient.KubeClient, clusterName string, providerType deployment.ProviderType) {
+	GinkgoHelper()
+
+	filterLabels := []string{
+		"app.kubernetes.io/name=hmc-controller-manager",
+		"app.kubernetes.io/name=cluster-api",
+		fmt.Sprintf("app.kubernetes.io/name=cluster-api-provider-%s", providerType),
+	}
+
+	for _, label := range filterLabels {
+		pods, _ := kc.Client.CoreV1().Pods(kc.Namespace).List(context.Background(), metav1.ListOptions{
+			LabelSelector: label,
+		})
+
+		for _, pod := range pods.Items {
+			req := kc.Client.CoreV1().Pods(kc.Namespace).GetLogs(pod.Name, &corev1.PodLogOptions{
+				TailLines: ptr.To(int64(1000)),
+			})
+			podLogs, err := req.Stream(context.Background())
+			Expect(err).NotTo(HaveOccurred(), "failed to get log stream for pod %s", pod.Name)
+			DeferCleanup(Expect(podLogs.Close()).NotTo(HaveOccurred()))
+
+			output, err := os.Create(fmt.Sprintf("test/e2e/%s.log", pod.Name))
+			Expect(err).NotTo(HaveOccurred(), "failed to create log file for pod %s", pod.Name)
+			DeferCleanup(Expect(output.Close()).NotTo(HaveOccurred()))
+
+			r := bufio.NewReader(podLogs)
+			_, err = r.WriteTo(output)
+			Expect(err).NotTo(HaveOccurred(), "failed to write log file for pod %s", pod.Name)
+		}
+	}
+
+	cmd := exec.Command("./bin/clusterctl",
+		"describe", "cluster", clusterName, "--show-conditions=all")
+	output, err := utils.Run(cmd)
+	Expect(err).NotTo(HaveOccurred(), "failed to get clusterctl log")
+
+	err = os.WriteFile(filepath.Join("test/e2e", "clusterctl.log"), output, 0644)
+	Expect(err).NotTo(HaveOccurred(), "failed to write clusterctl log")
+}