[e2e] Support to provide e2e testing configuration

Closes #641

.github/workflows/build_test.yml

@@ -29,7 +29,7 @@ jobs:
     runs-on: ubuntu-latest
     outputs:
       version: ${{ steps.vars.outputs.version }}
-      clustername: ${{ steps.vars.outputs.clustername }}
+      clusterprefix: ${{ steps.vars.outputs.clusterprefix }}
       pr: ${{ steps.pr.outputs.result }}
     steps:
       - name: Get PR ref
@@ -72,7 +72,7 @@ jobs:
         run: |
           GIT_VERSION=$(git describe --tags --always)
           echo "version=${GIT_VERSION:1}" >> $GITHUB_OUTPUT
-          echo "clustername=ci-$(date +%s | cut -b6-10)" >> $GITHUB_OUTPUT
+          echo "clusterprefix=ci-$(date +%s | cut -b6-10)-e2e-test" >> $GITHUB_OUTPUT
       - name: Build and push HMC controller image
         uses: docker/build-push-action@v6
         with:
@@ -98,7 +98,7 @@ jobs:
       group: controller-${{ github.head_ref || github.run_id }}
       cancel-in-progress: true
     outputs:
-      clustername: ${{ needs.build.outputs.clustername }}
+      clusterprefix: ${{ needs.build.outputs.clusterprefix }}
       version: ${{ needs.build.outputs.version }}
       pr: ${{ needs.build.outputs.pr }}
     steps:
@@ -112,7 +112,7 @@ jobs:
       - name: Run E2E tests
         env:
           GINKGO_LABEL_FILTER: 'controller'
-          CLUSTER_DEPLOYMENT_NAME: ${{ needs.build.outputs.clustername }}
+          CLUSTER_DEPLOYMENT_PREFIX: ${{ needs.build.outputs.clusterprefix }}
           IMG: 'ghcr.io/mirantis/hmc/controller-ci:${{ needs.build.outputs.version }}'
           VERSION: ${{ needs.build.outputs.version }}
         run: |
@@ -134,7 +134,7 @@ jobs:
       group: cloud-${{ github.head_ref || github.run_id }}
       cancel-in-progress: true
     outputs:
-      clustername: ${{ needs.build.outputs.clustername }}
+      clusterprefix: ${{ needs.build.outputs.clusterprefix }}
       version: ${{ needs.build.outputs.version }}
       pr: ${{ needs.build.outputs.pr }}
     env:
@@ -162,7 +162,7 @@ jobs:
       - name: Run E2E tests
         env:
           GINKGO_LABEL_FILTER: 'provider:cloud'
-          CLUSTER_DEPLOYMENT_NAME: ${{ needs.build.outputs.clustername }}
+          CLUSTER_DEPLOYMENT_PREFIX: ${{ needs.build.outputs.clusterprefix }}
           IMG: 'ghcr.io/mirantis/hmc/controller-ci:${{ needs.build.outputs.version }}'
           VERSION: ${{ needs.build.outputs.version }}
         run: |
@@ -184,7 +184,7 @@ jobs:
       group: onprem-${{ github.head_ref || github.run_id }}
       cancel-in-progress: true
     outputs:
-      clustername: ${{ needs.build.outputs.clustername }}
+      clusterprefix: ${{ needs.build.outputs.clusterprefix }}
       version: ${{ needs.build.outputs.version }}
       pr: ${{ needs.build.outputs.pr }}
     env:
@@ -215,7 +215,7 @@ jobs:
       - name: Run E2E tests
         env:
           GINKGO_LABEL_FILTER: 'provider:onprem'
-          CLUSTER_DEPLOYMENT_NAME: ${{ needs.build.outputs.clustername }}
+          CLUSTER_DEPLOYMENT_PREFIX: ${{ needs.build.outputs.clusterprefix }}
           IMG: 'ghcr.io/mirantis/hmc/controller-ci:${{ needs.build.outputs.version }}'
           VERSION: ${{ needs.build.outputs.version }}
         run: |
@@ -237,7 +237,7 @@ jobs:
     if: ${{ always() && !contains(needs.provider-cloud-e2etest.result, 'skipped') && contains(needs.build.result, 'success') }}
     timeout-minutes: 15
     outputs:
-      clustername: ${{ needs.build.outputs.clustername }}
+      clusterprefix: ${{ needs.build.outputs.clusterprefix }}
       version: ${{ needs.build.outputs.version }}
       pr: ${{ needs.build.outputs.pr }}
     steps:
@@ -260,7 +260,7 @@ jobs:
           AZURE_TENANT_ID: ${{ secrets.CI_AZURE_TENANT_ID }}
           AZURE_CLIENT_ID: ${{ secrets.CI_AZURE_CLIENT_ID }}
           AZURE_CLIENT_SECRET: ${{ secrets.CI_AZURE_CLIENT_SECRET }}
-          CLUSTER_NAME: '${{ needs.build.outputs.clustername }}'
+          CLUSTER_NAME: '${{ needs.build.outputs.clusterprefix }}'
         run: |
           make dev-aws-nuke
           make dev-azure-nuke

Makefile

@@ -108,14 +108,18 @@ tidy:
 test: generate-all envtest tidy external-crd ## Run tests.
 	KUBEBUILDER_ASSETS="$(shell $(ENVTEST) use $(ENVTEST_K8S_VERSION) --bin-dir $(LOCALBIN) -p path)" go test $$(go list ./... | grep -v /e2e) -coverprofile cover.out
 
+# E2E_CONFIG_B64 contains the configuration for e2e testing.
+E2E_CONFIG_B64 ?= ""
+
 # Utilize Kind or modify the e2e tests to load the image locally, enabling
 # compatibility with other vendors.
 .PHONY: test-e2e # Run the e2e tests using a Kind k8s instance as the management cluster.
 test-e2e: cli-install
 	@if [ "$$GINKGO_LABEL_FILTER" ]; then \
 		ginkgo_label_flag="-ginkgo.label-filter=$$GINKGO_LABEL_FILTER"; \
 	fi; \
-	KIND_CLUSTER_NAME="hmc-test" KIND_VERSION=$(KIND_VERSION) go test ./test/e2e/ -v -ginkgo.v -ginkgo.timeout=3h -timeout=3h $$ginkgo_label_flag
+	KIND_CLUSTER_NAME="hmc-test" KIND_VERSION=$(KIND_VERSION) E2E_CONFIG_B64=$(E2E_CONFIG_B64) \
+	go test ./test/e2e/ -v -ginkgo.v -ginkgo.timeout=3h -timeout=3h $$ginkgo_label_flag
 
 .PHONY: lint
 lint: golangci-lint fmt vet ## Run golangci-lint linter & yamllint

docs/dev.md

@@ -149,13 +149,10 @@ IMG="ghcr.io/mirantis/hmc/controller-ci:v0.0.1-179-ga5bdf29" \
 Optionally, the `NO_CLEANUP=1` env var can be used to disable `After` nodes from
 running within some specs, this will allow users to debug tests by re-running
 them without the need to wait a while for an infrastructure deployment to occur.
-For subsequent runs the `CLUSTER_DEPLOYMENT_NAME=<cluster name>` env var should be
-passed to tell the test what cluster name to use so that it does not try to
-generate a new name and deploy a new cluster.
 
-Tests that run locally use autogenerated names like `12345678-e2e-test` while
-tests that run in CI use names such as `ci-1234567890-e2e-test`.  You can always
-pass `CLUSTER_DEPLOYMENT_NAME=` from the get-go to customize the name used by the
+Tests that run locally use autogenerated names prefixes like `e2e-test-12345` while
+tests that run in CI use names such as `ci-12345-e2e-test`.  You can always
+pass `CLUSTER_DEPLOYMENT_PREFIX=` from the get-go to customize the prefix used by the
 test.
 
 ### Filtering test runs

test/e2e/clusterdeployment/aws/aws.go

@@ -12,7 +12,7 @@
 // See the License for the specific language governing permissions and
 // limitations under the License.
 
-// Package aws contains specific helpers for testing a managed cluster
+// Package aws contains specific helpers for testing a cluster deployment
 // that uses the AWS infrastructure provider.
 package aws
 

test/e2e/clusterdeployment/clusterdeployment.go

@@ -18,7 +18,6 @@ import (
 	_ "embed"
 	"fmt"
 	"os"
-	"strings"
 
 	"github.com/a8m/envsubst"
 	"github.com/google/uuid"
@@ -27,6 +26,7 @@ import (
 	"gopkg.in/yaml.v3"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 
+	"github.com/Mirantis/hmc/test/e2e/templates"
 	"github.com/Mirantis/hmc/test/utils"
 )
 
@@ -41,17 +41,6 @@ const (
 	providerLabel = "cluster.x-k8s.io/provider"
 )
 
-type Template string
-
-const (
-	TemplateAWSStandaloneCP     Template = "aws-standalone-cp"
-	TemplateAWSHostedCP         Template = "aws-hosted-cp"
-	TemplateAzureHostedCP       Template = "azure-hosted-cp"
-	TemplateAzureStandaloneCP   Template = "azure-standalone-cp"
-	TemplateVSphereStandaloneCP Template = "vsphere-standalone-cp"
-	TemplateVSphereHostedCP     Template = "vsphere-hosted-cp"
-)
-
 //go:embed resources/aws-standalone-cp.yaml.tpl
 var awsStandaloneCPClusterDeploymentTemplateBytes []byte
 
@@ -84,38 +73,39 @@ func GetProviderLabel(provider ProviderType) string {
 	return fmt.Sprintf("%s=%s", providerLabel, provider)
 }
 
-func setClusterName(templateName Template) {
-	var generatedName string
-
-	mcName := os.Getenv(EnvVarClusterDeploymentName)
-	if mcName == "" {
-		mcName = "e2e-test-" + uuid.New().String()[:8]
+func GenerateClusterName(postfix string) string {
+	mcPrefix := os.Getenv(EnvVarClusterDeploymentPrefix)
+	if mcPrefix == "" {
+		mcPrefix = "e2e-test-" + uuid.New().String()[:8]
 	}
 
-	providerName := strings.Split(string(templateName), "-")[0]
-
-	// Append the provider name to the cluster name to ensure uniqueness between
-	// different deployed ClusterDeployments.
-	generatedName = fmt.Sprintf("%s-%s", mcName, providerName)
-	if strings.Contains(string(templateName), "hosted") {
-		generatedName = fmt.Sprintf("%s-%s", mcName, "hosted")
+	if postfix != "" {
+		return fmt.Sprintf("%s-%s", mcPrefix, postfix)
 	}
+	return mcPrefix
+}
+
+func setClusterName(name string) {
+	GinkgoT().Setenv(EnvVarClusterDeploymentName, name)
+}
 
-	GinkgoT().Setenv(EnvVarClusterDeploymentName, generatedName)
+func setTemplate(templateName string) {
+	GinkgoT().Setenv(EnvVarClusterDeploymentTemplate, templateName)
 }
 
 // GetUnstructured returns an unstructured ClusterDeployment object based on the
 // provider and template.
-func GetUnstructured(templateName Template) *unstructured.Unstructured {
+func GetUnstructured(templateType templates.Type, clusterName, template string) *unstructured.Unstructured {
 	GinkgoHelper()
 
-	setClusterName(templateName)
+	setClusterName(clusterName)
+	setTemplate(template)
 
 	var clusterDeploymentTemplateBytes []byte
-	switch templateName {
-	case TemplateAWSStandaloneCP:
+	switch templateType {
+	case templates.TemplateAWSStandaloneCP:
 		clusterDeploymentTemplateBytes = awsStandaloneCPClusterDeploymentTemplateBytes
-	case TemplateAWSHostedCP:
+	case templates.TemplateAWSHostedCP:
 		// Validate environment vars that do not have defaults are populated.
 		// We perform this validation here instead of within a Before block
 		// since we populate the vars from standalone prior to this step.
@@ -126,16 +116,16 @@ func GetUnstructured(templateName Template) *unstructured.Unstructured {
 			EnvVarAWSSecurityGroupID,
 		})
 		clusterDeploymentTemplateBytes = awsHostedCPClusterDeploymentTemplateBytes
-	case TemplateVSphereStandaloneCP:
+	case templates.TemplateVSphereStandaloneCP:
 		clusterDeploymentTemplateBytes = vsphereStandaloneCPClusterDeploymentTemplateBytes
-	case TemplateVSphereHostedCP:
+	case templates.TemplateVSphereHostedCP:
 		clusterDeploymentTemplateBytes = vsphereHostedCPClusterDeploymentTemplateBytes
-	case TemplateAzureHostedCP:
+	case templates.TemplateAzureHostedCP:
 		clusterDeploymentTemplateBytes = azureHostedCPClusterDeploymentTemplateBytes
-	case TemplateAzureStandaloneCP:
+	case templates.TemplateAzureStandaloneCP:
 		clusterDeploymentTemplateBytes = azureStandaloneCPClusterDeploymentTemplateBytes
 	default:
-		Fail(fmt.Sprintf("Unsupported template: %s", templateName))
+		Fail(fmt.Sprintf("Unsupported template type: %s", templateType))
 	}
 
 	clusterDeploymentConfigBytes, err := envsubst.Bytes(clusterDeploymentTemplateBytes)

test/e2e/clusterdeployment/constants.go

@@ -16,10 +16,12 @@ package clusterdeployment
 
 const (
 	// Common
-	EnvVarClusterDeploymentName = "CLUSTER_DEPLOYMENT_NAME"
-	EnvVarControlPlaneNumber    = "CONTROL_PLANE_NUMBER"
-	EnvVarWorkerNumber          = "WORKER_NUMBER"
-	EnvVarNamespace             = "NAMESPACE"
+	EnvVarClusterDeploymentName     = "CLUSTER_DEPLOYMENT_NAME"
+	EnvVarClusterDeploymentPrefix   = "CLUSTER_DEPLOYMENT_PREFIX"
+	EnvVarClusterDeploymentTemplate = "CLUSTER_DEPLOYMENT_TEMPLATE"
+	EnvVarControlPlaneNumber        = "CONTROL_PLANE_NUMBER"
+	EnvVarWorkerNumber              = "WORKER_NUMBER"
+	EnvVarNamespace                 = "NAMESPACE"
 	// EnvVarNoCleanup disables After* cleanup in provider specs to allow for
 	// debugging of test failures.
 	EnvVarNoCleanup = "NO_CLEANUP"

test/e2e/clusterdeployment/providervalidator.go

@@ -21,14 +21,15 @@ import (
 	. "github.com/onsi/ginkgo/v2"
 
 	"github.com/Mirantis/hmc/test/e2e/kubeclient"
+	"github.com/Mirantis/hmc/test/e2e/templates"
 )
 
 // ProviderValidator is a struct that contains the necessary information to
 // validate a provider's resources.  Some providers do not support all of the
 // resources that can potentially be validated.
 type ProviderValidator struct {
-	// Template is the name of the template being validated.
-	template Template
+	// Template is the type of the template being validated.
+	templateType templates.Type
 	// ClusterName is the name of the cluster to validate.
 	clusterName string
 	// ResourcesToValidate is a map of resource names to their validation
@@ -46,7 +47,7 @@ const (
 	ValidationActionDelete ValidationAction = "delete"
 )
 
-func NewProviderValidator(template Template, clusterName string, action ValidationAction) *ProviderValidator {
+func NewProviderValidator(templateType templates.Type, clusterName string, action ValidationAction) *ProviderValidator {
 	var (
 		resourcesToValidate map[string]resourceValidationFunc
 		resourceOrder       []string
@@ -61,11 +62,11 @@ func NewProviderValidator(template Template, clusterName string, action Validati
 		}
 		resourceOrder = []string{"clusters", "machines", "control-planes", "csi-driver"}
 
-		switch template {
-		case TemplateAWSStandaloneCP, TemplateAWSHostedCP:
+		switch templateType {
+		case templates.TemplateAWSStandaloneCP, templates.TemplateAWSHostedCP:
 			resourcesToValidate["ccm"] = validateCCM
 			resourceOrder = append(resourceOrder, "ccm")
-		case TemplateAzureStandaloneCP, TemplateVSphereStandaloneCP:
+		case templates.TemplateAzureStandaloneCP, templates.TemplateVSphereStandaloneCP:
 			delete(resourcesToValidate, "csi-driver")
 		}
 	} else {
@@ -78,7 +79,7 @@ func NewProviderValidator(template Template, clusterName string, action Validati
 	}
 
 	return &ProviderValidator{
-		template:            template,
+		templateType:        templateType,
 		clusterName:         clusterName,
 		resourcesToValidate: resourcesToValidate,
 		resourceOrder:       resourceOrder,
@@ -104,11 +105,11 @@ func (p *ProviderValidator) Validate(ctx context.Context, kc *kubeclient.KubeCli
 		}
 
 		if err := validator(ctx, kc, p.clusterName); err != nil {
-			_, _ = fmt.Fprintf(GinkgoWriter, "[%s/%s] validation error: %v\n", p.template, name, err)
+			_, _ = fmt.Fprintf(GinkgoWriter, "[%s/%s] validation error: %v\n", p.templateType, name, err)
 			return err
 		}
 
-		_, _ = fmt.Fprintf(GinkgoWriter, "[%s/%s] validation succeeded\n", p.template, name)
+		_, _ = fmt.Fprintf(GinkgoWriter, "[%s/%s] validation succeeded\n", p.templateType, name)
 		delete(p.resourcesToValidate, name)
 	}
 

test/e2e/clusterdeployment/resources/aws-hosted-cp.yaml.tpl

@@ -3,7 +3,7 @@ kind: ClusterDeployment
 metadata:
   name: ${CLUSTER_DEPLOYMENT_NAME}
 spec:
-  template: aws-hosted-cp-0-0-3
+  template: ${CLUSTER_DEPLOYMENT_TEMPLATE}
   credential: ${AWS_CLUSTER_IDENTITY}-cred
   config:
     clusterIdentity:

test/e2e/clusterdeployment/resources/aws-standalone-cp.yaml.tpl

@@ -3,7 +3,7 @@ kind: ClusterDeployment
 metadata:
   name: ${CLUSTER_DEPLOYMENT_NAME}
 spec:
-  template: aws-standalone-cp-0-0-4
+  template: ${CLUSTER_DEPLOYMENT_TEMPLATE}
   credential: ${AWS_CLUSTER_IDENTITY}-cred
   config:
     clusterIdentity:

test/e2e/clusterdeployment/resources/azure-hosted-cp.yaml.tpl

@@ -4,7 +4,7 @@ metadata:
   name: ${CLUSTER_DEPLOYMENT_NAME}
   namespace: ${NAMESPACE}
 spec:
-  template: azure-hosted-cp-0-0-3
+  template: ${CLUSTER_DEPLOYMENT_TEMPLATE}
   credential: ${AZURE_CLUSTER_IDENTITY}-cred
   config:
     location: "${AZURE_REGION}"

test/e2e/clusterdeployment/resources/azure-standalone-cp.yaml.tpl

@@ -4,7 +4,7 @@ metadata:
   name: ${CLUSTER_DEPLOYMENT_NAME}
   namespace: ${NAMESPACE}
 spec:
-  template: azure-standalone-cp-0-0-4
+  template: ${CLUSTER_DEPLOYMENT_TEMPLATE}
   credential: ${AZURE_CLUSTER_IDENTITY}-cred
   config:
     controlPlaneNumber: 1

test/e2e/clusterdeployment/resources/vsphere-hosted-cp.yaml.tpl

@@ -3,7 +3,7 @@ kind: ClusterDeployment
 metadata:
   name: ${CLUSTER_DEPLOYMENT_NAME}
 spec:
-  template: vsphere-hosted-cp-0-0-3
+  template: ${CLUSTER_DEPLOYMENT_TEMPLATE}
   credential: ${VSPHERE_CLUSTER_IDENTITY}-cred
   config:
     controlPlaneNumber: ${CONTROL_PLANE_NUMBER:=1}

test/e2e/clusterdeployment/resources/vsphere-standalone-cp.yaml.tpl

@@ -3,7 +3,7 @@ kind: ClusterDeployment
 metadata:
   name: ${CLUSTER_DEPLOYMENT_NAME}
 spec:
-  template: vsphere-standalone-cp-0-0-3
+  template: ${CLUSTER_DEPLOYMENT_TEMPLATE}
   credential: ${VSPHERE_CLUSTER_IDENTITY}-cred
   config:
     controlPlaneNumber: ${CONTROL_PLANE_NUMBER:=1}