Implement adopted cluster e2e tests

test/e2e/clusterdeployment/clusterdeployment.go

@@ -15,6 +15,7 @@
 package clusterdeployment
 
 import (
+	"context"
 	_ "embed"
 	"fmt"
 	"os"
@@ -27,6 +28,7 @@ import (
 	"gopkg.in/yaml.v3"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 
+	"github.com/Mirantis/hmc/test/e2e/kubeclient"
 	"github.com/Mirantis/hmc/test/utils"
 )
 
@@ -37,8 +39,8 @@ const (
 	ProviderAWS     ProviderType = "infrastructure-aws"
 	ProviderAzure   ProviderType = "infrastructure-azure"
 	ProviderVSphere ProviderType = "infrastructure-vsphere"
-
-	providerLabel = "cluster.x-k8s.io/provider"
+	ProviderAdopted ProviderType = "infrastructure-internal"
+	providerLabel                = "cluster.x-k8s.io/provider"
 )
 
 type Template string
@@ -50,6 +52,7 @@ const (
 	TemplateAzureStandaloneCP   Template = "azure-standalone-cp"
 	TemplateVSphereStandaloneCP Template = "vsphere-standalone-cp"
 	TemplateVSphereHostedCP     Template = "vsphere-hosted-cp"
+	TemplateAdoptedCluster      Template = "adopted-cluster"
 )
 
 //go:embed resources/aws-standalone-cp.yaml.tpl
@@ -70,6 +73,9 @@ var vsphereStandaloneCPClusterDeploymentTemplateBytes []byte
 //go:embed resources/vsphere-hosted-cp.yaml.tpl
 var vsphereHostedCPClusterDeploymentTemplateBytes []byte
 
+//go:embed resources/adopted-cluster.yaml.tpl
+var adoptedClusterDeploymentTemplateBytes []byte
+
 func FilterAllProviders() []string {
 	return []string{
 		utils.HMCControllerLabel,
@@ -134,6 +140,8 @@ func GetUnstructured(templateName Template) *unstructured.Unstructured {
 		clusterDeploymentTemplateBytes = azureHostedCPClusterDeploymentTemplateBytes
 	case TemplateAzureStandaloneCP:
 		clusterDeploymentTemplateBytes = azureStandaloneCPClusterDeploymentTemplateBytes
+	case TemplateAdoptedCluster:
+		clusterDeploymentTemplateBytes = adoptedClusterDeploymentTemplateBytes
 	default:
 		Fail(fmt.Sprintf("Unsupported template: %s", templateName))
 	}
@@ -156,3 +164,27 @@ func ValidateDeploymentVars(v []string) {
 		Expect(os.Getenv(envVar)).NotTo(BeEmpty(), envVar+" must be set")
 	}
 }
+
+func ValidateClusterTemplates(ctx context.Context, client *kubeclient.KubeClient) error {
+	templates, err := client.ListClusterTemplates(ctx)
+	if err != nil {
+		return fmt.Errorf("failed to list cluster templates: %w", err)
+	}
+
+	for _, template := range templates {
+		valid, found, err := unstructured.NestedBool(template.Object, "status", "valid")
+		if err != nil {
+			return fmt.Errorf("failed to get valid flag for template %s: %w", template.GetName(), err)
+		}
+
+		if !found {
+			return fmt.Errorf("valid flag for template %s not found", template.GetName())
+		}
+
+		if !valid {
+			return fmt.Errorf("template %s is still invalid", template.GetName())
+		}
+	}
+
+	return nil
+}

test/e2e/clusterdeployment/clusteridentity/clusteridentity.go

@@ -40,6 +40,7 @@ type ClusterIdentity struct {
 	SecretData           map[string]string
 	Spec                 map[string]any
 	Namespaced           bool
+	CredentialName       string
 }
 
 // New creates a ClusterIdentity resource, credential and associated secret for
@@ -59,15 +60,30 @@ func New(kc *kubeclient.KubeClient, provider clusterdeployment.ProviderType) *Cl
 
 	secretName := fmt.Sprintf("%s-cluster-identity-secret", provider)
 	identityName := fmt.Sprintf("%s-cluster-identity", provider)
+	group := "infrastructure.cluster.x-k8s.io"
 
 	switch provider {
+	case clusterdeployment.ProviderAdopted:
+		kubeCfgBytes, err := os.ReadFile(os.Getenv(clusterdeployment.EnvVarAdoptedKubeconfigPath))
+		Expect(err).NotTo(HaveOccurred())
+
+		kind = "Secret"
+		version = "v1"
+		group = ""
+		identityName = secretName
+
+		secretStringData = map[string]string{
+			"Value": string(kubeCfgBytes),
+		}
+
 	case clusterdeployment.ProviderAWS:
 		resource = "awsclusterstaticidentities"
 		kind = "AWSClusterStaticIdentity"
 		version = "v1beta2"
 		secretStringData = map[string]string{
 			"AccessKeyID":     os.Getenv(clusterdeployment.EnvVarAWSAccessKeyID),
 			"SecretAccessKey": os.Getenv(clusterdeployment.EnvVarAWSSecretAccessKey),
+			"SessionToken":    os.Getenv("AWS_SESSION_TOKEN"),
 		}
 		spec = map[string]any{
 			"secretRef": secretName,
@@ -117,22 +133,26 @@ func New(kc *kubeclient.KubeClient, provider clusterdeployment.ProviderType) *Cl
 
 	ci := ClusterIdentity{
 		GroupVersionResource: schema.GroupVersionResource{
-			Group:    "infrastructure.cluster.x-k8s.io",
+			Group:    group,
 			Version:  version,
 			Resource: resource,
 		},
-		Kind:         kind,
-		SecretName:   secretName,
-		IdentityName: identityName,
-		SecretData:   secretStringData,
-		Spec:         spec,
-		Namespaced:   namespaced,
+		Kind:           kind,
+		SecretName:     secretName,
+		IdentityName:   identityName,
+		SecretData:     secretStringData,
+		Spec:           spec,
+		Namespaced:     namespaced,
+		CredentialName: fmt.Sprintf("%s-cred", identityName),
 	}
 
 	validateSecretDataPopulated(secretStringData)
-	ci.waitForResourceCRD(kc)
 	ci.createSecret(kc)
-	ci.createClusterIdentity(kc)
+
+	if provider != clusterdeployment.ProviderAdopted {
+		ci.waitForResourceCRD(kc)
+		ci.createClusterIdentity(kc)
+	}
 	ci.createCredential(kc)
 
 	return &ci
@@ -203,20 +223,19 @@ func (ci *ClusterIdentity) createSecret(kc *kubeclient.KubeClient) {
 func (ci *ClusterIdentity) createCredential(kc *kubeclient.KubeClient) {
 	GinkgoHelper()
 
-	credName := fmt.Sprintf("%s-cred", ci.IdentityName)
-	By(fmt.Sprintf("creating Credential: %s", credName))
+	By(fmt.Sprintf("creating Credential: %s", ci.CredentialName))
 
 	cred := &unstructured.Unstructured{
 		Object: map[string]any{
 			"apiVersion": "hmc.mirantis.com/v1alpha1",
 			"kind":       "Credential",
 			"metadata": map[string]any{
-				"name":      credName,
+				"name":      ci.CredentialName,
 				"namespace": kc.Namespace,
 			},
 			"spec": map[string]any{
 				"identityRef": map[string]any{
-					"apiVersion": ci.GroupVersionResource.Group + "/" + ci.GroupVersionResource.Version,
+					"apiVersion": ci.GroupVersionResource.GroupVersion().String(),
 					"kind":       ci.Kind,
 					"name":       ci.IdentityName,
 					"namespace":  kc.Namespace,
@@ -252,3 +271,28 @@ func (ci *ClusterIdentity) createClusterIdentity(kc *kubeclient.KubeClient) {
 
 	kc.CreateOrUpdateUnstructuredObject(ci.GroupVersionResource, id, ci.Namespaced)
 }
+
+func (ci *ClusterIdentity) WaitForValidCredential(kc *kubeclient.KubeClient) {
+	GinkgoHelper()
+
+	By(fmt.Sprintf("waiting for %s credential to be ready", ci.CredentialName))
+
+	ctx := context.Background()
+
+	Eventually(func() error {
+		cred, err := kc.GetCredential(ctx, ci.CredentialName)
+		if err != nil {
+			return fmt.Errorf("failed to get credntial: %w", err)
+		}
+
+		ready, found, err := unstructured.NestedBool(cred.Object, "status", "ready")
+		if !found {
+			return fmt.Errorf("failed to get ready status: %w", err)
+		}
+		if !ready {
+			_, _ = fmt.Fprintf(GinkgoWriter, "credential is not ready, retrying...\n")
+			return fmt.Errorf("credential is not ready: %s", ci.GroupVersionResource.String())
+		}
+		return nil
+	}).WithTimeout(time.Minute).WithPolling(5 * time.Second).Should(Succeed())
+}

test/e2e/clusterdeployment/constants.go

@@ -47,4 +47,8 @@ const (
 	EnvVarAzureSubscription    = "AZURE_SUBSCRIPTION"
 	EnvVarAzureClusterIdentity = "AZURE_CLUSTER_IDENTITY"
 	EnvVarAzureRegion          = "AZURE_REGION"
+
+	// Adopted
+	EnvVarAdoptedKubeconfigPath = "KUBECONFIG_DATA_PATH"
+	EnvVarAdoptedCredential     = "ADOPTED_CREDENTIAL"
 )

test/e2e/clusterdeployment/providervalidator.go

@@ -67,13 +67,19 @@ func NewProviderValidator(template Template, clusterName string, action Validati
 			resourceOrder = append(resourceOrder, "ccm")
 		case TemplateAzureStandaloneCP, TemplateVSphereStandaloneCP:
 			delete(resourcesToValidate, "csi-driver")
+
+		case TemplateAdoptedCluster:
+			resourcesToValidate = map[string]resourceValidationFunc{
+				"sveltoscluster": validateSveltosCluster,
+			}
 		}
 	} else {
 		resourcesToValidate = map[string]resourceValidationFunc{
 			"clusters":           validateClusterDeleted,
 			"machinedeployments": validateMachineDeploymentsDeleted,
 			"control-planes":     validateK0sControlPlanesDeleted,
 		}
+
 		resourceOrder = []string{"clusters", "machinedeployments", "control-planes"}
 	}
 

test/e2e/clusterdeployment/resources/adopted-cluster.yaml.tpl

@@ -0,0 +1,16 @@
+apiVersion: hmc.mirantis.com/v1alpha1
+kind: ClusterDeployment
+metadata:
+  name: ${CLUSTER_DEPLOYMENT_NAME}
+  namespace: ${NAMESPACE}
+spec:
+  template: adopted-cluster-0-0-1
+  credential: ${ADOPTED_CREDENTIAL}
+  config: {}
+  services:
+    - template: kyverno-3-2-6
+      name: kyverno
+      namespace: kyverno
+    - template: ingress-nginx-4-11-0
+      name: ingress-nginx
+      namespace: ingress-nginx

test/e2e/clusterdeployment/validate_deployed.go

@@ -273,3 +273,22 @@ func validateCCM(ctx context.Context, kc *kubeclient.KubeClient, clusterName str
 
 	return fmt.Errorf("%s Service does not yet have an external hostname", service.Name)
 }
+
+// validateSveltosCluster validates that the sveltos cluster is ready
+func validateSveltosCluster(ctx context.Context, kc *kubeclient.KubeClient, clusterName string) error {
+	sveltosCluster, err := kc.GetSveltosCluster(ctx, clusterName)
+	if err != nil {
+		return fmt.Errorf("error getting sveltos cluster: %v", err)
+	}
+
+	ready, found, err := unstructured.NestedBool(sveltosCluster.Object, "status", "ready")
+	if err != nil {
+		return fmt.Errorf("error checking sveltos cluster ready: %v", err)
+	}
+
+	if !found || !ready {
+		return fmt.Errorf("sveltos cluster %s is not ready", clusterName)
+	}
+
+	return nil
+}

test/e2e/e2e_suite_test.go

@@ -47,7 +47,6 @@ func TestE2E(t *testing.T) {
 
 var _ = BeforeSuite(func() {
 	GinkgoT().Setenv(clusterdeployment.EnvVarNamespace, internalutils.DefaultSystemNamespace)
-
 	By("building and deploying the controller-manager")
 	cmd := exec.Command("make", "kind-deploy")
 	_, err := utils.Run(cmd)
@@ -66,6 +65,15 @@ var _ = BeforeSuite(func() {
 		}
 		return nil
 	}).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed())
+
+	Eventually(func() error {
+		err = clusterdeployment.ValidateClusterTemplates(context.Background(), kc)
+		if err != nil {
+			_, _ = fmt.Fprintf(GinkgoWriter, "cluster template validation failed: %v\n", err)
+			return err
+		}
+		return nil
+	}).WithTimeout(15 * time.Minute).WithPolling(10 * time.Second).Should(Succeed())
 })
 
 var _ = AfterSuite(func() {

test/e2e/kubeclient/kubeclient.go

@@ -56,7 +56,7 @@ func NewFromLocal(namespace string) *KubeClient {
 // the kubeconfig from secret it needs an existing kubeclient.
 func (kc *KubeClient) NewFromCluster(ctx context.Context, namespace, clusterName string) *KubeClient {
 	GinkgoHelper()
-	return newKubeClient(kc.getKubeconfigSecretData(ctx, clusterName), namespace)
+	return newKubeClient(kc.GetKubeconfigSecretData(ctx, clusterName), namespace)
 }
 
 // WriteKubeconfig writes the kubeconfig for the given clusterName to the
@@ -65,7 +65,7 @@ func (kc *KubeClient) NewFromCluster(ctx context.Context, namespace, clusterName
 func (kc *KubeClient) WriteKubeconfig(ctx context.Context, clusterName string) (string, func() error) {
 	GinkgoHelper()
 
-	secretData := kc.getKubeconfigSecretData(ctx, clusterName)
+	secretData := kc.GetKubeconfigSecretData(ctx, clusterName)
 
 	dir, err := os.Getwd()
 	Expect(err).NotTo(HaveOccurred())
@@ -89,7 +89,7 @@ func (kc *KubeClient) WriteKubeconfig(ctx context.Context, clusterName string) (
 	return path, deleteFunc
 }
 
-func (kc *KubeClient) getKubeconfigSecretData(ctx context.Context, clusterName string) []byte {
+func (kc *KubeClient) GetKubeconfigSecretData(ctx context.Context, clusterName string) []byte {
 	GinkgoHelper()
 
 	secret, err := kc.Client.CoreV1().Secrets(kc.Namespace).Get(ctx, clusterName+"-kubeconfig", metav1.GetOptions{})
@@ -279,3 +279,48 @@ func (kc *KubeClient) ListK0sControlPlanes(
 		Resource: "k0scontrolplanes",
 	}, clusterName)
 }
+
+func (kc *KubeClient) ListClusterTemplates(ctx context.Context) ([]unstructured.Unstructured, error) {
+	client := kc.GetDynamicClient(schema.GroupVersionResource{
+		Group:    "hmc.mirantis.com",
+		Version:  "v1alpha1",
+		Resource: "clustertemplates",
+	}, true)
+
+	resources, err := client.List(ctx, metav1.ListOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("failed to list cluster templates")
+	}
+
+	return resources.Items, nil
+}
+
+func (kc *KubeClient) GetCredential(ctx context.Context, name string) (*unstructured.Unstructured, error) {
+	client := kc.GetDynamicClient(schema.GroupVersionResource{
+		Group:    "hmc.mirantis.com",
+		Version:  "v1alpha1",
+		Resource: "credentials",
+	}, true)
+
+	credential, err := client.Get(ctx, name, metav1.GetOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("failed to get credential %s: %w", name, err)
+	}
+
+	return credential, nil
+}
+
+func (kc *KubeClient) GetSveltosCluster(ctx context.Context, name string) (*unstructured.Unstructured, error) {
+	client := kc.GetDynamicClient(schema.GroupVersionResource{
+		Group:    "lib.projectsveltos.io",
+		Version:  "v1beta1",
+		Resource: "sveltosclusters",
+	}, true)
+
+	sveltosCluster, err := client.Get(ctx, name, metav1.GetOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("failed to get sveltos cluster %s: %w", name, err)
+	}
+
+	return sveltosCluster, nil
+}