Add additional validations for CSI, CCM

* Support obtaining a KubeClient from a managed cluster.
* Generate a random name to use as clusterName for the test,
  this will enable the ability to use cloud-nuke to nuke
  specifically named resources.

Signed-off-by: Kyle Squizzato <[email protected]>

go.mod

@@ -9,13 +9,14 @@ require (
 	github.com/fluxcd/pkg/runtime v0.49.0
 	github.com/fluxcd/source-controller/api v1.3.0
 	github.com/go-logr/logr v1.4.2
+	github.com/google/uuid v1.6.0
 	github.com/hashicorp/go-retryablehttp v0.7.7
 	github.com/onsi/ginkgo/v2 v2.20.0
 	github.com/onsi/gomega v1.34.1
 	github.com/opencontainers/go-digest v1.0.1-0.20231025023718-d50d2fec9c98
 	github.com/pkg/errors v0.9.1
 	github.com/segmentio/analytics-go v3.1.0+incompatible
-	gopkg.in/yaml.v2 v2.4.0
+	gopkg.in/yaml.v3 v3.0.1
 	helm.sh/helm/v3 v3.15.4
 	k8s.io/api v0.31.0
 	k8s.io/apiextensions-apiserver v0.31.0
@@ -78,7 +79,6 @@ require (
 	github.com/google/gofuzz v1.2.0 // indirect
 	github.com/google/pprof v0.0.0-20240727154555-813a5fbdbec8 // indirect
 	github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510 // indirect
-	github.com/google/uuid v1.6.0 // indirect
 	github.com/gorilla/mux v1.8.0 // indirect
 	github.com/gorilla/websocket v1.5.1 // indirect
 	github.com/gosuri/uitable v0.0.4 // indirect
@@ -155,7 +155,7 @@ require (
 	google.golang.org/protobuf v1.34.2 // indirect
 	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/yaml.v3 v3.0.1 // indirect
+	gopkg.in/yaml.v2 v2.4.0 // indirect
 	k8s.io/apiserver v0.31.0 // indirect
 	k8s.io/cli-runtime v0.31.0 // indirect
 	k8s.io/component-base v0.31.0 // indirect

test/e2e/e2e_test.go

@@ -48,7 +48,7 @@ var _ = Describe("controller", Ordered, func() {
 
 	Context("Operator", func() {
 		It("should run successfully", func() {
-			kc, err := kubeclient.New(namespace)
+			kc, err := kubeclient.NewFromLocal(namespace)
 			ExpectWithOffset(1, err).NotTo(HaveOccurred())
 
 			By("validating that the controller-manager pod is running as expected")
@@ -94,25 +94,32 @@ var _ = Describe("controller", Ordered, func() {
 
 		BeforeAll(func() {
 			By("ensuring AWS credentials are set")
-			kc, err = kubeclient.New(namespace)
+			kc, err = kubeclient.NewFromLocal(namespace)
 			ExpectWithOffset(2, err).NotTo(HaveOccurred())
 			ExpectWithOffset(2, kc.CreateAWSCredentialsKubeSecret(context.Background())).To(Succeed())
 		})
 
 		AfterAll(func() {
-			// TODO: Purge the AWS resources
+			// Purge the AWS resources, the AfterAll for the controller will
+			// clean up the management cluster.
 		})
 
 		It("should work with an AWS provider", func() {
 			By("using the aws-standalone-cp template")
-			ExpectWithOffset(2, utils.ConfigureDeploymentConfig()).To(Succeed())
+			clusterName, err := utils.ConfigureDeploymentConfig(utils.ProviderAWS, utils.AWSStandaloneCPTemplate)
+			ExpectWithOffset(2, err).NotTo(HaveOccurred())
 
 			cmd := exec.Command("make", "dev-aws-apply")
-			_, err := utils.Run(cmd)
+			_, err = utils.Run(cmd)
 			ExpectWithOffset(2, err).NotTo(HaveOccurred())
 			EventuallyWithOffset(2, func() error {
-				return verifyProviderDeployed(context.Background(), kc, "aws-dev")
+				return verifyProviderDeployed(context.Background(), kc, clusterName)
 			}(), 30*time.Minute, 10*time.Second).Should(Succeed())
+
+			By("using the aws-hosted-cp template")
+			// TODO: Use the standalone control plane resources to craft a hosted
+			// control plane and test it.
+
 		})
 	})
 })

test/e2e/validate_provider.go

@@ -22,22 +22,34 @@ import (
 
 	"github.com/Mirantis/hmc/test/kubeclient"
 	"github.com/Mirantis/hmc/test/utils"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/api/resource"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime/schema"
+	"k8s.io/apimachinery/pkg/util/intstr"
 )
 
+// resourceValidationFunc is intended to validate a specific kubernetes
+// resource.  It is meant to be used in conjunction with an Eventually block,
+// however, in some cases it may be necessary to end the Eventually block early
+// if the resource will never reach a ready state, in these instances Ginkgo's
+// Fail function should be used.
 type resourceValidationFunc func(context.Context, *kubeclient.KubeClient, string) error
 
 // verifyProviderDeployed is a provider-agnostic verification that checks for
-// the presence of cluster, machine and k0scontrolplane resources and their
-// underlying status conditions.  It is meant to be used in conjunction with
-// an Eventually block.
+// the presence of specific resources in the cluster using
+// resourceValidationFuncs and clusterValidationFuncs.
 func verifyProviderDeployed(ctx context.Context, kc *kubeclient.KubeClient, clusterName string) error {
+	// Sequentially validate each resource type, only returning the first error
+	// as to not move on to the next resource type until the first is resolved.
 	for _, resourceValidator := range []resourceValidationFunc{
 		validateClusters,
 		validateMachines,
 		validateK0sControlPlanes,
+		validateCSIDriver,
+		validateCCM,
 	} {
 		// XXX: Once we validate for the first time should we move the
 		// validation out and consider it "done"?  Or is there a possibility
@@ -81,7 +93,7 @@ func validateNameAndStatus(ctx context.Context, kc *kubeclient.KubeClient,
 	for _, item := range list.Items {
 		phase, _, err := unstructured.NestedString(item.Object, "status", "phase")
 		if err != nil {
-			Fail(fmt.Sprintf("failed to get phase for %s: %v", item.GetName(), err))
+			return fmt.Errorf("failed to get status.phase for %s: %v", item.GetName(), err)
 		}
 
 		if phase == "Deleting" {
@@ -100,14 +112,6 @@ func validateNameAndStatus(ctx context.Context, kc *kubeclient.KubeClient,
 	return nil
 }
 
-type k0smotronControlPlaneStatus struct {
-	// Ready denotes that the control plane is ready
-	Ready                       bool `json:"ready"`
-	ControlPlaneReady           bool `json:"controlPlaneReady"`
-	Inititalized                bool `json:"initialized"`
-	ExternalManagedControlPlane bool `json:"externalManagedControlPlane"`
-}
-
 func validateK0sControlPlanes(ctx context.Context, kc *kubeclient.KubeClient, clusterName string) error {
 	k0sControlPlaneClient, err := kc.GetDynamicClient(schema.GroupVersionResource{
 		Group:    "controlplane.cluster.x-k8s.io",
@@ -130,9 +134,9 @@ func validateK0sControlPlanes(ctx context.Context, kc *kubeclient.KubeClient, cl
 
 		objKind, objName := utils.ObjKindName(&controlPlane)
 
-		// k0smotron does not use the metav1.Condition type for status
+		// k0s does not use the metav1.Condition type for status
 		// conditions, instead it uses a custom type so we can't use
-		// ValidateConditionsTrue here.
+		// ValidateConditionsTrue here, instead we'll check for "ready: true".
 		conditions, found, err := unstructured.NestedFieldCopy(controlPlane.Object, "status", "conditions")
 		if !found {
 			return fmt.Errorf("no status conditions found for %s: %s", objKind, objName)
@@ -141,15 +145,113 @@ func validateK0sControlPlanes(ctx context.Context, kc *kubeclient.KubeClient, cl
 			return fmt.Errorf("failed to get status conditions for %s: %s: %w", objKind, objName, err)
 		}
 
-		c, ok := conditions.(k0smotronControlPlaneStatus)
+		c, ok := conditions.(map[string]interface{})
 		if !ok {
-			return fmt.Errorf("expected K0sControlPlane condition to be type K0smotronControlPlaneStatus, got: %T", conditions)
+			return fmt.Errorf("expected K0sControlPlane condition to be type map[string]interface{}, got: %T", conditions)
 		}
 
-		if !c.Ready {
-			return fmt.Errorf("K0sControlPlane %s is not ready, status: %+v", controlPlane.GetName(), c)
+		if c["ready"] != "true" {
+			return fmt.Errorf("K0sControlPlane %s is not ready, status: %v", controlPlane.GetName(), conditions)
 		}
 	}
 
 	return nil
 }
+
+// validateCSIDriver validates that the provider CSI driver is functioning
+// by creating a PVC and verifying it enters "Bound" status.
+func validateCSIDriver(ctx context.Context, kc *kubeclient.KubeClient, clusterName string) error {
+	clusterKC, err := kc.NewFromCluster(ctx, "default", clusterName)
+	if err != nil {
+		Fail(fmt.Sprintf("failed to create KubeClient for managed cluster %s: %v", clusterName, err))
+	}
+
+	pvcName := clusterName + "-test-pvc"
+
+	_, err = clusterKC.Client.CoreV1().PersistentVolumeClaims(clusterKC.Namespace).
+		Create(ctx, &corev1.PersistentVolumeClaim{
+			ObjectMeta: metav1.ObjectMeta{
+				Name: pvcName,
+			},
+			Spec: corev1.PersistentVolumeClaimSpec{
+				AccessModes: []corev1.PersistentVolumeAccessMode{
+					corev1.ReadWriteOnce,
+				},
+				Resources: corev1.VolumeResourceRequirements{
+					Requests: corev1.ResourceList{
+						corev1.ResourceStorage: resource.MustParse("1Gi"),
+					},
+				},
+			},
+		}, metav1.CreateOptions{})
+	if err != nil {
+		// Since these resourceValidationFuncs are intended to be used in
+		// Eventually we should ensure a follow-up PVCreate is a no-op.
+		if !apierrors.IsAlreadyExists(err) {
+			return fmt.Errorf("failed to create test PVC: %w", err)
+		}
+	}
+
+	// Verify the PVC enters "Bound" status.
+	pvc, err := clusterKC.Client.CoreV1().PersistentVolumeClaims(clusterKC.Namespace).
+		Get(ctx, pvcName, metav1.GetOptions{})
+	if err != nil {
+		return fmt.Errorf("failed to get test PVC: %w", err)
+	}
+
+	if pvc.Status.Phase == corev1.ClaimBound {
+		return nil
+	}
+
+	return fmt.Errorf("%s PersistentVolume not yet 'Bound', current phase: %q", pvcName, pvc.Status.Phase)
+}
+
+// validateCCM validates that the provider's cloud controller manager is
+// functional by creating a LoadBalancer service and verifying it is assigned
+// an external IP.
+func validateCCM(ctx context.Context, kc *kubeclient.KubeClient, clusterName string) error {
+	clusterKC, err := kc.NewFromCluster(ctx, "default", clusterName)
+	if err != nil {
+		Fail(fmt.Sprintf("failed to create KubeClient for managed cluster %s: %v", clusterName, err))
+	}
+
+	_, err = clusterKC.Client.CoreV1().Services(clusterKC.Namespace).Create(ctx, &corev1.Service{
+		ObjectMeta: metav1.ObjectMeta{
+			Name: clusterName + "-test-service",
+		},
+		Spec: corev1.ServiceSpec{
+			Selector: map[string]string{
+				"some": "selector",
+			},
+			Ports: []corev1.ServicePort{
+				{
+					Port:       8765,
+					TargetPort: intstr.FromInt(9376),
+				},
+			},
+			Type: corev1.ServiceTypeLoadBalancer,
+		},
+	}, metav1.CreateOptions{})
+	if err != nil {
+		// Since these resourceValidationFuncs are intended to be used in
+		// Eventually we should ensure a follow-up ServiceCreate is a no-op.
+		if !apierrors.IsAlreadyExists(err) {
+			return fmt.Errorf("failed to create test Service: %w", err)
+		}
+	}
+
+	// Verify the Service is assigned an external IP.
+	service, err := clusterKC.Client.CoreV1().Services(clusterKC.Namespace).
+		Get(ctx, clusterName+"-test-service", metav1.GetOptions{})
+	if err != nil {
+		return fmt.Errorf("failed to get test Service: %w", err)
+	}
+
+	for _, i := range service.Status.LoadBalancer.Ingress {
+		if i.Hostname != "" {
+			return nil
+		}
+	}
+
+	return fmt.Errorf("%s Service does not yet have an external hostname", service.Name)
+}

test/kubeclient/kubeclient.go

@@ -31,8 +31,36 @@ type KubeClient struct {
 	Config         *rest.Config
 }
 
-// getKubeConfig returns the kubeconfig file content.
-func getKubeConfig() ([]byte, error) {
+// NewFromLocal creates a new instance of KubeClient from a given namespace
+// using the locally found kubeconfig.
+func NewFromLocal(namespace string) (*KubeClient, error) {
+	configBytes, err := getLocalKubeConfig()
+	if err != nil {
+		return nil, fmt.Errorf("failed to get local kubeconfig: %w", err)
+	}
+
+	return new(configBytes, namespace)
+}
+
+// NewFromCluster creates a new KubeClient using the kubeconfig stored in the
+// secret affiliated with the given clusterName.  Since it relies on fetching
+// the kubeconfig from secret it needs an existing kubeclient.
+func (kc *KubeClient) NewFromCluster(ctx context.Context, namespace, clusterName string) (*KubeClient, error) {
+	secret, err := kc.Client.CoreV1().Secrets(kc.Namespace).Get(ctx, clusterName+"-kubeconfig", metav1.GetOptions{})
+	if err != nil {
+		return nil, fmt.Errorf("failed to get cluster: %q kubeconfig secret: %w", clusterName, err)
+	}
+
+	secretData, ok := secret.Data["value"]
+	if !ok {
+		return nil, fmt.Errorf("kubeconfig secret %q has no 'value' key", clusterName)
+	}
+
+	return new(secretData, namespace)
+}
+
+// getLocalKubeConfig returns the kubeconfig file content.
+func getLocalKubeConfig() ([]byte, error) {
 	homeDir, err := os.UserHomeDir()
 	if err != nil {
 		return nil, fmt.Errorf("failed to get user home directory: %w", err)
@@ -53,13 +81,9 @@ func getKubeConfig() ([]byte, error) {
 	return configBytes, nil
 }
 
-// New creates a new instance of KubeClient from a given namespace.
-func New(namespace string) (*KubeClient, error) {
-	configBytes, err := getKubeConfig()
-	if err != nil {
-		return nil, fmt.Errorf("failed to get kubeconfig: %w", err)
-	}
-
+// new creates a new instance of KubeClient from a given namespace using
+// the local kubeconfig.
+func new(configBytes []byte, namespace string) (*KubeClient, error) {
 	config, err := clientcmd.RESTConfigFromKubeConfig(configBytes)
 	if err != nil {
 		return nil, fmt.Errorf("failed to parse kubeconfig: %w", err)