diff --git a/.gitignore b/.gitignore index 0656b090b..d2670ed72 100644 --- a/.gitignore +++ b/.gitignore @@ -25,3 +25,6 @@ test/e2e/*.log *.swp *.swo *~ + +# Vendoring directory +vendor diff --git a/api/v1alpha1/managedcluster_types.go b/api/v1alpha1/managedcluster_types.go index e95be1d93..158f4c7a7 100644 --- a/api/v1alpha1/managedcluster_types.go +++ b/api/v1alpha1/managedcluster_types.go @@ -22,11 +22,14 @@ import ( ) const ( + BlockingFinalizer = "hmc.mirantis.com/cleanup" ManagedClusterFinalizer = "hmc.mirantis.com/managed-cluster" FluxHelmChartNameKey = "helm.toolkit.fluxcd.io/name" HMCManagedLabelKey = "hmc.mirantis.com/managed" HMCManagedLabelValue = "true" + + ClusterNameLabelKey = "cluster.x-k8s.io/cluster-name" ) const ( diff --git a/internal/controller/managedcluster_controller.go b/internal/controller/managedcluster_controller.go index 5f3c5ea03..4ceb922ca 100644 --- a/internal/controller/managedcluster_controller.go +++ b/internal/controller/managedcluster_controller.go @@ -20,8 +20,6 @@ import ( "fmt" "time" - "k8s.io/apimachinery/pkg/labels" - hcv2 "github.com/fluxcd/helm-controller/api/v2" fluxmeta "github.com/fluxcd/pkg/apis/meta" fluxconditions "github.com/fluxcd/pkg/runtime/conditions" @@ -33,6 +31,7 @@ import ( apimeta "k8s.io/apimachinery/pkg/api/meta" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/apis/meta/v1/unstructured" + "k8s.io/apimachinery/pkg/labels" "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/apimachinery/pkg/types" @@ -59,6 +58,30 @@ type ManagedClusterReconciler struct { SystemNamespace string } +type providerSchema struct { + machine, cluster schema.GroupVersionKind +} + +var ( + gvkAWSCluster = schema.GroupVersionKind{ + Group: "infrastructure.cluster.x-k8s.io", + Version: "v1beta2", + Kind: "awscluster", + } + + gvkAzureCluster = schema.GroupVersionKind{ + Group: "infrastructure.cluster.x-k8s.io", + Version: "v1beta1", + Kind: "azurecluster", + } + + gvkMachine = schema.GroupVersionKind{ + Group: "cluster.x-k8s.io", + Version: "v1beta1", + Kind: "machine", + } +) + // Reconcile is part of the main kubernetes reconciliation loop which aims to // move the current state of the cluster closer to the desired state. func (r *ManagedClusterReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { @@ -385,14 +408,111 @@ func (r *ManagedClusterReconciler) Delete(ctx context.Context, l logr.Logger, ma } return ctrl.Result{}, err } + err = helm.DeleteHelmRelease(ctx, r.Client, managedCluster.Name, managedCluster.Namespace) if err != nil { return ctrl.Result{}, err } + + err = r.releaseCluster(ctx, managedCluster.Namespace, managedCluster.Name, managedCluster.Spec.Template) + if err != nil { + return ctrl.Result{}, err + } + l.Info("HelmRelease still exists, retrying") return ctrl.Result{RequeueAfter: 10 * time.Second}, nil } +func (r *ManagedClusterReconciler) releaseCluster(ctx context.Context, namespace, name, templateName string) error { + providers, err := r.getProviders(ctx, templateName) + if err != nil { + return err + } + + providerGVKs := map[string]providerSchema{ + "aws": {machine: gvkMachine, cluster: gvkAWSCluster}, + "azure": {machine: gvkMachine, cluster: gvkAzureCluster}, + } + + // Associate the provider with it's GVK + for _, provider := range providers { + gvk, ok := providerGVKs[provider] + if !ok { + continue + } + + cluster, err := r.getCluster(ctx, namespace, name, gvk.cluster) + if err != nil { + return err + } + + found, err := r.machinesAvailable(ctx, namespace, cluster.Name, gvk.machine) + if err != nil { + return err + } + + if !found { + return r.removeClusterFinalizer(ctx, cluster) + } + } + + return nil +} + +func (r *ManagedClusterReconciler) getProviders(ctx context.Context, templateName string) ([]string, error) { + template := &hmc.ClusterTemplate{} + templateRef := types.NamespacedName{Name: templateName, Namespace: r.SystemNamespace} + if err := r.Get(ctx, templateRef, template); err != nil { + log.FromContext(ctx).Error(err, "Failed to get Template", "templateName", templateName) + return nil, err + } + return template.Status.Providers.InfrastructureProviders, nil +} + +func (r *ManagedClusterReconciler) getCluster(ctx context.Context, namespace, name string, gvk schema.GroupVersionKind) (*metav1.PartialObjectMetadata, error) { + opts := &client.ListOptions{ + LabelSelector: labels.SelectorFromSet(map[string]string{hmc.FluxHelmChartNameKey: name}), + Namespace: namespace, + } + itemsList := &metav1.PartialObjectMetadataList{} + itemsList.SetGroupVersionKind(gvk) + if err := r.Client.List(ctx, itemsList, opts); err != nil { + return nil, err + } + if len(itemsList.Items) == 0 { + return nil, fmt.Errorf("%s with name %s was not found", gvk.Kind, name) + } + + return &itemsList.Items[0], nil +} + +func (r *ManagedClusterReconciler) removeClusterFinalizer(ctx context.Context, cluster *metav1.PartialObjectMetadata) error { + originalCluster := *cluster + finalizersUpdated := controllerutil.RemoveFinalizer(cluster, hmc.BlockingFinalizer) + if finalizersUpdated { + log.FromContext(ctx).Info("Allow to stop cluster", "finalizer", hmc.BlockingFinalizer) + if err := r.Client.Patch(ctx, cluster, client.MergeFrom(&originalCluster)); err != nil { + return fmt.Errorf("failed to patch cluster %s/%s: %w", cluster.Namespace, cluster.Name, err) + } + } + + return nil +} + +func (r *ManagedClusterReconciler) machinesAvailable(ctx context.Context, namespace, clusterName string, gvk schema.GroupVersionKind) (bool, error) { + opts := &client.ListOptions{ + LabelSelector: labels.SelectorFromSet(map[string]string{hmc.ClusterNameLabelKey: clusterName}), + Namespace: namespace, + Limit: 1, + } + itemsList := &metav1.PartialObjectMetadataList{} + itemsList.SetGroupVersionKind(gvk) + if err := r.Client.List(ctx, itemsList, opts); err != nil { + return false, err + } + return len(itemsList.Items) != 0, nil +} + // SetupWithManager sets up the controller with the Manager. func (r *ManagedClusterReconciler) SetupWithManager(mgr ctrl.Manager) error { return ctrl.NewControllerManagedBy(mgr). diff --git a/templates/cluster/aws-hosted-cp/templates/awscluster.yaml b/templates/cluster/aws-hosted-cp/templates/awscluster.yaml index c6dc896fc..ea52af698 100644 --- a/templates/cluster/aws-hosted-cp/templates/awscluster.yaml +++ b/templates/cluster/aws-hosted-cp/templates/awscluster.yaml @@ -4,6 +4,8 @@ metadata: name: {{ include "cluster.name" . }} annotations: cluster.x-k8s.io/managed-by: k0smotron + finalizers: + - hmc.mirantis.com/cleanup spec: region: {{ .Values.region }} # identityRef: diff --git a/templates/cluster/azure-hosted-cp/templates/azurecluster.yaml b/templates/cluster/azure-hosted-cp/templates/azurecluster.yaml index 07cb15845..b1734116d 100644 --- a/templates/cluster/azure-hosted-cp/templates/azurecluster.yaml +++ b/templates/cluster/azure-hosted-cp/templates/azurecluster.yaml @@ -4,6 +4,8 @@ metadata: name: {{ include "cluster.name" . }} annotations: cluster.x-k8s.io/managed-by: k0smotron + finalizers: + - hmc.mirantis.com/cleanup spec: identityRef: kind: AzureClusterIdentity diff --git a/templates/provider/hmc/templates/rbac/roles.yaml b/templates/provider/hmc/templates/rbac/roles.yaml index 96c613214..bb0fa7794 100644 --- a/templates/provider/hmc/templates/rbac/roles.yaml +++ b/templates/provider/hmc/templates/rbac/roles.yaml @@ -138,6 +138,24 @@ rules: - certificates verbs: - create +- apiGroups: + - infrastructure.cluster.x-k8s.io + resources: + - awsclusters + - azureclusters + verbs: + - get + - list + - patch + - watch +- apiGroups: + - cluster.x-k8s.io + resources: + - machines + verbs: + - get + - list + - watch --- apiVersion: rbac.authorization.k8s.io/v1 kind: Role