Validate available upgrades for managed clusters (#391)

* Set availableUpgrades in ManagedCluster status
* Validate availability of the ManagedCluster update
* Reflect available upgrades in cluster status: addressed comments
* Several watcher improvements: events, code enhancements
* Changed availableUpgrades type into []string

api/v1alpha1/common.go

@@ -71,7 +71,15 @@ func SetupIndexers(ctx context.Context, mgr ctrl.Manager) error {
 		return err
 	}
 
-	return SetupManagedClusterServicesIndexer(ctx, mgr)
+	if err := SetupManagedClusterServicesIndexer(ctx, mgr); err != nil {
+		return err
+	}
+
+	if err := SetupClusterTemplateChainIndexer(ctx, mgr); err != nil {
+		return err
+	}
+
+	return SetupServiceTemplateChainIndexer(ctx, mgr)
 }
 
 const TemplateKey = ".spec.template"
@@ -121,3 +129,32 @@ func ExtractServiceTemplateName(rawObj client.Object) []string {
 
 	return templates
 }
+
+const SupportedTemplateKey = ".spec.supportedTemplates[].Name"
+
+func SetupClusterTemplateChainIndexer(ctx context.Context, mgr ctrl.Manager) error {
+	return mgr.GetFieldIndexer().IndexField(ctx, &ClusterTemplateChain{}, SupportedTemplateKey, ExtractSupportedTemplatesNames)
+}
+
+func SetupServiceTemplateChainIndexer(ctx context.Context, mgr ctrl.Manager) error {
+	return mgr.GetFieldIndexer().IndexField(ctx, &ServiceTemplateChain{}, SupportedTemplateKey, ExtractSupportedTemplatesNames)
+}
+
+func ExtractSupportedTemplatesNames(rawObj client.Object) []string {
+	chainSpec := TemplateChainSpec{}
+	switch chain := rawObj.(type) {
+	case *ClusterTemplateChain:
+		chainSpec = chain.Spec
+	case *ServiceTemplateChain:
+		chainSpec = chain.Spec
+	default:
+		return nil
+	}
+
+	supportedTemplates := make([]string, 0, len(chainSpec.SupportedTemplates))
+	for _, t := range chainSpec.SupportedTemplates {
+		supportedTemplates = append(supportedTemplates, t.Name)
+	}
+
+	return supportedTemplates
+}

api/v1alpha1/managedcluster_types.go

@@ -95,8 +95,13 @@ type ManagedClusterStatus struct {
 	// Currently compatible exact Kubernetes version of the cluster. Being set only if
 	// provided by the corresponding ClusterTemplate.
 	KubernetesVersion string `json:"k8sVersion,omitempty"`
-	// Conditions contains details for the current state of the ManagedCluster
+	// Conditions contains details for the current state of the ManagedCluster.
 	Conditions []metav1.Condition `json:"conditions,omitempty"`
+
+	// AvailableUpgrades is the list of ClusterTemplate names to which
+	// this cluster can be upgraded. It can be an empty array, which means no upgrades are
+	// available.
+	AvailableUpgrades []string `json:"availableUpgrades,omitempty"`
 	// ObservedGeneration is the last observed generation.
 	ObservedGeneration int64 `json:"observedGeneration,omitempty"`
 }

internal/controller/managedcluster_controller.go

@@ -46,9 +46,12 @@ import (
 	capz "sigs.k8s.io/cluster-api-provider-azure/api/v1beta1"
 	capv "sigs.k8s.io/cluster-api-provider-vsphere/apis/v1beta1"
 	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/builder"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
+	"sigs.k8s.io/controller-runtime/pkg/event"
 	"sigs.k8s.io/controller-runtime/pkg/handler"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
 	"sigs.k8s.io/controller-runtime/pkg/reconcile"
 	"sigs.k8s.io/yaml"
 
@@ -180,11 +183,12 @@ func (r *ManagedClusterReconciler) Update(ctx context.Context, managedCluster *h
 		managedCluster.InitConditions()
 	}
 
+	template := &hmc.ClusterTemplate{}
+
 	defer func() {
-		err = errors.Join(err, r.updateStatus(ctx, managedCluster))
+		err = errors.Join(err, r.updateStatus(ctx, managedCluster, template))
 	}()
 
-	template := &hmc.ClusterTemplate{}
 	templateRef := client.ObjectKey{Name: managedCluster.Spec.Template, Namespace: managedCluster.Namespace}
 	if err := r.Get(ctx, templateRef, template); err != nil {
 		l.Error(err, "Failed to get Template")
@@ -419,7 +423,7 @@ func validateReleaseWithValues(ctx context.Context, actionConfig *action.Configu
 	return nil
 }
 
-func (r *ManagedClusterReconciler) updateStatus(ctx context.Context, managedCluster *hmc.ManagedCluster) error {
+func (r *ManagedClusterReconciler) updateStatus(ctx context.Context, managedCluster *hmc.ManagedCluster, template *hmc.ClusterTemplate) error {
 	managedCluster.Status.ObservedGeneration = managedCluster.Generation
 	warnings := ""
 	errs := ""
@@ -451,6 +455,11 @@ func (r *ManagedClusterReconciler) updateStatus(ctx context.Context, managedClus
 		condition.Message = errs
 	}
 	apimeta.SetStatusCondition(managedCluster.GetConditions(), condition)
+
+	err := r.setAvailableUpgrades(ctx, managedCluster, template)
+	if err != nil {
+		return errors.New("failed to set available upgrades")
+	}
 	if err := r.Status().Update(ctx, managedCluster); err != nil {
 		return fmt.Errorf("failed to update status for managedCluster %s/%s: %w", managedCluster.Namespace, managedCluster.Name, err)
 	}
@@ -997,6 +1006,38 @@ func setIdentityHelmValues(values *apiextensionsv1.JSON, idRef *corev1.ObjectRef
 	}, nil
 }
 
+func (r *ManagedClusterReconciler) setAvailableUpgrades(ctx context.Context, managedCluster *hmc.ManagedCluster, template *hmc.ClusterTemplate) error {
+	if template == nil {
+		return nil
+	}
+	chains := &hmc.ClusterTemplateChainList{}
+	err := r.List(ctx, chains,
+		client.InNamespace(template.Namespace),
+		client.MatchingFields{hmc.SupportedTemplateKey: template.GetName()},
+	)
+	if err != nil {
+		return err
+	}
+
+	availableUpgradesMap := make(map[string]hmc.AvailableUpgrade)
+	for _, chain := range chains.Items {
+		for _, supportedTemplate := range chain.Spec.SupportedTemplates {
+			if supportedTemplate.Name == template.Name {
+				for _, availableUpgrade := range supportedTemplate.AvailableUpgrades {
+					availableUpgradesMap[availableUpgrade.Name] = availableUpgrade
+				}
+			}
+		}
+	}
+	availableUpgrades := make([]string, 0, len(availableUpgradesMap))
+	for _, availableUpgrade := range availableUpgradesMap {
+		availableUpgrades = append(availableUpgrades, availableUpgrade.Name)
+	}
+
+	managedCluster.Status.AvailableUpgrades = availableUpgrades
+	return nil
+}
+
 // SetupWithManager sets up the controller with the Manager.
 func (r *ManagedClusterReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	return ctrl.NewControllerManagedBy(mgr).
@@ -1019,5 +1060,37 @@ func (r *ManagedClusterReconciler) SetupWithManager(mgr ctrl.Manager) error {
 				}
 			}),
 		).
+		Watches(&hmc.ClusterTemplateChain{},
+			handler.EnqueueRequestsFromMapFunc(func(ctx context.Context, o client.Object) []ctrl.Request {
+				chain, ok := o.(*hmc.ClusterTemplateChain)
+				if !ok {
+					return nil
+				}
+
+				var req []ctrl.Request
+				for _, template := range getTemplateNamesManagedByChain(chain) {
+					managedClusters := &hmc.ManagedClusterList{}
+					err := r.Client.List(ctx, managedClusters,
+						client.InNamespace(chain.Namespace),
+						client.MatchingFields{hmc.TemplateKey: template})
+					if err != nil {
+						return []ctrl.Request{}
+					}
+					for _, cluster := range managedClusters.Items {
+						req = append(req, ctrl.Request{
+							NamespacedName: client.ObjectKey{
+								Namespace: cluster.Namespace,
+								Name:      cluster.Name,
+							},
+						})
+					}
+				}
+				return req
+			}),
+			builder.WithPredicates(predicate.Funcs{
+				UpdateFunc:  func(event.UpdateEvent) bool { return false },
+				GenericFunc: func(event.GenericEvent) bool { return false },
+			}),
+		).
 		Complete(r)
 }

internal/controller/templatechain_controller.go

@@ -211,6 +211,14 @@ func getCurrentTemplates(ctx context.Context, cl client.Client, templateKind, sy
 	return systemTemplates, managedTemplates, nil
 }
 
+func getTemplateNamesManagedByChain(chain templateChain) []string {
+	result := make([]string, 0, len(chain.GetSpec().SupportedTemplates))
+	for _, template := range chain.GetSpec().SupportedTemplates {
+		result = append(result, template.Name)
+	}
+	return result
+}
+
 // SetupWithManager sets up the controller with the Manager.
 func (r *ClusterTemplateChainReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	return ctrl.NewControllerManagedBy(mgr).

internal/webhook/managedcluster_webhook.go

@@ -18,6 +18,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"slices"
 	"strings"
 
 	"github.com/Masterminds/semver/v3"
@@ -38,6 +39,8 @@ type ManagedClusterValidator struct {
 
 const invalidManagedClusterMsg = "the ManagedCluster is invalid"
 
+var errClusterUpgradeForbidden = errors.New("cluster upgrade is forbidden")
+
 func (v *ManagedClusterValidator) SetupWebhookWithManager(mgr ctrl.Manager) error {
 	v.Client = mgr.GetClient()
 	return ctrl.NewWebhookManagedBy(mgr).
@@ -89,12 +92,20 @@ func (v *ManagedClusterValidator) ValidateUpdate(ctx context.Context, oldObj, ne
 	if !ok {
 		return nil, apierrors.NewBadRequest(fmt.Sprintf("expected ManagedCluster but got a %T", newObj))
 	}
-	template, err := v.getManagedClusterTemplate(ctx, newManagedCluster.Namespace, newManagedCluster.Spec.Template)
+	oldTemplate := oldManagedCluster.Spec.Template
+	newTemplate := newManagedCluster.Spec.Template
+
+	template, err := v.getManagedClusterTemplate(ctx, newManagedCluster.Namespace, newTemplate)
 	if err != nil {
 		return nil, fmt.Errorf("%s: %v", invalidManagedClusterMsg, err)
 	}
 
-	if oldManagedCluster.Spec.Template != newManagedCluster.Spec.Template {
+	if oldTemplate != newTemplate {
+		if !slices.Contains(oldManagedCluster.Status.AvailableUpgrades, newTemplate) {
+			msg := fmt.Sprintf("Cluster can't be upgraded from %s to %s. This upgrade sequence is not allowed", oldTemplate, newTemplate)
+			return admission.Warnings{msg}, errClusterUpgradeForbidden
+		}
+
 		if err := isTemplateValid(template); err != nil {
 			return nil, fmt.Errorf("%s: %v", invalidManagedClusterMsg, err)
 		}

internal/webhook/managedcluster_webhook_test.go

@@ -257,6 +257,11 @@ func TestManagedClusterValidateCreate(t *testing.T) {
 }
 
 func TestManagedClusterValidateUpdate(t *testing.T) {
+	const (
+		upgradeTargetTemplateName  = "upgrade-target-template"
+		unmanagedByHMCTemplateName = "unmanaged-template"
+	)
+
 	g := NewWithT(t)
 
 	ctx := admission.NewContextWithRequest(context.Background(), admission.Request{
@@ -274,8 +279,11 @@ func TestManagedClusterValidateUpdate(t *testing.T) {
 		warnings          admission.Warnings
 	}{
 		{
-			name:              "should fail if the new cluster template was found but is invalid (some validation error)",
-			oldManagedCluster: managedcluster.NewManagedCluster(managedcluster.WithClusterTemplate(testTemplateName)),
+			name: "update spec.template: should fail if the new cluster template was found but is invalid (some validation error)",
+			oldManagedCluster: managedcluster.NewManagedCluster(
+				managedcluster.WithClusterTemplate(testTemplateName),
+				managedcluster.WithAvailableUpgrades([]string{newTemplateName}),
+			),
 			newManagedCluster: managedcluster.NewManagedCluster(managedcluster.WithClusterTemplate(newTemplateName)),
 			existingObjects: []runtime.Object{
 				mgmt,
@@ -290,7 +298,75 @@ func TestManagedClusterValidateUpdate(t *testing.T) {
 			err: "the ManagedCluster is invalid: the template is not valid: validation error example",
 		},
 		{
-			name: "should succeed if template is not changed",
+			name: "update spec.template: should fail if the template is not in the list of available",
+			oldManagedCluster: managedcluster.NewManagedCluster(
+				managedcluster.WithClusterTemplate(testTemplateName),
+				managedcluster.WithCredential(testCredentialName),
+				managedcluster.WithAvailableUpgrades([]string{}),
+			),
+			newManagedCluster: managedcluster.NewManagedCluster(
+				managedcluster.WithClusterTemplate(upgradeTargetTemplateName),
+				managedcluster.WithCredential(testCredentialName),
+			),
+			existingObjects: []runtime.Object{
+				mgmt, cred,
+				template.NewClusterTemplate(
+					template.WithName(testTemplateName),
+					template.WithValidationStatus(v1alpha1.TemplateValidationStatus{Valid: true}),
+					template.WithProvidersStatus(v1alpha1.Providers{
+						"infrastructure-aws",
+						"control-plane-k0smotron",
+						"bootstrap-k0smotron",
+					}),
+				),
+				template.NewClusterTemplate(
+					template.WithName(upgradeTargetTemplateName),
+					template.WithValidationStatus(v1alpha1.TemplateValidationStatus{Valid: true}),
+					template.WithProvidersStatus(v1alpha1.Providers{
+						"infrastructure-aws",
+						"control-plane-k0smotron",
+						"bootstrap-k0smotron",
+					}),
+				),
+			},
+			warnings: admission.Warnings{fmt.Sprintf("Cluster can't be upgraded from %s to %s. This upgrade sequence is not allowed", testTemplateName, upgradeTargetTemplateName)},
+			err:      "cluster upgrade is forbidden",
+		},
+		{
+			name: "update spec.template: should succeed if the template is in the list of available",
+			oldManagedCluster: managedcluster.NewManagedCluster(
+				managedcluster.WithClusterTemplate(testTemplateName),
+				managedcluster.WithCredential(testCredentialName),
+				managedcluster.WithAvailableUpgrades([]string{newTemplateName}),
+			),
+			newManagedCluster: managedcluster.NewManagedCluster(
+				managedcluster.WithClusterTemplate(newTemplateName),
+				managedcluster.WithCredential(testCredentialName),
+			),
+			existingObjects: []runtime.Object{
+				mgmt, cred,
+				template.NewClusterTemplate(
+					template.WithName(testTemplateName),
+					template.WithValidationStatus(v1alpha1.TemplateValidationStatus{Valid: true}),
+					template.WithProvidersStatus(v1alpha1.Providers{
+						"infrastructure-aws",
+						"control-plane-k0smotron",
+						"bootstrap-k0smotron",
+					}),
+				),
+				template.NewClusterTemplate(
+					template.WithName(newTemplateName),
+					template.WithValidationStatus(v1alpha1.TemplateValidationStatus{Valid: true}),
+					template.WithProvidersStatus(v1alpha1.Providers{
+						"infrastructure-aws",
+						"control-plane-k0smotron",
+						"bootstrap-k0smotron",
+					}),
+				),
+			},
+		},
+		{
+			name: "should succeed if spec.template is not changed",
 			oldManagedCluster: managedcluster.NewManagedCluster(
 				managedcluster.WithClusterTemplate(testTemplateName),
 				managedcluster.WithConfig(`{"foo":"bar"}`),

templates/provider/hmc/templates/crds/hmc.mirantis.com_managedclusters.yaml

@@ -132,9 +132,17 @@ spec:
           status:
             description: ManagedClusterStatus defines the observed state of ManagedCluster
             properties:
+              availableUpgrades:
+                description: |-
+                  AvailableUpgrades is the list of ClusterTemplate names to which
+                  this cluster can be upgraded. It can be an empty array, which means no upgrades are
+                  available.
+                items:
+                  type: string
+                type: array
               conditions:
                 description: Conditions contains details for the current state of
-                  the ManagedCluster
+                  the ManagedCluster.
                 items:
                   description: Condition contains details for one aspect of the current
                     state of this API Resource.

test/objects/managedcluster/managedcluster.go

@@ -87,3 +87,9 @@ func WithCredential(credName string) Opt {
 		p.Spec.Credential = credName
 	}
 }
+
+func WithAvailableUpgrades(availableUpgrades []string) Opt {
+	return func(p *v1alpha1.ManagedCluster) {
+		p.Status.AvailableUpgrades = availableUpgrades
+	}
+}