From 52f1a92f9c2b50815b8f2f02209a7b2d96357836 Mon Sep 17 00:00:00 2001 From: vbadrina Date: Wed, 17 Jul 2024 09:16:30 +0530 Subject: [PATCH] feat: Enhance MirrorPeer and DRPolicy controllers for StorageClientType handling - Updated `MirrorPeerReconciler` to handle `StorageClientType` by adding conditions in `Reconcile` method. - Introduced `createStorageClusterPeer` function for creating `StorageClusterPeer` objects. - Added utility functions `fetchClientInfoConfigMap` and `getClientInfoFromConfigMap` for handling client info. - Modified `processManagedClusterAddon` to utilize new config structure. - Enhanced `DRPolicyReconciler` to skip certain operations based on `StorageClientType`. Signed-off-by: vbadrina --- addons/agent_mirrorpeer_controller.go | 2 +- api/v1alpha1/mirrorpeer_types.go | 4 +- ...icluster.odf.openshift.io_mirrorpeers.yaml | 1 - ...icluster.odf.openshift.io_mirrorpeers.yaml | 1 - controllers/drpolicy_controller.go | 5 + controllers/mirrorpeer_controller.go | 221 +++++++++++++++++- controllers/utils/configmap.go | 22 ++ controllers/utils/manifestwork.go | 45 ++++ controllers/utils/peer_ref.go | 4 + go.mod | 2 +- go.sum | 4 +- 11 files changed, 293 insertions(+), 18 deletions(-) create mode 100644 controllers/utils/configmap.go create mode 100644 controllers/utils/manifestwork.go diff --git a/addons/agent_mirrorpeer_controller.go b/addons/agent_mirrorpeer_controller.go index 9baa6ed9..206555f6 100644 --- a/addons/agent_mirrorpeer_controller.go +++ b/addons/agent_mirrorpeer_controller.go @@ -125,7 +125,7 @@ func (r *MirrorPeerReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, err } - if mirrorPeer.Spec.Type == multiclusterv1alpha1.Async { + if mirrorPeer.Spec.Type == multiclusterv1alpha1.Async && !utils.IsStorageClientType(mirrorPeer.Spec.Items) { clusterFSIDs := make(map[string]string) logger.Info("Fetching clusterFSIDs") err = r.fetchClusterFSIDs(ctx, &mirrorPeer, clusterFSIDs) diff --git a/api/v1alpha1/mirrorpeer_types.go b/api/v1alpha1/mirrorpeer_types.go index d66d23aa..f88d9b8c 100644 --- a/api/v1alpha1/mirrorpeer_types.go +++ b/api/v1alpha1/mirrorpeer_types.go @@ -35,7 +35,9 @@ const ( // StorageClusterRef holds a reference to a StorageCluster type StorageClusterRef struct { - Name string `json:"name"` + Name string `json:"name"` + + // +kubebuilder:validation:Optional Namespace string `json:"namespace"` } diff --git a/bundle/manifests/multicluster.odf.openshift.io_mirrorpeers.yaml b/bundle/manifests/multicluster.odf.openshift.io_mirrorpeers.yaml index ace7cd7a..94c31b67 100644 --- a/bundle/manifests/multicluster.odf.openshift.io_mirrorpeers.yaml +++ b/bundle/manifests/multicluster.odf.openshift.io_mirrorpeers.yaml @@ -60,7 +60,6 @@ spec: type: string required: - name - - namespace type: object required: - clusterName diff --git a/config/crd/bases/multicluster.odf.openshift.io_mirrorpeers.yaml b/config/crd/bases/multicluster.odf.openshift.io_mirrorpeers.yaml index 10f68bd1..1aa994dd 100644 --- a/config/crd/bases/multicluster.odf.openshift.io_mirrorpeers.yaml +++ b/config/crd/bases/multicluster.odf.openshift.io_mirrorpeers.yaml @@ -60,7 +60,6 @@ spec: type: string required: - name - - namespace type: object required: - clusterName diff --git a/controllers/drpolicy_controller.go b/controllers/drpolicy_controller.go index 173381a5..251c8362 100644 --- a/controllers/drpolicy_controller.go +++ b/controllers/drpolicy_controller.go @@ -113,6 +113,11 @@ func (r *DRPolicyReconciler) Reconcile(ctx context.Context, req ctrl.Request) (c return ctrl.Result{}, err } + if utils.IsStorageClientType(mirrorPeer.Spec.Items) { + logger.Info("MirrorPeer contains StorageClient reference. Skipping creation of VolumeReplicationClasses", "MirrorPeer", mirrorPeer.Name) + return ctrl.Result{}, nil + } + if mirrorPeer.Spec.Type == multiclusterv1alpha1.Async { clusterFSIDs := make(map[string]string) logger.Info("Fetching cluster FSIDs") diff --git a/controllers/mirrorpeer_controller.go b/controllers/mirrorpeer_controller.go index 33cc2a9f..2dbb3876 100644 --- a/controllers/mirrorpeer_controller.go +++ b/controllers/mirrorpeer_controller.go @@ -18,12 +18,15 @@ package controllers import ( "context" + "encoding/json" + "fmt" "log/slog" "os" "github.com/red-hat-storage/odf-multicluster-orchestrator/addons/setup" ramenv1alpha1 "github.com/ramendr/ramen/api/v1alpha1" + ocsv1 "github.com/red-hat-storage/ocs-operator/api/v4/v1" addons "github.com/red-hat-storage/odf-multicluster-orchestrator/addons" multiclusterv1alpha1 "github.com/red-hat-storage/odf-multicluster-orchestrator/api/v1alpha1" "github.com/red-hat-storage/odf-multicluster-orchestrator/controllers/utils" @@ -53,8 +56,11 @@ type MirrorPeerReconciler struct { Logger *slog.Logger } -const mirrorPeerFinalizer = "hub.multicluster.odf.openshift.io" -const spokeClusterRoleBindingName = "spoke-clusterrole-bindings" +const ( + mirrorPeerFinalizer = "hub.multicluster.odf.openshift.io" + spokeClusterRoleBindingName = "spoke-clusterrole-bindings" + ClientConfigMapKeyTemplate = "%s/%s" +) //+kubebuilder:rbac:groups=multicluster.odf.openshift.io,resources=mirrorpeers,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=multicluster.odf.openshift.io,resources=mirrorpeers/status,verbs=get;update;patch @@ -241,35 +247,228 @@ func (r *MirrorPeerReconciler) Reconcile(ctx context.Context, req ctrl.Request) return ctrl.Result{}, err } + if utils.IsStorageClientType(mirrorPeer.Spec.Items) { + result, err := createStorageClusterPeer(ctx, r.Client, logger, mirrorPeer) + if err != nil { + logger.Error("Failed to create StorageClusterPeer", "error", err) + return result, err + } + } return r.updateMirrorPeerStatus(ctx, mirrorPeer) } +func getKey(clusterName, clientName string) string { + return fmt.Sprintf(ClientConfigMapKeyTemplate, clusterName, clientName) +} + +func createStorageClusterPeer(ctx context.Context, client client.Client, logger *slog.Logger, mirrorPeer multiclusterv1alpha1.MirrorPeer) (ctrl.Result, error) { + logger = logger.With("MirrorPeer", mirrorPeer.Name) + clientInfoMap, err := fetchClientInfoConfigMap(ctx, client) + if err != nil { + if k8serrors.IsNotFound(err) { + logger.Info("Client info config map not found. Retrying request another time...") + return ctrl.Result{Requeue: true}, nil + } + return ctrl.Result{}, err + } + items := mirrorPeer.Spec.Items + clientInfo := make([]ClientInfo, 0) + + for _, item := range items { + ci, err := getClientInfoFromConfigMap(clientInfoMap.Data, getKey(item.ClusterName, item.StorageClusterRef.Name)) + if err != nil { + return ctrl.Result{}, err + } + clientInfo = append(clientInfo, ci) + } + + for i := range items { + var storageClusterPeerName string + var oppositeClient ClientInfo + currentClient := clientInfo[i] + // Provider A StorageClusterPeer contains info of Provider B endpoint and ticket, hence this + if i == 0 { + oppositeClient := clientInfo[1] + storageClusterPeerName = getStorageClusterPeerName(oppositeClient.ProviderInfo.ProviderManagedClusterName) + } else { + oppositeClient = clientInfo[0] + storageClusterPeerName = getStorageClusterPeerName(oppositeClient.ProviderInfo.ProviderManagedClusterName) + } + + // Provider B's onboarding token will be used for Provider A's StorageClusterPeer + onboardingToken, err := fetchOnboardingTicket(ctx, client, oppositeClient, mirrorPeer) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to fetch onboarding token for provider %s. err %w", oppositeClient.ProviderInfo.ProviderManagedClusterName, err) + } + storageClusterPeer := ocsv1.StorageClusterPeer{ + ObjectMeta: metav1.ObjectMeta{ + Name: storageClusterPeerName, + // This provider A namespace on which the Storage object exists + Namespace: currentClient.ProviderInfo.NamespacedName.Namespace, + }, + Spec: ocsv1.StorageClusterPeerSpec{ + RemoteCluster: ocsv1.RemoteClusterSpec{ + OnboardingTicket: onboardingToken, + ApiEndpoint: oppositeClient.ProviderInfo.StorageProviderEndpoint, + StorageClusterName: ocsv1.NamespacedName{Name: oppositeClient.ProviderInfo.NamespacedName.Name, Namespace: oppositeClient.ProviderInfo.NamespacedName.Namespace}, + }, + LocalCluster: ocsv1.LocalClusterSpec{Name: corev1.LocalObjectReference{Name: currentClient.ProviderInfo.NamespacedName.Name}}, + }, + } + storageClusterPeerJson, err := json.Marshal(storageClusterPeer) + if err != nil { + logger.Error("Failed to marshal StorageClusterPeer to JSON", "StorageClusterPeer", storageClusterPeerName) + return ctrl.Result{}, err + } + + ownerRef := metav1.OwnerReference{ + APIVersion: mirrorPeer.APIVersion, + Kind: mirrorPeer.Kind, + Name: mirrorPeer.Name, + UID: mirrorPeer.UID, + } + + // ManifestWork created for Provider A will be called storageclusterpeer-{ProviderA} since that is where Manifests will be applied + // Provider names are unique hence only 1 ManifestWork per ProviderCluster + manifestWorkName := fmt.Sprintf("storageclusterpeer-%s", currentClient.ProviderInfo.ProviderManagedClusterName) + + // The namespace of Provider A is where this ManifestWork will be created on the hub + namespace := currentClient.ProviderInfo.ProviderManagedClusterName + + operationResult, err := utils.CreateOrUpdateManifestWork(ctx, client, manifestWorkName, namespace, storageClusterPeerJson, ownerRef) + if err != nil { + return ctrl.Result{}, err + } + + logger.Info(fmt.Sprintf("ManifestWork was %s for StorageClusterPeer %s", operationResult, storageClusterPeerName)) + } + + return ctrl.Result{}, nil +} + +func fetchOnboardingTicket(ctx context.Context, client client.Client, clientInfo ClientInfo, mirrorPeer multiclusterv1alpha1.MirrorPeer) (string, error) { + secretName := string(mirrorPeer.GetUID()) + secretNamespace := clientInfo.ProviderInfo.ProviderManagedClusterName + tokenSecret, err := utils.FetchSecretWithName(ctx, client, types.NamespacedName{Name: secretName, Namespace: secretNamespace}) + if err != nil { + if k8serrors.IsNotFound(err) { + return "", fmt.Errorf("secret %s not found in namespace %s", secretName, secretNamespace) + } + return "", fmt.Errorf("failed to fetch secret %s in namespace %s", secretName, secretNamespace) + } + + tokenData, exists := tokenSecret.Data["storagecluster-peer-token"] + if !exists { + return "", fmt.Errorf("token data not found in secret %s", secretName) + } + return string(tokenData), nil +} + +func getStorageClusterPeerName(providerClusterName string) string { + // Provider A will have SCP named {ProviderB}-peer + return fmt.Sprintf("%s-peer", providerClusterName) +} + +func fetchClientInfoConfigMap(ctx context.Context, c client.Client) (*corev1.ConfigMap, error) { + currentNamespace := os.Getenv("POD_NAMESPACE") + if currentNamespace == "" { + return nil, fmt.Errorf("cannot detect the current namespace") + } + clientInfoMap, err := utils.FetchConfigMap(ctx, c, ClientInfoConfigMapName, currentNamespace) + if err != nil { + return nil, err + } + return clientInfoMap, nil +} + +type ManagedClusterAddonConfig struct { + // Namespace on the managedCluster where it will be deployed + InstallNamespace string + + // Name of the MCA + Name string + + // Namespace on the hub where MCA will be created, it represents the Managed cluster where the addons will be deployed + Namespace string +} + +// Helper function to extract and unmarshal ClientInfo from ConfigMap +func getClientInfoFromConfigMap(clientInfoMap map[string]string, key string) (ClientInfo, error) { + clientInfoJSON, ok := clientInfoMap[key] + if !ok { + return ClientInfo{}, fmt.Errorf("client info for %s not found in ConfigMap", key) + } + + var clientInfo ClientInfo + if err := json.Unmarshal([]byte(clientInfoJSON), &clientInfo); err != nil { + return ClientInfo{}, fmt.Errorf("failed to unmarshal client info for %s: %v", key, err) + } + + return clientInfo, nil +} + +func getConfig(ctx context.Context, c client.Client, mp multiclusterv1alpha1.MirrorPeer) ([]ManagedClusterAddonConfig, error) { + managedClusterAddonsConfig := make([]ManagedClusterAddonConfig, 0) + if utils.IsStorageClientType(mp.Spec.Items) { + clientInfoMap, err := fetchClientInfoConfigMap(ctx, c) + if err != nil { + return []ManagedClusterAddonConfig{}, err + } + for _, item := range mp.Spec.Items { + clientName := item.StorageClusterRef.Name + clientInfo, err := getClientInfoFromConfigMap(clientInfoMap.Data, getKey(item.ClusterName, clientName)) + if err != nil { + return []ManagedClusterAddonConfig{}, err + } + config := ManagedClusterAddonConfig{ + Name: setup.TokenExchangeName, + Namespace: clientInfo.ProviderInfo.ProviderManagedClusterName, + InstallNamespace: clientInfo.ProviderInfo.NamespacedName.Namespace, + } + managedClusterAddonsConfig = append(managedClusterAddonsConfig, config) + } + } else { + for _, item := range mp.Spec.Items { + managedClusterAddonsConfig = append(managedClusterAddonsConfig, ManagedClusterAddonConfig{ + Name: setup.TokenExchangeName, + Namespace: item.ClusterName, + InstallNamespace: item.StorageClusterRef.Namespace, + }) + } + } + return managedClusterAddonsConfig, nil +} + // processManagedClusterAddon creates an addon for the cluster management in all the peer refs, // the resources gets an owner ref of the mirrorpeer to let the garbage collector handle it if the mirrorpeer gets deleted func (r *MirrorPeerReconciler) processManagedClusterAddon(ctx context.Context, mirrorPeer multiclusterv1alpha1.MirrorPeer) error { logger := r.Logger.With("MirrorPeer", mirrorPeer.Name) logger.Info("Processing ManagedClusterAddons for MirrorPeer") - for _, item := range mirrorPeer.Spec.Items { - logger.Info("Handling ManagedClusterAddon for cluster", "ClusterName", item.ClusterName) + addonConfigs, err := getConfig(ctx, r.Client, mirrorPeer) + if err != nil { + return fmt.Errorf("failed to get managedclusteraddon config %w", err) + } + for _, config := range addonConfigs { + logger.Info("Handling ManagedClusterAddon for cluster", "ClusterName", config.Namespace) var managedClusterAddOn addonapiv1alpha1.ManagedClusterAddOn namespacedName := types.NamespacedName{ - Name: setup.TokenExchangeName, - Namespace: item.ClusterName, + Name: config.Name, + Namespace: config.Namespace, } err := r.Client.Get(ctx, namespacedName, &managedClusterAddOn) if err != nil { if k8serrors.IsNotFound(err) { - logger.Info("ManagedClusterAddon not found, will create a new one", "ClusterName", item.ClusterName) + logger.Info("ManagedClusterAddon not found, will create a new one", "ClusterName", config.Namespace) annotations := make(map[string]string) annotations[utils.DRModeAnnotationKey] = string(mirrorPeer.Spec.Type) managedClusterAddOn = addonapiv1alpha1.ManagedClusterAddOn{ ObjectMeta: metav1.ObjectMeta{ Name: setup.TokenExchangeName, - Namespace: item.ClusterName, + Namespace: config.Namespace, Annotations: annotations, }, } @@ -277,9 +476,9 @@ func (r *MirrorPeerReconciler) processManagedClusterAddon(ctx context.Context, m } _, err = controllerutil.CreateOrUpdate(ctx, r.Client, &managedClusterAddOn, func() error { - managedClusterAddOn.Spec.InstallNamespace = item.StorageClusterRef.Namespace + managedClusterAddOn.Spec.InstallNamespace = config.InstallNamespace if err := controllerutil.SetOwnerReference(&mirrorPeer, &managedClusterAddOn, r.Scheme); err != nil { - logger.Error("Failed to set owner reference on ManagedClusterAddon", "error", err, "ClusterName", item.ClusterName) + logger.Error("Failed to set owner reference on ManagedClusterAddon", "error", err, "ClusterName", config.Namespace) return err } return nil @@ -290,7 +489,7 @@ func (r *MirrorPeerReconciler) processManagedClusterAddon(ctx context.Context, m return err } - logger.Info("Successfully reconciled ManagedClusterAddOn", "ClusterName", item.ClusterName) + logger.Info("Successfully reconciled ManagedClusterAddOn", "ClusterName", config.Namespace) } logger.Info("Successfully processed all ManagedClusterAddons for MirrorPeer") diff --git a/controllers/utils/configmap.go b/controllers/utils/configmap.go new file mode 100644 index 00000000..9bd9c4c2 --- /dev/null +++ b/controllers/utils/configmap.go @@ -0,0 +1,22 @@ +package utils + +import ( + "context" + "fmt" + + corev1 "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +// FetchConfigMap fetches a ConfigMap with a given name from a given namespace +func FetchConfigMap(ctx context.Context, c client.Client, name, namespace string) (*corev1.ConfigMap, error) { + configMap := &corev1.ConfigMap{} + err := c.Get(ctx, client.ObjectKey{ + Name: name, + Namespace: namespace, + }, configMap) + if err != nil { + return nil, fmt.Errorf("failed to fetch ConfigMap %s in namespace %s: %v", name, namespace, err) + } + return configMap, nil +} diff --git a/controllers/utils/manifestwork.go b/controllers/utils/manifestwork.go new file mode 100644 index 00000000..ad640bab --- /dev/null +++ b/controllers/utils/manifestwork.go @@ -0,0 +1,45 @@ +package utils + +import ( + "context" + "fmt" + + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/apimachinery/pkg/runtime" + workv1 "open-cluster-management.io/api/work/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" +) + +func CreateOrUpdateManifestWork(ctx context.Context, c client.Client, name string, namespace string, objJson []byte, ownerRef metav1.OwnerReference) (controllerutil.OperationResult, error) { + mw := workv1.ManifestWork{ + ObjectMeta: metav1.ObjectMeta{ + Name: name, + Namespace: namespace, + OwnerReferences: []metav1.OwnerReference{ + ownerRef, + }, + }, + } + + operationResult, err := controllerutil.CreateOrUpdate(ctx, c, &mw, func() error { + mw.Spec = workv1.ManifestWorkSpec{ + Workload: workv1.ManifestsTemplate{ + Manifests: []workv1.Manifest{ + { + RawExtension: runtime.RawExtension{ + Raw: objJson, + }, + }, + }, + }, + } + return nil + }) + + if err != nil { + return operationResult, fmt.Errorf("failed to create and update ManifestWork %s for namespace %s. error %w", name, namespace, err) + } + + return operationResult, nil +} diff --git a/controllers/utils/peer_ref.go b/controllers/utils/peer_ref.go index 479e5ac9..e3d7d366 100644 --- a/controllers/utils/peer_ref.go +++ b/controllers/utils/peer_ref.go @@ -33,3 +33,7 @@ func GetPeerRefForSpokeCluster(mp *multiclusterv1alpha1.MirrorPeer, spokeCluster } return nil, fmt.Errorf("PeerRef for cluster %s under mirrorpeer %s not found", spokeClusterName, mp.Name) } + +func IsStorageClientType(peerRefs []multiclusterv1alpha1.PeerRef) bool { + return peerRefs[0].StorageClusterRef.Namespace == "" && peerRefs[1].StorageClusterRef.Namespace == "" +} diff --git a/go.mod b/go.mod index da4c2997..abe5c159 100644 --- a/go.mod +++ b/go.mod @@ -12,7 +12,7 @@ require ( github.com/openshift/api v0.0.0-20240701145226-44d00ee80f5e github.com/openshift/library-go v0.0.0-20240124134907-4dfbf6bc7b11 github.com/ramendr/ramen/api v0.0.0-20240409201920-10024cae3bfd - github.com/red-hat-storage/ocs-operator/api/v4 v4.0.0-20240717095253-b12449490cc8 + github.com/red-hat-storage/ocs-operator/api/v4 v4.0.0-20240723180620-bb238df8306b github.com/rook/rook/pkg/apis v0.0.0-20240701212738-8e25300ad55a github.com/spf13/cobra v1.8.0 github.com/stolostron/multicloud-operators-foundation v0.0.0-20220824091202-e9cd9710d009 diff --git a/go.sum b/go.sum index f03357d4..1150e264 100644 --- a/go.sum +++ b/go.sum @@ -733,8 +733,8 @@ github.com/prometheus/procfs v0.12.0/go.mod h1:pcuDEFsWDnvcgNzo4EEweacyhjeA9Zk3c github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/ramendr/ramen/api v0.0.0-20240409201920-10024cae3bfd h1:TsDaQqfb1BcR78JWXBmUyj6Qx4By5loUZ95CxmA/6zo= github.com/ramendr/ramen/api v0.0.0-20240409201920-10024cae3bfd/go.mod h1:PCb0ODjdi4eYuxY/nSw+/rQqmzkmBVqGNoDr2JXdlKE= -github.com/red-hat-storage/ocs-operator/api/v4 v4.0.0-20240717095253-b12449490cc8 h1:xEepsbAeXeBx4V2BXNONrfmnM6VkG3n3ZxMTHcEji8M= -github.com/red-hat-storage/ocs-operator/api/v4 v4.0.0-20240717095253-b12449490cc8/go.mod h1:qpUjv+0s8e8ZjrO7tqISZ70GovRGFUY7N8YRaF1AMZE= +github.com/red-hat-storage/ocs-operator/api/v4 v4.0.0-20240723180620-bb238df8306b h1:Jk87pQLx2FF49y6S6CM9CtLvbk/HSo4Z+dsBL5xF4nM= +github.com/red-hat-storage/ocs-operator/api/v4 v4.0.0-20240723180620-bb238df8306b/go.mod h1:qpUjv+0s8e8ZjrO7tqISZ70GovRGFUY7N8YRaF1AMZE= github.com/rogpeppe/fastuuid v0.0.0-20150106093220-6724a57986af/go.mod h1:XWv6SoW27p1b0cqNHllgS5HIMJraePCO15w5zCzIWYg= github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6LYCDYWNEvQ= github.com/rogpeppe/go-charset v0.0.0-20180617210344-2471d30d28b4/go.mod h1:qgYeAmZ5ZIpBWTGllZSQnw97Dj+woV0toclVaRGI8pc=