From 1085795b807d6417b98e25c36e20e8892691fd95 Mon Sep 17 00:00:00 2001 From: Rewant Soni Date: Thu, 24 Oct 2024 20:17:04 +0530 Subject: [PATCH] controllers: new controller for maintenance mode Signed-off-by: Rewant Soni --- api/v1alpha1/storageclient_types.go | 6 + api/v1alpha1/zz_generated.deepcopy.go | 16 ++ ...client-operator.clusterserviceversion.yaml | 27 ++- .../ocs.openshift.io_storageclients.yaml | 7 + cmd/main.go | 12 ++ .../ocs.openshift.io_storageclients.yaml | 7 + config/rbac/role.yaml | 25 +++ .../controller/maintenancemode_controller.go | 202 ++++++++++++++++++ .../controller/storageclaim_controller.go | 2 + .../controller/storageclient_controller.go | 21 +- .../api/v1alpha1/storageclient_types.go | 6 + .../api/v1alpha1/zz_generated.deepcopy.go | 16 ++ 12 files changed, 342 insertions(+), 5 deletions(-) create mode 100644 internal/controller/maintenancemode_controller.go diff --git a/api/v1alpha1/storageclient_types.go b/api/v1alpha1/storageclient_types.go index 6ca4d7c8..d4da3a02 100644 --- a/api/v1alpha1/storageclient_types.go +++ b/api/v1alpha1/storageclient_types.go @@ -46,10 +46,16 @@ type StorageClientSpec struct { OnboardingTicket string `json:"onboardingTicket"` } +type ProviderAttributes struct { + InMaintenanceMode bool `json:"inMaintenanceMode"` +} + // StorageClientStatus defines the observed state of StorageClient type StorageClientStatus struct { Phase storageClientPhase `json:"phase,omitempty"` + Provider ProviderAttributes `json:"providerAttributes,omitempty"` + // ConsumerID will hold the identity of this cluster inside the attached provider cluster ConsumerID string `json:"id,omitempty"` } diff --git a/api/v1alpha1/zz_generated.deepcopy.go b/api/v1alpha1/zz_generated.deepcopy.go index 9c3bdede..ba580975 100644 --- a/api/v1alpha1/zz_generated.deepcopy.go +++ b/api/v1alpha1/zz_generated.deepcopy.go @@ -24,6 +24,21 @@ import ( "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProviderAttributes) DeepCopyInto(out *ProviderAttributes) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProviderAttributes. +func (in *ProviderAttributes) DeepCopy() *ProviderAttributes { + if in == nil { + return nil + } + out := new(ProviderAttributes) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *StorageClaim) DeepCopyInto(out *StorageClaim) { *out = *in @@ -190,6 +205,7 @@ func (in *StorageClientSpec) DeepCopy() *StorageClientSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *StorageClientStatus) DeepCopyInto(out *StorageClientStatus) { *out = *in + out.Provider = in.Provider } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageClientStatus. diff --git a/bundle/manifests/ocs-client-operator.clusterserviceversion.yaml b/bundle/manifests/ocs-client-operator.clusterserviceversion.yaml index 00088b41..df98438e 100644 --- a/bundle/manifests/ocs-client-operator.clusterserviceversion.yaml +++ b/bundle/manifests/ocs-client-operator.clusterserviceversion.yaml @@ -7,7 +7,7 @@ metadata: categories: Storage console.openshift.io/plugins: '["odf-client-console"]' containerImage: quay.io/ocs-dev/ocs-client-operator:latest - createdAt: "2024-11-18T12:48:54Z" + createdAt: "2024-11-20T12:06:55Z" description: OpenShift Data Foundation client operator enables consumption of storage services from a remote centralized OpenShift Data Foundation provider cluster. @@ -326,6 +326,31 @@ spec: - list - update - watch + - apiGroups: + - ramendr.openshift.io + resources: + - maintenancemodes + verbs: + - create + - delete + - get + - list + - update + - watch + - apiGroups: + - ramendr.openshift.io + resources: + - maintenancemodes/finalizers + verbs: + - update + - apiGroups: + - ramendr.openshift.io + resources: + - maintenancemodes/status + verbs: + - get + - patch + - update - apiGroups: - security.openshift.io resources: diff --git a/bundle/manifests/ocs.openshift.io_storageclients.yaml b/bundle/manifests/ocs.openshift.io_storageclients.yaml index ef513ed9..c68bb554 100644 --- a/bundle/manifests/ocs.openshift.io_storageclients.yaml +++ b/bundle/manifests/ocs.openshift.io_storageclients.yaml @@ -67,6 +67,13 @@ spec: type: string phase: type: string + providerAttributes: + properties: + inMaintenanceMode: + type: boolean + required: + - inMaintenanceMode + type: object type: object type: object served: true diff --git a/cmd/main.go b/cmd/main.go index e348a777..0fc52ab9 100644 --- a/cmd/main.go +++ b/cmd/main.go @@ -174,6 +174,7 @@ func main() { Client: mgr.GetClient(), Scheme: mgr.GetScheme(), OperatorNamespace: utils.GetOperatorNamespace(), + AvailableCrds: availCrds, }).SetupWithManager(mgr); err != nil { setupLog.Error(err, "unable to create controller", "controller", "StorageClient") os.Exit(1) @@ -208,6 +209,17 @@ func main() { os.Exit(1) } + if availCrds[controller.MaintenanceModeCRDName] { + if err = (&controller.MaintenanceModeReconciler{ + Client: mgr.GetClient(), + Scheme: mgr.GetScheme(), + OperatorNamespace: utils.GetOperatorNamespace(), + }).SetupWithManager(mgr); err != nil { + setupLog.Error(err, "unable to create controller", "controller", "MaintenanceMode") + os.Exit(1) + } + } + setupLog.Info("starting manager") if err := mgr.Start(ctrl.SetupSignalHandler()); err != nil { setupLog.Error(err, "problem running manager") diff --git a/config/crd/bases/ocs.openshift.io_storageclients.yaml b/config/crd/bases/ocs.openshift.io_storageclients.yaml index 90b12081..ba3135a4 100644 --- a/config/crd/bases/ocs.openshift.io_storageclients.yaml +++ b/config/crd/bases/ocs.openshift.io_storageclients.yaml @@ -67,6 +67,13 @@ spec: type: string phase: type: string + providerAttributes: + properties: + inMaintenanceMode: + type: boolean + required: + - inMaintenanceMode + type: object type: object type: object served: true diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index 0099a921..915ccdf9 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -283,6 +283,31 @@ rules: - list - update - watch +- apiGroups: + - ramendr.openshift.io + resources: + - maintenancemodes + verbs: + - create + - delete + - get + - list + - update + - watch +- apiGroups: + - ramendr.openshift.io + resources: + - maintenancemodes/finalizers + verbs: + - update +- apiGroups: + - ramendr.openshift.io + resources: + - maintenancemodes/status + verbs: + - get + - patch + - update - apiGroups: - security.openshift.io resources: diff --git a/internal/controller/maintenancemode_controller.go b/internal/controller/maintenancemode_controller.go new file mode 100644 index 00000000..fa436ea0 --- /dev/null +++ b/internal/controller/maintenancemode_controller.go @@ -0,0 +1,202 @@ +package controller + +import ( + "context" + "fmt" + "k8s.io/apimachinery/pkg/api/meta" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "time" + + "github.com/red-hat-storage/ocs-client-operator/api/v1alpha1" + + "github.com/go-logr/logr" + ramenv1alpha1 "github.com/ramendr/ramen/api/v1alpha1" + providerclient "github.com/red-hat-storage/ocs-operator/services/provider/api/v4/client" + storagev1 "k8s.io/api/storage/v1" + kerrors "k8s.io/apimachinery/pkg/api/errors" + "k8s.io/apimachinery/pkg/runtime" + ctrl "sigs.k8s.io/controller-runtime" + "sigs.k8s.io/controller-runtime/pkg/builder" + "sigs.k8s.io/controller-runtime/pkg/client" + "sigs.k8s.io/controller-runtime/pkg/controller/controllerutil" + "sigs.k8s.io/controller-runtime/pkg/handler" + "sigs.k8s.io/controller-runtime/pkg/log" + "sigs.k8s.io/controller-runtime/pkg/predicate" + "sigs.k8s.io/controller-runtime/pkg/reconcile" +) + +const ( + maintenanceModeFinalizer = "ocs-client-operator.ocs.openshift.io/maintenance-mode" + ramenReplicationIDLabel = "ramendr.openshift.io/replicationID" + MaintenanceModeCRDName = "maintenancemodes.ramendr.openshift.io" +) + +// MaintenanceModeReconciler reconciles a ClusterVersion object +type MaintenanceModeReconciler struct { + client.Client + OperatorNamespace string + Scheme *runtime.Scheme + + log logr.Logger + ctx context.Context +} + +// SetupWithManager sets up the controller with the Manager. +func (r *MaintenanceModeReconciler) SetupWithManager(mgr ctrl.Manager) error { + generationChangePredicate := predicate.GenerationChangedPredicate{} + bldr := ctrl.NewControllerManagedBy(mgr). + For(&ramenv1alpha1.MaintenanceMode{}, builder.WithPredicates(generationChangePredicate)). + Watches(&storagev1.StorageClass{}, &handler.EnqueueRequestForObject{}, builder.WithPredicates(generationChangePredicate)). + Watches(&v1alpha1.StorageClient{}, &handler.EnqueueRequestForObject{}, builder.WithPredicates(generationChangePredicate)) + + return bldr.Complete(r) +} + +//+kubebuilder:rbac:groups=ramendr.openshift.io,resources=maintenancemodes,verbs=get;list;update;create;watch;delete +//+kubebuilder:rbac:groups=ramendr.openshift.io,resources=maintenancemodes/status,verbs=get;update;patch +//+kubebuilder:rbac:groups=ramendr.openshift.io,resources=maintenancemodes/finalizers,verbs=update +//+kubebuilder:rbac:groups=ocs.openshift.io,resources=storageclients,verbs=get;list;watch +//+kubebuilder:rbac:groups=storage.k8s.io,resources=storageclasses,verbs=get;list;watch + +func (r *MaintenanceModeReconciler) Reconcile(ctx context.Context, req ctrl.Request) (ctrl.Result, error) { + r.ctx = ctx + r.log = log.FromContext(ctx, "MaintenanceMode", req) + r.log.Info("Reconciling MaintenanceMode") + + maintenanceMode := &ramenv1alpha1.MaintenanceMode{} + maintenanceMode.Name = req.Name + if err := r.get(maintenanceMode); err != nil { + if kerrors.IsNotFound(err) { + r.log.Info("Maintenance Mode resource not found. Ignoring since object might be deleted.") + return reconcile.Result{}, nil + } + r.log.Error(err, "failed to get the Maintenance Mode") + return reconcile.Result{}, err + } + + // find the storageClass with the targetID + storageClassList := &storagev1.StorageClassList{} + err := r.list(storageClassList, client.MatchingLabels{ramenReplicationIDLabel: maintenanceMode.Spec.TargetID}) + if err != nil { + return ctrl.Result{}, err + } + if len(storageClassList.Items) != 1 { + return ctrl.Result{}, fmt.Errorf("failed to find storageClass for maintenance mode") + } + + storageClass := &storageClassList.Items[0] + + // find the storageClient + // TODO: when storageClass is owned by storageClient, move to find the Client by ownerRef + storageClient := &v1alpha1.StorageClient{} + storageClient.Name = storageClass.GetLabels()[storageClientLabel] + err = r.get(storageClient) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to get the storage client: %v", err) + } + + result, reconcileErr := r.reconcileStates(maintenanceMode, storageClient) + + // Apply status changes to the StorageClient + statusErr := r.Client.Status().Update(ctx, maintenanceMode) + if statusErr != nil { + r.log.Error(statusErr, "Failed to update MaintenanceMode status.") + } + if reconcileErr != nil { + err = reconcileErr + } else if statusErr != nil { + err = statusErr + } + return result, err +} + +func (r *MaintenanceModeReconciler) reconcileStates(maintenanceMode *ramenv1alpha1.MaintenanceMode, storageClient *v1alpha1.StorageClient) (ctrl.Result, error) { + + if !maintenanceMode.GetDeletionTimestamp().IsZero() { + // deletion phase + if storageClient.Status.Provider.InMaintenanceMode { + providerClient, err := providerclient.NewProviderClient( + r.ctx, + storageClient.Spec.StorageProviderEndpoint, + 10*time.Second, + ) + if err != nil { + return reconcile.Result{}, fmt.Errorf( + "failed to create provider client with endpoint %v: %v", + storageClient.Spec.StorageProviderEndpoint, + err, + ) + } + // Close client-side connections. + defer providerClient.Close() + + _, err = providerClient.RequestMaintenanceMode(r.ctx, storageClient.Status.ConsumerID, false) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to disble maintenance mode: %v", err) + } + return ctrl.Result{Requeue: true}, nil + } + + // remove finalizer + if controllerutil.RemoveFinalizer(maintenanceMode, maintenanceModeFinalizer) { + if err := r.Client.Update(r.ctx, maintenanceMode); err != nil { + return ctrl.Result{}, err + } + r.log.Info("finalizer removed successfully") + } + return ctrl.Result{}, nil + } + + // ensure finalizer + if controllerutil.AddFinalizer(maintenanceMode, maintenanceModeFinalizer) { + r.log.Info("finalizer missing on the Maintenance Mode resource, adding...") + if err := r.Client.Update(r.ctx, maintenanceMode); err != nil { + return ctrl.Result{}, err + } + } + + if !storageClient.Status.Provider.InMaintenanceMode { + providerClient, err := providerclient.NewProviderClient( + r.ctx, + storageClient.Spec.StorageProviderEndpoint, + 10*time.Second, + ) + if err != nil { + return reconcile.Result{}, fmt.Errorf( + "failed to create provider client with endpoint %v: %v", + storageClient.Spec.StorageProviderEndpoint, + err, + ) + } + // Close client-side connections. + defer providerClient.Close() + + _, err = providerClient.RequestMaintenanceMode(r.ctx, storageClient.Status.ConsumerID, true) + if err != nil { + return ctrl.Result{}, fmt.Errorf("failed to Request maintenance mode: %v", err) + } + maintenanceMode.Status.State = ramenv1alpha1.MModeStateUnknown + return ctrl.Result{Requeue: true}, nil + } + + // Ramen reads the State and Conditions in order to determine that the MaintenanceMode is Completed + maintenanceMode.Status.State = ramenv1alpha1.MModeStateCompleted + maintenanceMode.Status.ObservedGeneration = maintenanceMode.Generation + meta.SetStatusCondition(&maintenanceMode.Status.Conditions, + metav1.Condition{ + Type: string(ramenv1alpha1.MModeConditionFailoverActivated), + ObservedGeneration: maintenanceMode.Generation, + Reason: string(ramenv1alpha1.MModeStateCompleted), + Status: metav1.ConditionTrue, + }, + ) + return ctrl.Result{}, nil +} + +func (r *MaintenanceModeReconciler) get(obj client.Object, opts ...client.GetOption) error { + return r.Get(r.ctx, client.ObjectKeyFromObject(obj), obj, opts...) +} + +func (r *MaintenanceModeReconciler) list(obj client.ObjectList, opts ...client.ListOption) error { + return r.List(r.ctx, obj, opts...) +} diff --git a/internal/controller/storageclaim_controller.go b/internal/controller/storageclaim_controller.go index fe52bf50..b516dcac 100644 --- a/internal/controller/storageclaim_controller.go +++ b/internal/controller/storageclaim_controller.go @@ -54,6 +54,7 @@ import ( const ( storageClaimFinalizer = "storageclaim.ocs.openshift.io" storageClaimAnnotation = "ocs.openshift.io/storageclaim" + storageClientLabel = "ocs.openshift.io/storageclient" keyRotationAnnotation = "keyrotation.csiaddons.openshift.io/schedule" pvClusterIDIndexName = "index:persistentVolumeClusterID" @@ -391,6 +392,7 @@ func (r *StorageClaimReconciler) reconcilePhases() (reconcile.Result, error) { } err = utils.CreateOrReplace(r.ctx, r.Client, storageClass, func() error { utils.AddLabels(storageClass, resource.Labels) + utils.AddLabel(storageClass, storageClientLabel, r.storageClient.Name) utils.AddAnnotation(storageClass, storageClaimAnnotation, r.storageClaim.Name) storageClass.Parameters = data return nil diff --git a/internal/controller/storageclient_controller.go b/internal/controller/storageclient_controller.go index 35754e99..416f7947 100644 --- a/internal/controller/storageclient_controller.go +++ b/internal/controller/storageclient_controller.go @@ -20,6 +20,7 @@ import ( "context" "encoding/json" "fmt" + extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" "os" "strings" "time" @@ -72,12 +73,13 @@ const ( type StorageClientReconciler struct { ctx context.Context client.Client - Log klog.Logger - Scheme *runtime.Scheme + Log klog.Logger + Scheme *runtime.Scheme + AvailableCrds map[string]bool + OperatorNamespace string + recorder *utils.EventReporter storageClient *v1alpha1.StorageClient - - OperatorNamespace string } // SetupWithManager sets up the controller with the Manager. @@ -127,6 +129,15 @@ func (r *StorageClientReconciler) Reconcile(ctx context.Context, req ctrl.Reques r.Log = log.FromContext(ctx, "StorageClient", req) r.Log.Info("Reconciling StorageClient") + crd := &metav1.PartialObjectMetadata{} + crd.SetGroupVersionKind(extv1.SchemeGroupVersion.WithKind("CustomResourceDefinition")) + crd.Name = MaintenanceModeCRDName + if err := r.Client.Get(ctx, client.ObjectKeyFromObject(crd), crd); client.IgnoreNotFound(err) != nil { + r.Log.Error(err, "Failed to get CRD", "CRD", crd.Name) + return reconcile.Result{}, err + } + utils.AssertEqual(r.AvailableCrds[crd.Name], crd.UID != "", utils.ExitCodeThatShouldRestartTheProcess) + r.storageClient = &v1alpha1.StorageClient{} r.storageClient.Name = req.Name if err = r.get(r.storageClient); err != nil { @@ -207,6 +218,8 @@ func (r *StorageClientReconciler) reconcilePhases() (ctrl.Result, error) { return reconcile.Result{}, fmt.Errorf("failed to get StorageConfig: %v", err) } + r.storageClient.Status.Provider.InMaintenanceMode = storageClientResponse.SystemAttributes.SystemInMaintenanceMode + if res, err := r.reconcileClientStatusReporterJob(); err != nil { return res, err } diff --git a/vendor/github.com/red-hat-storage/ocs-client-operator/api/v1alpha1/storageclient_types.go b/vendor/github.com/red-hat-storage/ocs-client-operator/api/v1alpha1/storageclient_types.go index 6ca4d7c8..d4da3a02 100644 --- a/vendor/github.com/red-hat-storage/ocs-client-operator/api/v1alpha1/storageclient_types.go +++ b/vendor/github.com/red-hat-storage/ocs-client-operator/api/v1alpha1/storageclient_types.go @@ -46,10 +46,16 @@ type StorageClientSpec struct { OnboardingTicket string `json:"onboardingTicket"` } +type ProviderAttributes struct { + InMaintenanceMode bool `json:"inMaintenanceMode"` +} + // StorageClientStatus defines the observed state of StorageClient type StorageClientStatus struct { Phase storageClientPhase `json:"phase,omitempty"` + Provider ProviderAttributes `json:"providerAttributes,omitempty"` + // ConsumerID will hold the identity of this cluster inside the attached provider cluster ConsumerID string `json:"id,omitempty"` } diff --git a/vendor/github.com/red-hat-storage/ocs-client-operator/api/v1alpha1/zz_generated.deepcopy.go b/vendor/github.com/red-hat-storage/ocs-client-operator/api/v1alpha1/zz_generated.deepcopy.go index 9c3bdede..ba580975 100644 --- a/vendor/github.com/red-hat-storage/ocs-client-operator/api/v1alpha1/zz_generated.deepcopy.go +++ b/vendor/github.com/red-hat-storage/ocs-client-operator/api/v1alpha1/zz_generated.deepcopy.go @@ -24,6 +24,21 @@ import ( "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *ProviderAttributes) DeepCopyInto(out *ProviderAttributes) { + *out = *in +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new ProviderAttributes. +func (in *ProviderAttributes) DeepCopy() *ProviderAttributes { + if in == nil { + return nil + } + out := new(ProviderAttributes) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *StorageClaim) DeepCopyInto(out *StorageClaim) { *out = *in @@ -190,6 +205,7 @@ func (in *StorageClientSpec) DeepCopy() *StorageClientSpec { // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *StorageClientStatus) DeepCopyInto(out *StorageClientStatus) { *out = *in + out.Provider = in.Provider } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StorageClientStatus.