Merge pull request red-hat-storage#253 from rewantsoni/avail-crd

Available CRDs check feature

Signed-off-by: rchikatw <[email protected]>

Dockerfile

@@ -29,6 +29,7 @@ WORKDIR /
 COPY --from=builder /workspace/bin/manager .
 COPY --from=builder /workspace/bin/status-reporter .
 COPY --from=builder /workspace/bin/deployment-guard .
+COPY --from=builder /workspace/hack/entrypoint.sh entrypoint
 USER 65532:65532
 
-ENTRYPOINT ["/manager"]
+ENTRYPOINT ["/entrypoint"]

bundle/manifests/ocs-client-operator.clusterserviceversion.yaml

@@ -7,7 +7,7 @@ metadata:
     categories: Storage
     console.openshift.io/plugins: '["odf-client-console"]'
     containerImage: quay.io/ocs-dev/ocs-client-operator:latest
-    createdAt: "2024-09-24T09:48:54Z"
+    createdAt: "2024-10-07T10:43:06Z"
     description: OpenShift Data Foundation client operator enables consumption of
       storage services from a remote centralized OpenShift Data Foundation provider
       cluster.
@@ -774,7 +774,7 @@ spec:
                 - --leader-elect
                 - --console-port=9001
                 command:
-                - /manager
+                - /entrypoint
                 env:
                 - name: OPERATOR_NAMESPACE
                   valueFrom:

cmd/main.go

@@ -17,7 +17,9 @@ limitations under the License.
 package main
 
 import (
+	"context"
 	"flag"
+	"fmt"
 	"os"
 
 	apiv1alpha1 "github.com/red-hat-storage/ocs-client-operator/api/v1alpha1"
@@ -37,9 +39,11 @@ import (
 	secv1 "github.com/openshift/api/security/v1"
 	opv1a1 "github.com/operator-framework/api/pkg/operators/v1alpha1"
 	monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
+	ramenv1alpha1 "github.com/ramendr/ramen/api/v1alpha1"
 	admrv1 "k8s.io/api/admissionregistration/v1"
 	appsv1 "k8s.io/api/apps/v1"
 	extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/fields"
 	"k8s.io/apimachinery/pkg/runtime"
 	utilruntime "k8s.io/apimachinery/pkg/util/runtime"
@@ -75,6 +79,7 @@ func init() {
 	utilruntime.Must(quotav1.AddToScheme(scheme))
 	utilruntime.Must(csiopv1a1.AddToScheme(scheme))
 	utilruntime.Must(nbapis.AddToScheme(scheme))
+	utilruntime.Must(ramenv1alpha1.AddToScheme(scheme))
 	//+kubebuilder:scaffold:scheme
 }
 
@@ -129,11 +134,6 @@ func main() {
 		os.Exit(1)
 	}
 
-	if err != nil {
-		setupLog.Error(err, "Unable to get Client")
-		os.Exit(1)
-	}
-
 	// set namespace
 	err = utils.ValidateOperatorNamespace()
 	if err != nil {
@@ -147,6 +147,22 @@ func main() {
 		os.Exit(1)
 	}
 
+	// apiclient.New() returns a client without cache. cache is not initialized before mgr.Start()
+	// we need this because we need to watch for CRDs the operator is dependent on
+	apiClient, err := client.New(mgr.GetConfig(), client.Options{
+		Scheme: mgr.GetScheme(),
+	})
+	if err != nil {
+		setupLog.Error(err, "Unable to get Client")
+		os.Exit(1)
+	}
+
+	availCrds, err := getAvailableCRDNames(context.Background(), apiClient)
+	if err != nil {
+		setupLog.Error(err, "Unable get a list of available CRD names")
+		os.Exit(1)
+	}
+
 	setupLog.Info("setting up webhook server")
 	hookServer := mgr.GetWebhookServer()
 
@@ -172,6 +188,7 @@ func main() {
 		Client:            mgr.GetClient(),
 		Scheme:            mgr.GetScheme(),
 		OperatorNamespace: utils.GetOperatorNamespace(),
+		AvailableCrds:     availCrds,
 	}).SetupWithManager(mgr); err != nil {
 		setupLog.Error(err, "unable to create controller", "controller", "StorageClaim")
 		os.Exit(1)
@@ -202,3 +219,17 @@ func main() {
 		os.Exit(1)
 	}
 }
+
+func getAvailableCRDNames(ctx context.Context, cl client.Client) (map[string]bool, error) {
+	crdExist := map[string]bool{}
+	crdList := &metav1.PartialObjectMetadataList{}
+	crdList.SetGroupVersionKind(extv1.SchemeGroupVersion.WithKind("CustomResourceDefinitionList"))
+	if err := cl.List(ctx, crdList); err != nil {
+		return nil, fmt.Errorf("error listing CRDs, %v", err)
+	}
+	// Iterate over the list and populate the map
+	for i := range crdList.Items {
+		crdExist[crdList.Items[i].Name] = true
+	}
+	return crdExist, nil
+}

config/manager/manager.yaml

@@ -41,7 +41,7 @@ spec:
               fieldPath: metadata.namespace
       containers:
       - command:
-        - /manager
+        - /entrypoint
         args:
         - --leader-elect
         image: controller:latest

go.mod

@@ -27,8 +27,9 @@ require (
 	github.com/operator-framework/api v0.27.0
 	github.com/pkg/errors v0.9.1
 	github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring v0.76.0
+	github.com/ramendr/ramen/api v0.0.0-20241001141243-29d6f22ad237
 	github.com/red-hat-storage/ocs-client-operator/api v0.0.0-00010101000000-000000000000
-	github.com/red-hat-storage/ocs-operator/services/provider/api/v4 v4.0.0-20240917115204-741b9d6f263d
+	github.com/red-hat-storage/ocs-operator/services/provider/api/v4 v4.0.0-20241015071140-98c8184c6eec
 	github.com/stretchr/testify v1.9.0
 	google.golang.org/grpc v1.66.0
 	gopkg.in/yaml.v2 v2.4.0
@@ -48,6 +49,7 @@ require (
 	github.com/openshift/custom-resource-status v1.1.3-0.20220503160415-f2fdb4999d87 // indirect
 	github.com/x448/float16 v0.8.4 // indirect
 	gopkg.in/evanphx/json-patch.v4 v4.12.0 // indirect
+	k8s.io/component-base v0.31.0 // indirect
 	sigs.k8s.io/container-object-storage-interface-api v0.1.0 // indirect
 )
 

go.sum

@@ -323,8 +323,10 @@ github.com/prometheus/common v0.57.0 h1:Ro/rKjwdq9mZn1K5QPctzh+MA4Lp0BuYk5ZZEVho
 github.com/prometheus/common v0.57.0/go.mod h1:7uRPFSUTbfZWsJ7MHY56sqt7hLQu3bxXHDnNhl8E9qI=
 github.com/prometheus/procfs v0.15.1 h1:YagwOFzUgYfKKHX6Dr+sHT7km/hxC76UB0learggepc=
 github.com/prometheus/procfs v0.15.1/go.mod h1:fB45yRUv8NstnjriLhBQLuOUt+WW4BsoGhij/e3PBqk=
-github.com/red-hat-storage/ocs-operator/services/provider/api/v4 v4.0.0-20240917115204-741b9d6f263d h1:RK/zCM6xRwyeJ06u6xSLEwl5Q1g/6EZRQmSgzbqleT0=
-github.com/red-hat-storage/ocs-operator/services/provider/api/v4 v4.0.0-20240917115204-741b9d6f263d/go.mod h1:t9GJk69TGXABBF8fFPB+ImpbA9mJyRS86wW6+Qn8xHo=
+github.com/ramendr/ramen/api v0.0.0-20241001141243-29d6f22ad237 h1:ig6ePD0yopC5Qi5BRmhsIsKaOkdsGXTSmG3HTYIpquo=
+github.com/ramendr/ramen/api v0.0.0-20241001141243-29d6f22ad237/go.mod h1:nO6VM/+PEhcPGyFIQJdhY6ip822cA61PAy/s6IjenAA=
+github.com/red-hat-storage/ocs-operator/services/provider/api/v4 v4.0.0-20241015071140-98c8184c6eec h1:M64BdwKMV3jKxcRsZiaGbRKsvlbhRGVZgcb4V/MFeWk=
+github.com/red-hat-storage/ocs-operator/services/provider/api/v4 v4.0.0-20241015071140-98c8184c6eec/go.mod h1:t9GJk69TGXABBF8fFPB+ImpbA9mJyRS86wW6+Qn8xHo=
 github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4=
 github.com/rogpeppe/go-internal v1.12.0 h1:exVL4IDcn6na9z1rAb56Vxr+CgyK3nn3O+epU5NdKM8=
 github.com/rogpeppe/go-internal v1.12.0/go.mod h1:E+RYuTGaKKdloAfM02xzb0FW3Paa99yedzYV+kq4uf4=
@@ -776,6 +778,8 @@ k8s.io/client-go v0.31.0 h1:QqEJzNjbN2Yv1H79SsS+SWnXkBgVu4Pj3CJQgbx0gI8=
 k8s.io/client-go v0.31.0/go.mod h1:Y9wvC76g4fLjmU0BA+rV+h2cncoadjvjjkkIGoTLcGU=
 k8s.io/code-generator v0.20.1/go.mod h1:UsqdF+VX4PU2g46NC2JRs4gc+IfrctnwHb76RNbWHJg=
 k8s.io/code-generator v0.23.3/go.mod h1:S0Q1JVA+kSzTI1oUvbKAxZY/DYbA/ZUb4Uknog12ETk=
+k8s.io/component-base v0.31.0 h1:/KIzGM5EvPNQcYgwq5NwoQBaOlVFrghoVGr8lG6vNRs=
+k8s.io/component-base v0.31.0/go.mod h1:TYVuzI1QmN4L5ItVdMSXKvH7/DtvIuas5/mm8YT3rTo=
 k8s.io/gengo v0.0.0-20200413195148-3a45101e95ac/go.mod h1:ezvh/TsK7cY6rbqRK0oQQ8IAqLxYwwyPxAX1Pzy0ii0=
 k8s.io/gengo v0.0.0-20201113003025-83324d819ded/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=
 k8s.io/gengo v0.0.0-20210813121822-485abfe95c7c/go.mod h1:FiNAH4ZV3gBg2Kwh89tzAEV2be7d5xI0vBa/VySYy3E=

hack/entrypoint.sh

@@ -0,0 +1,11 @@
+#!/bin/bash
+
+RESTART_EXIT_CODE=42
+
+while true; do
+    ./usr/local/bin/ocs-operator $@
+    EXIT_CODE=$?
+    if [ $EXIT_CODE -ne $RESTART_EXIT_CODE ]; then
+      exit $EXIT_CODE
+    fi
+done

internal/controller/storageclaim_controller.go

@@ -21,6 +21,7 @@ import (
 	"encoding/json"
 	"fmt"
 	"reflect"
+	"sigs.k8s.io/controller-runtime/pkg/handler"
 	"slices"
 	"strings"
 	"time"
@@ -32,9 +33,11 @@ import (
 	csiopv1a1 "github.com/ceph/ceph-csi-operator/api/v1alpha1"
 	"github.com/go-logr/logr"
 	snapapi "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1"
+	ramenv1alpha1 "github.com/ramendr/ramen/api/v1alpha1"
 	providerclient "github.com/red-hat-storage/ocs-operator/services/provider/api/v4/client"
 	corev1 "k8s.io/api/core/v1"
 	storagev1 "k8s.io/api/storage/v1"
+	extv1 "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -56,6 +59,8 @@ const (
 
 	pvClusterIDIndexName  = "index:persistentVolumeClusterID"
 	vscClusterIDIndexName = "index:volumeSnapshotContentCSIDriver"
+
+	drClusterConfigCRDName = "drclusterconfigs.ramendr.openshift.io"
 )
 
 // StorageClaimReconciler reconciles a StorageClaim object
@@ -64,6 +69,7 @@ type StorageClaimReconciler struct {
 	cache.Cache
 	Scheme            *runtime.Scheme
 	OperatorNamespace string
+	AvailableCrds     map[string]bool
 
 	log              logr.Logger
 	ctx              context.Context
@@ -110,12 +116,25 @@ func (r *StorageClaimReconciler) SetupWithManager(mgr ctrl.Manager) error {
 	bldr := ctrl.NewControllerManagedBy(mgr).
 		For(&v1alpha1.StorageClaim{}, builder.WithPredicates(generationChangePredicate)).
 		Owns(&storagev1.StorageClass{}).
-		Owns(&snapapi.VolumeSnapshotClass{})
+		Owns(&snapapi.VolumeSnapshotClass{}).
+		Watches(
+			&extv1.CustomResourceDefinition{},
+			&handler.EnqueueRequestForObject{},
+			builder.WithPredicates(
+				utils.NamePredicate(drClusterConfigCRDName),
+				utils.CrdCreateAndDeletePredicate(&r.log, drClusterConfigCRDName, r.AvailableCrds[drClusterConfigCRDName]),
+			),
+			builder.OnlyMetadata,
+		)
 
 	if utils.DelegateCSI {
 		bldr = bldr.Owns(&csiopv1a1.ClientProfile{}, builder.WithPredicates(generationChangePredicate))
 	}
 
+	if r.AvailableCrds[drClusterConfigCRDName] {
+		bldr = bldr.Owns(&ramenv1alpha1.DRClusterConfig{}, builder.WithPredicates(generationChangePredicate))
+	}
+
 	return bldr.Complete(r)
 }
 
@@ -144,6 +163,15 @@ func (r *StorageClaimReconciler) Reconcile(ctx context.Context, req ctrl.Request
 	r.ctx = ctrllog.IntoContext(ctx, r.log)
 	r.log.Info("Reconciling StorageClaim.")
 
+	crd := &metav1.PartialObjectMetadata{}
+	crd.SetGroupVersionKind(extv1.SchemeGroupVersion.WithKind("CustomResourceDefinition"))
+	crd.Name = drClusterConfigCRDName
+	if err := r.Client.Get(ctx, client.ObjectKeyFromObject(crd), crd); client.IgnoreNotFound(err) != nil {
+		r.log.Error(err, "Failed to get CRD", "CRD", drClusterConfigCRDName)
+		return reconcile.Result{}, err
+	}
+	utils.AssertEqual(r.AvailableCrds[drClusterConfigCRDName], crd.UID != "", utils.ExitCodeThatShouldRestartTheProcess)
+
 	// Fetch the StorageClaim instance
 	r.storageClaim = &v1alpha1.StorageClaim{}
 	r.storageClaim.Name = req.Name

pkg/utils/k8sutils.go

@@ -50,6 +50,8 @@ const CronScheduleWeekly = "@weekly"
 
 const CSIReconcileEnvVar = "CSI_RECONCILE"
 
+const ExitCodeThatShouldRestartTheProcess = 42
+
 // GetOperatorNamespace returns the namespace where the operator is deployed.
 func GetOperatorNamespace() string {
 	return os.Getenv(OperatorNamespaceEnvVar)

pkg/utils/predicates.go

@@ -0,0 +1,39 @@
+package utils
+
+import (
+	"github.com/go-logr/logr"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/event"
+	"sigs.k8s.io/controller-runtime/pkg/predicate"
+)
+
+// Name Predicate return a predicate the filter events produced
+// by resources that matches the given name
+func NamePredicate(name string) predicate.Predicate {
+	return predicate.NewPredicateFuncs(func(obj client.Object) bool {
+		return obj.GetName() == name
+	})
+}
+
+func CrdCreateAndDeletePredicate(log *logr.Logger, crdName string, crdExists bool) predicate.Predicate {
+	return predicate.Funcs{
+		CreateFunc: func(_ event.CreateEvent) bool {
+			if !crdExists {
+				log.Info("CustomResourceDefinition %s was Created.", crdName)
+			}
+			return !crdExists
+		},
+		DeleteFunc: func(_ event.DeleteEvent) bool {
+			if crdExists {
+				log.Info("CustomResourceDefinition %s was Deleted.", crdName)
+			}
+			return crdExists
+		},
+		UpdateFunc: func(_ event.UpdateEvent) bool {
+			return false
+		},
+		GenericFunc: func(_ event.GenericEvent) bool {
+			return false
+		},
+	}
+}

pkg/utils/slices.go

@@ -16,6 +16,8 @@ limitations under the License.
 
 package utils
 
+import "os"
+
 // Find returns the first entry matching the function "f" or else return nil
 func Find[T any](list []T, f func(item *T) bool) *T {
 	for idx := range list {
@@ -26,3 +28,9 @@ func Find[T any](list []T, f func(item *T) bool) *T {
 	}
 	return nil
 }
+
+func AssertEqual[T comparable](actual T, expected T, exitCode int) {
+	if actual != expected {
+		os.Exit(exitCode)
+	}
+}

service/status-report/main.go

@@ -20,6 +20,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"math"
 	"os"
 	"reflect"
 	"strings"
@@ -33,6 +34,7 @@ import (
 
 	csiopv1a1 "github.com/ceph/ceph-csi-operator/api/v1alpha1"
 	configv1 "github.com/openshift/api/config/v1"
+	quotav1 "github.com/openshift/api/quota/v1"
 	opv1a1 "github.com/operator-framework/api/pkg/operators/v1alpha1"
 	providerclient "github.com/red-hat-storage/ocs-operator/services/provider/api/v4/client"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -70,6 +72,10 @@ func main() {
 		klog.Exitf("Failed to add csiopv1a1 to scheme: %v", err)
 	}
 
+	if err := quotav1.AddToScheme(scheme); err != nil {
+		klog.Exitf("Failed to add quotav1 to scheme: %v", err)
+	}
+
 	config, err := config.GetConfig()
 	if err != nil {
 		klog.Exitf("Failed to get config: %v", err)
@@ -112,6 +118,7 @@ func main() {
 	setPlatformInformation(ctx, cl, status)
 	setOperatorInformation(ctx, cl, status, operatorNamespace)
 	setClusterInformation(ctx, cl, status)
+	setStorageQuotaUtilizationRatio(ctx, cl, status)
 	statusResponse, err := providerClient.ReportStatus(ctx, storageClient.Status.ConsumerID, status)
 	if err != nil {
 		klog.Exitf("Failed to report status of storageClient %v: %v", storageClient.Status.ConsumerID, err)
@@ -215,6 +222,26 @@ func setClusterInformation(ctx context.Context, cl client.Client, status interfa
 
 }
 
+func setStorageQuotaUtilizationRatio(ctx context.Context, cl client.Client, status interfaces.StorageClientStatus) {
+	clusterResourceQuota := &quotav1.ClusterResourceQuota{}
+	clusterResourceQuota.Name = status.GetClientName()
+
+	// No need to check for NotFound because unlimited quota client will not have CRQ resource
+	if err := cl.Get(ctx, client.ObjectKeyFromObject(clusterResourceQuota), clusterResourceQuota); client.IgnoreNotFound(err) != nil {
+		klog.Warningf("Failed to get clusterResourceQuota %q: %v", clusterResourceQuota.Name, err)
+	}
+
+	if clusterResourceQuota.Status.Total.Hard != nil {
+		total := clusterResourceQuota.Status.Total.Hard["requests.storage"]
+		if total.AsApproximateFloat64() > 0 {
+			used := clusterResourceQuota.Status.Total.Used["requests.storage"]
+			ratio := used.AsApproximateFloat64() / total.AsApproximateFloat64()
+			status.SetStorageQuotaUtilizationRatio(float32(math.Min(float64(ratio), float64(1.0))))
+		}
+	}
+
+}
+
 func setOperatorInformation(ctx context.Context, cl client.Client, status interfaces.StorageClientStatus,
 	operatorNamespace string) {
 	var operatorVersion string