Add CSI tolerations to rook-ceph-operator-config via controller

Enhance the configuration process by incorporating CSI tolerations
directly into the rook-ceph-operator-config configmap through the
controller. Previously, end users were required to manually edit the
configmap for adding CSI tolerations. With this update, users can now
seamlessly include CSI tolerations in the storage clusters, and the
ocs-operator will automatically propagate these tolerations to the
configmap.

This improvement simplifies the configuration workflow and ensures that
CSI tolerations are seamlessly applied to the Rook Ceph Operator without
the need for manual intervention in the configmap.

Signed-off-by: Nitin Goyal <[email protected]>

controllers/defaults/placements.go

@@ -9,6 +9,8 @@ import (
 var (
 	APIServerKey       = "api-server"
 	MetricsExporterKey = "metrics-exporter"
+	CsiPluginKey       = "csi-plugin"
+	CsiProvisionerKey  = "csi-provisioner"
 
 	// osdLabelSelector is the key in OSD pod. Used
 	// as a label selector for topology spread constraints.
@@ -141,6 +143,18 @@ var (
 				getOcsToleration(),
 			},
 		},
+
+		CsiPluginKey: {
+			Tolerations: []corev1.Toleration{
+				getOcsToleration(),
+			},
+		},
+
+		CsiProvisionerKey: {
+			Tolerations: []corev1.Toleration{
+				getOcsToleration(),
+			},
+		},
 	}
 )
 

controllers/ocsinitialization/ocsinitialization_controller.go

@@ -9,7 +9,10 @@ import (
 	"github.com/go-logr/logr"
 	secv1client "github.com/openshift/client-go/security/clientset/versioned/typed/security/v1"
 	ocsv1 "github.com/red-hat-storage/ocs-operator/api/v4/v1"
+	"github.com/red-hat-storage/ocs-operator/v4/controllers/defaults"
 	"github.com/red-hat-storage/ocs-operator/v4/controllers/util"
+	rookCephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1"
+	"gopkg.in/yaml.v2"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -241,7 +244,8 @@ func (r *OCSInitializationReconciler) SetupWithManager(mgr ctrl.Manager) error {
 }
 
 // ensureRookCephOperatorConfigExists ensures that the rook-ceph-operator-config cm exists
-// This configmap is purely reserved for any user overrides to be applied.
+// This configmap is semi-reserved for any user overrides to be applied 4.16 onwards.
+// Earlier it used to be purely reserved for user overrides.
 // We don't reconcile it if it exists as it can reset any values the user has set
 // The configmap is watched by the rook operator and values set here have higher precedence
 // than the default values set in the rook operator pod env vars.
@@ -252,14 +256,75 @@ func (r *OCSInitializationReconciler) ensureRookCephOperatorConfigExists(initial
 			Namespace: initialData.Namespace,
 		},
 	}
-	err := r.Client.Create(r.ctx, rookCephOperatorConfig)
-	if err != nil && !errors.IsAlreadyExists(err) {
-		r.Log.Error(err, fmt.Sprintf("Failed to create %s configmap", util.RookCephOperatorConfigName))
+
+	opResult, err := ctrl.CreateOrUpdate(r.ctx, r.Client, rookCephOperatorConfig, func() error {
+
+		if rookCephOperatorConfig.Data == nil {
+			rookCephOperatorConfig.Data = make(map[string]string)
+		}
+
+		csiPluginDefaults := defaults.DaemonPlacements[defaults.CsiPluginKey]
+		csiPluginTolerations := r.getCsiTolerations(defaults.CsiPluginKey)
+		if err := updateTolerationsConfigFunc(rookCephOperatorConfig,
+			csiPluginTolerations, csiPluginDefaults.Tolerations, "CSI_PLUGIN_TOLERATIONS",
+			&initialData.Status.RookCephOperatorConfig.CsiPluginTolerationsModified); err != nil {
+			return err
+		}
+
+		csiProvisionerDefaults := defaults.DaemonPlacements[defaults.CsiProvisionerKey]
+		csiProvisionerTolerations := r.getCsiTolerations(defaults.CsiProvisionerKey)
+		if err := updateTolerationsConfigFunc(rookCephOperatorConfig,
+			csiProvisionerTolerations, csiProvisionerDefaults.Tolerations, "CSI_PROVISIONER_TOLERATIONS",
+			&initialData.Status.RookCephOperatorConfig.CsiProvisionerTolerationsModified); err != nil {
+			return err
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		r.Log.Error(err, "Failed to create/update rook-ceph-operator-config configmap")
 		return err
 	}
+	r.Log.Info("Successfully created/updated rook-ceph-operator-config configmap", "OperationResult", opResult)
+
+	return nil
+}
+
+func updateTolerationsConfigFunc(rookCephOperatorConfig *corev1.ConfigMap,
+	tolerations, defaults []corev1.Toleration, configMapKey string, modifiedFlag *bool) error {
+
+	if tolerations != nil {
+		updatedTolerations := append(tolerations, defaults...)
+		tolerationsYAML, err := yaml.Marshal(updatedTolerations)
+		if err != nil {
+			return err
+		}
+		rookCephOperatorConfig.Data[configMapKey] = string(tolerationsYAML)
+		*modifiedFlag = true
+	} else if tolerations == nil && *modifiedFlag {
+		delete(rookCephOperatorConfig.Data, configMapKey)
+		*modifiedFlag = false
+	}
+
 	return nil
 }
 
+func (r *OCSInitializationReconciler) getCsiTolerations(csiTolerationKey string) []corev1.Toleration {
+
+	var tolerations []corev1.Toleration
+
+	clusters := r.clusters.GetStorageClusters()
+
+	for i := range clusters {
+		if val, ok := clusters[i].Spec.Placement[rookCephv1.KeyType(csiTolerationKey)]; ok {
+			tolerations = append(tolerations, val.Tolerations...)
+		}
+	}
+
+	return tolerations
+}
+
 // ensureOcsOperatorConfigExists ensures that the ocs-operator-config exists & if not create/update it with required values
 // This configmap is reserved just for ocs operator use, primarily meant for passing values to rook-ceph-operator
 // It is not meant to be modified by the user

go.mod

@@ -36,6 +36,7 @@ require (
 	google.golang.org/grpc v1.60.0
 	google.golang.org/protobuf v1.33.0
 	gopkg.in/ini.v1 v1.67.0
+	gopkg.in/yaml.v2 v2.4.0
 	gotest.tools/v3 v3.5.0
 	k8s.io/api v0.28.4
 	k8s.io/apiextensions-apiserver v0.28.4
@@ -127,7 +128,6 @@ require (
 	google.golang.org/appengine v1.6.8 // indirect
 	google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect
 	gopkg.in/inf.v0 v0.9.1 // indirect
-	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // indirect
 	k8s.io/apiserver v0.28.4 // indirect
 	k8s.io/component-base v0.28.4 // indirect