From ae425eb33f58fd116858bb8d10a5b474707036f4 Mon Sep 17 00:00:00 2001 From: Nitin Goyal Date: Thu, 7 Mar 2024 12:11:43 +0530 Subject: [PATCH] Add CSI tolerations to rook-ceph-operator-config via controller Enhance the configuration process by incorporating CSI tolerations directly into the rook-ceph-operator-config configmap through the controller. Previously, end users were required to manually edit the configmap for adding CSI tolerations. With this update, users can now seamlessly include CSI tolerations in the storage clusters, and the ocs-operator will automatically propagate these tolerations to the configmap. This improvement simplifies the configuration workflow and ensures that CSI tolerations are seamlessly applied to the Rook Ceph Operator without the need for manual intervention in the configmap. Signed-off-by: Nitin Goyal --- controllers/defaults/placements.go | 14 ++++ .../ocsinitialization_controller.go | 73 ++++++++++++++++++- go.mod | 2 +- 3 files changed, 84 insertions(+), 5 deletions(-) diff --git a/controllers/defaults/placements.go b/controllers/defaults/placements.go index ea79bb8f04..bec7d20b5e 100644 --- a/controllers/defaults/placements.go +++ b/controllers/defaults/placements.go @@ -9,6 +9,8 @@ import ( var ( APIServerKey = "api-server" MetricsExporterKey = "metrics-exporter" + CsiPluginKey = "csi-plugin" + CsiProvisionerKey = "csi-provisioner" // osdLabelSelector is the key in OSD pod. Used // as a label selector for topology spread constraints. @@ -141,6 +143,18 @@ var ( getOcsToleration(), }, }, + + CsiPluginKey: { + Tolerations: []corev1.Toleration{ + getOcsToleration(), + }, + }, + + CsiProvisionerKey: { + Tolerations: []corev1.Toleration{ + getOcsToleration(), + }, + }, } ) diff --git a/controllers/ocsinitialization/ocsinitialization_controller.go b/controllers/ocsinitialization/ocsinitialization_controller.go index e969995f5a..688bc3a87b 100644 --- a/controllers/ocsinitialization/ocsinitialization_controller.go +++ b/controllers/ocsinitialization/ocsinitialization_controller.go @@ -9,7 +9,10 @@ import ( "github.com/go-logr/logr" secv1client "github.com/openshift/client-go/security/clientset/versioned/typed/security/v1" ocsv1 "github.com/red-hat-storage/ocs-operator/api/v4/v1" + "github.com/red-hat-storage/ocs-operator/v4/controllers/defaults" "github.com/red-hat-storage/ocs-operator/v4/controllers/util" + rookCephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" + "gopkg.in/yaml.v2" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -241,7 +244,8 @@ func (r *OCSInitializationReconciler) SetupWithManager(mgr ctrl.Manager) error { } // ensureRookCephOperatorConfigExists ensures that the rook-ceph-operator-config cm exists -// This configmap is purely reserved for any user overrides to be applied. +// This configmap is semi-reserved for any user overrides to be applied 4.16 onwards. +// Earlier it used to be purely reserved for user overrides. // We don't reconcile it if it exists as it can reset any values the user has set // The configmap is watched by the rook operator and values set here have higher precedence // than the default values set in the rook operator pod env vars. @@ -252,14 +256,75 @@ func (r *OCSInitializationReconciler) ensureRookCephOperatorConfigExists(initial Namespace: initialData.Namespace, }, } - err := r.Client.Create(r.ctx, rookCephOperatorConfig) - if err != nil && !errors.IsAlreadyExists(err) { - r.Log.Error(err, fmt.Sprintf("Failed to create %s configmap", util.RookCephOperatorConfigName)) + + opResult, err := ctrl.CreateOrUpdate(r.ctx, r.Client, rookCephOperatorConfig, func() error { + + csiPluginDefaults := defaults.DaemonPlacements[defaults.CsiPluginKey] + csiProvisionerDefaults := defaults.DaemonPlacements[defaults.CsiProvisionerKey] + + csiPluginTolerations := r.getCsiTolerations(defaults.CsiPluginKey) + csiProvisionerTolerations := r.getCsiTolerations(defaults.CsiProvisionerKey) + + if rookCephOperatorConfig.Data == nil { + rookCephOperatorConfig.Data = make(map[string]string) + } + + if err := updateTolerationsConfigFunc(rookCephOperatorConfig, + csiPluginTolerations, csiPluginDefaults.Tolerations, "CSI_PLUGIN_TOLERATIONS", + &initialData.Status.RookCephOperatorConfig.CsiPluginTolerationsModified); err != nil { + return err + } + + err := updateTolerationsConfigFunc(rookCephOperatorConfig, + csiProvisionerTolerations, csiProvisionerDefaults.Tolerations, "CSI_PROVISIONER_TOLERATIONS", + &initialData.Status.RookCephOperatorConfig.CsiProvisionerTolerationsModified) + + return err + }) + + if err != nil { + r.Log.Error(err, "Failed to create/update rook-ceph-operator-config configmap") return err } + r.Log.Info("Successfully created/updated rook-ceph-operator-config configmap", "OperationResult", opResult) + + return nil +} + +func updateTolerationsConfigFunc(rookCephOperatorConfig *corev1.ConfigMap, + tolerations, defaults []corev1.Toleration, configMapKey string, modifiedFlag *bool) error { + + if tolerations != nil { + updatedTolerations := append(tolerations, defaults...) + tolerationsYAML, err := yaml.Marshal(updatedTolerations) + if err != nil { + return err + } + rookCephOperatorConfig.Data[configMapKey] = string(tolerationsYAML) + *modifiedFlag = true + } else if tolerations == nil && *modifiedFlag { + delete(rookCephOperatorConfig.Data, configMapKey) + *modifiedFlag = false + } + return nil } +func (r *OCSInitializationReconciler) getCsiTolerations(csiTolerationKey string) []corev1.Toleration { + + var tolerations []corev1.Toleration + + clusters := r.clusters.GetStorageClusters() + + for i := range clusters { + if val, ok := clusters[i].Spec.Placement[rookCephv1.KeyType(csiTolerationKey)]; ok { + tolerations = append(tolerations, val.Tolerations...) + } + } + + return tolerations +} + // ensureOcsOperatorConfigExists ensures that the ocs-operator-config exists & if not create/update it with required values // This configmap is reserved just for ocs operator use, primarily meant for passing values to rook-ceph-operator // It is not meant to be modified by the user diff --git a/go.mod b/go.mod index 8ed0895cf1..6596d002b0 100644 --- a/go.mod +++ b/go.mod @@ -36,6 +36,7 @@ require ( google.golang.org/grpc v1.60.0 google.golang.org/protobuf v1.33.0 gopkg.in/ini.v1 v1.67.0 + gopkg.in/yaml.v2 v2.4.0 gotest.tools/v3 v3.5.0 k8s.io/api v0.28.4 k8s.io/apiextensions-apiserver v0.28.4 @@ -127,7 +128,6 @@ require ( google.golang.org/appengine v1.6.8 // indirect google.golang.org/genproto/googleapis/rpc v0.0.0-20231002182017-d307bd883b97 // indirect gopkg.in/inf.v0 v0.9.1 // indirect - gopkg.in/yaml.v2 v2.4.0 // indirect gopkg.in/yaml.v3 v3.0.1 // indirect k8s.io/apiserver v0.28.4 // indirect k8s.io/component-base v0.28.4 // indirect