Add annotation for multus in toolbox after checking the multus networ…

…k name for namespace

It is possible that the NetworkAttachmentDefinition for multus network
is configured in a different namespace than the one in which the operator is deployed
and multus is configured. In that case, the selector (eg: public) value in the
storagecluster CR is of the following pattern `NAD_namespace/NAD_name`.

Therefore, check for namespace value in the selector and add annotation to the
ceph toolbox accordingly.

BZ: https://bugzilla.redhat.com/show_bug.cgi?id=1982721

Signed-off-by: Nikhil-Ladha <[email protected]>

config/rbac/role.yaml

@@ -152,6 +152,14 @@ rules:
   - get
   - list
   - watch
+- apiGroups:
+  - k8s.cni.cncf.io
+  resources:
+  - network-attachment-definitions
+  verbs:
+  - get
+  - list
+  - watch
 - apiGroups:
   - monitoring.coreos.com
   resources:

controllers/storagecluster/cephtoolbox.go

@@ -3,15 +3,20 @@ package storagecluster
 import (
 	"context"
 	"fmt"
+	"os"
 	"reflect"
+	"strings"
 
+	nadclientset "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned"
 	ocsv1 "github.com/red-hat-storage/ocs-operator/api/v1"
 	"github.com/red-hat-storage/ocs-operator/controllers/defaults"
 	"github.com/red-hat-storage/ocs-operator/controllers/util"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/errors"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/types"
+	"k8s.io/client-go/tools/clientcmd"
 	"sigs.k8s.io/controller-runtime/pkg/controller/controllerutil"
 )
 
@@ -123,15 +128,41 @@ func getMultusPublicNetwork(sc *ocsv1.StorageCluster) (string, error) {
 		return "", err
 	}
 
-	multusNetNs, err := util.GetOperatorNamespace()
-	if err != nil {
-		return "", err
+	if sc.Spec.Network.Selectors["public"] == "" {
+		return "", nil
 	}
 
-	if sc.Spec.Network.Selectors["public"] != "" {
-		multusNetName := sc.Spec.Network.Selectors["public"]
-		return fmt.Sprintf("%s/%s", multusNetNs, multusNetName), nil
+	multusNetName := sc.Spec.Network.Selectors["public"]
+	multusNetNamespacedName := strings.Split(multusNetName, "/")
+	nadNS, nadName := "", ""
+	if len(multusNetNamespacedName) == 1 {
+		nadNS, nadName = os.Getenv(util.OperatorNamespaceEnvVar), multusNetName
+	} else if len(multusNetNamespacedName) == 2 {
+		nadNS, nadName = multusNetNamespacedName[0], multusNetNamespacedName[1]
+	} else {
+		return "", fmt.Errorf("Spec.Network.Selectors[\"public\"] value: %s in storagecluster CR is invalid", multusNetName)
+	}
+
+	nadClient, err := getNADClient()
+	if err != nil {
+		return "", fmt.Errorf("failed to get NAD client. %v", err)
+	}
+	_, err = nadClient.K8sCniCncfIoV1().NetworkAttachmentDefinitions(nadNS).Get(context.TODO(), nadName, metav1.GetOptions{})
+	if err != nil {
+		return "", fmt.Errorf("Either create the NetworkAttachmentDefinition or alter the storagecluster Spec.Network.Selectors[\"public\"] with correct value. %v", err)
 	}
 
-	return "", nil
+	return fmt.Sprintf("%s/%s", nadNS, nadName), nil
+}
+
+func getNADClient() (*nadclientset.Clientset, error) {
+	cfg, err := clientcmd.BuildConfigFromFlags("", "")
+	if err != nil {
+		return nil, fmt.Errorf("failed to build config for NAD. %v", err)
+	}
+	client, err := nadclientset.NewForConfig(cfg)
+	if err != nil {
+		return nil, fmt.Errorf("failed to create a new Clientset for NAD. %v", err)
+	}
+	return client, nil
 }

controllers/storagecluster/reconcile.go

@@ -128,6 +128,7 @@ var validTopologyLabelKeys = []string{
 // +kubebuilder:rbac:groups=batch,resources=cronjobs;jobs,verbs=get;list;create;update;watch;delete
 // +kubebuilder:rbac:groups=cluster.open-cluster-management.io,resources=clusterclaims,verbs=get;list;watch;create;update;delete
 // +kubebuilder:rbac:groups=operators.coreos.com,resources=clusterserviceversions,verbs=get;list;watch
+// +kubebuilder:rbac:groups=k8s.cni.cncf.io,resources=network-attachment-definitions,verbs=get;list;watch
 
 // Reconcile reads that state of the cluster for a StorageCluster object and makes changes based on the state read
 // and what is in the StorageCluster.Spec

deploy/csv-templates/ocs-operator.csv.yaml.in

@@ -355,6 +355,14 @@ spec:
           - get
           - list
           - watch
+        - apiGroups:
+          - k8s.cni.cncf.io
+          resources:
+          - network-attachment-definitions
+          verbs:
+          - get
+          - list
+          - watch
         - apiGroups:
           - monitoring.coreos.com
           resources:

deploy/ocs-operator/manifests/ocs-operator.clusterserviceversion.yaml

@@ -1933,6 +1933,14 @@ spec:
           - get
           - list
           - watch
+        - apiGroups:
+          - k8s.cni.cncf.io
+          resources:
+          - network-attachment-definitions
+          verbs:
+          - get
+          - list
+          - watch
         - apiGroups:
           - monitoring.coreos.com
           resources:

go.mod

@@ -11,6 +11,7 @@ require (
 	github.com/go-logr/logr v1.2.3
 	github.com/google/uuid v1.3.0
 	github.com/imdario/mergo v0.3.13
+	github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0
 	github.com/kube-object-storage/lib-bucket-provisioner v0.0.0-20221122204822-d1a8c34382f1
 	github.com/kubernetes-csi/external-snapshotter/client/v6 v6.2.0
 	github.com/noobaa/noobaa-operator/v5 v5.0.0-20230306134341-1874f52f83dd

go.sum

@@ -1294,6 +1294,8 @@ github.com/jstemmer/go-junit-report v0.9.1/go.mod h1:Brl9GWCQeLvo8nXZwPNNblvFj/X
 github.com/jtolds/gls v4.20.0+incompatible/go.mod h1:QJZ7F/aHp+rZTRtaJ1ow/lLfFfVYBRgL+9YlvaHOwJU=
 github.com/julienschmidt/httprouter v1.2.0/go.mod h1:SYymIcj16QtmaHHD7aYtjjsJG7VTCxuUUipMqKk8s4w=
 github.com/julienschmidt/httprouter v1.3.0/go.mod h1:JR6WtHb+2LUe8TCKY3cZOxFyyO8IZAc4RVcycCCAKdM=
+github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0 h1:VzM3TYHDgqPkettiP6I6q2jOeQFL4nrJM+UcAc4f6Fs=
+github.com/k8snetworkplumbingwg/network-attachment-definition-client v1.4.0/go.mod h1:nqCI7aelBJU61wiBeeZWJ6oi4bJy5nrjkM6lWIMA4j0=
 github.com/karrick/godirwalk v1.8.0/go.mod h1:H5KPZjojv4lE+QYImBI8xVtrBRgYrIVsaRPx4tDPEn4=
 github.com/karrick/godirwalk v1.10.3/go.mod h1:RoGL9dQei4vP9ilrpETWE8CLOZ1kiN0LhBygSwrAsHA=
 github.com/kelseyhightower/envconfig v1.4.0 h1:Im6hONhd3pLkfDFsbRgu68RDNkGF1r3dvMUtDTo2cv8=

main.go

@@ -23,6 +23,7 @@ import (
 	"os"
 	"runtime"
 
+	nadscheme "github.com/k8snetworkplumbingwg/network-attachment-definition-client/pkg/client/clientset/versioned/scheme"
 	snapapi "github.com/kubernetes-csi/external-snapshotter/client/v6/apis/volumesnapshot/v1"
 	nbapis "github.com/noobaa/noobaa-operator/v5/pkg/apis"
 	openshiftConfigv1 "github.com/openshift/api/config/v1"
@@ -87,6 +88,7 @@ func init() {
 	utilruntime.Must(ocsv1alpha1.AddToScheme(scheme))
 	utilruntime.Must(clusterv1alpha1.AddToScheme(scheme))
 	utilruntime.Must(operatorsv1alpha1.AddToScheme(scheme))
+	utilruntime.Must(nadscheme.AddToScheme(scheme))
 	// +kubebuilder:scaffold:scheme
 }