From 943f378df6173662339dbfe89b31497fe68aa93c Mon Sep 17 00:00:00 2001 From: Kaustav Majumder Date: Tue, 2 Jul 2024 09:59:37 +0530 Subject: [PATCH 1/2] added support for ocs provider server to fetch noobaa resources Signed-off-by: Kaustav Majumder --- rbac/provider-role.yaml | 16 +++ services/provider/server/consumer.go | 47 +++++++ services/provider/server/consumer_test.go | 4 +- services/provider/server/server.go | 82 ++++++++++++- services/provider/server/server_test.go | 143 ++++++++++++++++------ 5 files changed, 245 insertions(+), 47 deletions(-) diff --git a/rbac/provider-role.yaml b/rbac/provider-role.yaml index aea3ada2e5..260b5d135a 100644 --- a/rbac/provider-role.yaml +++ b/rbac/provider-role.yaml @@ -68,3 +68,19 @@ rules: verbs: - get - list +- apiGroups: + - noobaa.io + resources: + - noobaaaccounts + verbs: + - get + - list + - create + - delete +- apiGroups: + - route.openshift.io + resources: + - routes + verbs: + - get + - list diff --git a/services/provider/server/consumer.go b/services/provider/server/consumer.go index 6e050257fd..9415b8f014 100644 --- a/services/provider/server/consumer.go +++ b/services/provider/server/consumer.go @@ -4,9 +4,12 @@ import ( "context" "errors" "fmt" + "strings" "sync" + nbv1 "github.com/noobaa/noobaa-operator/v5/pkg/apis/noobaa/v1alpha1" ocsv1alpha1 "github.com/red-hat-storage/ocs-operator/api/v4/v1alpha1" + "github.com/red-hat-storage/ocs-operator/v4/controllers/util" ifaces "github.com/red-hat-storage/ocs-operator/v4/services/provider/interfaces" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -226,3 +229,47 @@ func (c *ocsConsumerManager) UpdateConsumerStatus(ctx context.Context, id string klog.Infof("successfully updated Status for StorageConsumer %v", consumerObj.Name) return nil } + +func (c *ocsConsumerManager) CreateNoobaaAccount(ctx context.Context, id string) error { + + consumerObj, err := c.Get(ctx, id) + if err != nil { + return err + } + consumerClusterID := strings.TrimPrefix(consumerObj.Name, "storageconsumer-") + if consumerClusterID != "" && len(consumerClusterID) == 0 { + return fmt.Errorf("failed to get clusterID from consumerResource Name: %s %v", consumerObj.Name, err) + } + + noobaaAccountName := fmt.Sprintf("noobaa-remote-%s", consumerClusterID) + nbAccountObj := &nbv1.NooBaaAccount{} + nbAccountObj.Name = noobaaAccountName + nbAccountObj.Namespace = consumerObj.Namespace + // the following annotation will enable noobaa-operator to create a auth_token secret based on this account + util.AddAnnotation(nbAccountObj, "remote-operator", "true") + + err = c.client.Create(ctx, nbAccountObj) + if err != nil { + return fmt.Errorf("failed to create noobaa account for storageConsumer %v: %v", consumerObj.Name, err) + } + return nil +} + +func (c *ocsConsumerManager) DeleteNoobaaAccount(ctx context.Context, id string) error { + consumerObj, err := c.Get(ctx, id) + if err != nil { + return err + } + clusterID := strings.TrimPrefix(consumerObj.Name, "storageconsumer-") + if clusterID != "" && len(clusterID) == 0 { + return fmt.Errorf("failed to get clusterID from consumerResource Name: %s %v", consumerObj.Name, err) + } + noobaaAccountName := fmt.Sprintf("noobaa-remote-%s", clusterID) + nbAccountObj := &nbv1.NooBaaAccount{} + nbAccountObj.Name = noobaaAccountName + nbAccountObj.Namespace = consumerObj.Namespace + if err := c.client.Delete(ctx, nbAccountObj); err != nil { + return fmt.Errorf("failed to delete Noobaa account %q. %v", nbAccountObj.Name, err) + } + return nil +} diff --git a/services/provider/server/consumer_test.go b/services/provider/server/consumer_test.go index ad0f30acb8..f3b60293bc 100644 --- a/services/provider/server/consumer_test.go +++ b/services/provider/server/consumer_test.go @@ -4,6 +4,7 @@ import ( "context" "testing" + routev1 "github.com/openshift/api/route/v1" opv1a1 "github.com/operator-framework/api/pkg/operators/v1alpha1" api "github.com/red-hat-storage/ocs-operator/api/v4/v1" ocsv1alpha1 "github.com/red-hat-storage/ocs-operator/api/v4/v1alpha1" @@ -13,7 +14,6 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) @@ -67,6 +67,8 @@ func newFakeClient(t *testing.T, obj ...client.Object) client.Client { err = opv1a1.AddToScheme(scheme) assert.NoError(t, err, "failed to add opv1a1 scheme") + err = routev1.AddToScheme(scheme) + assert.NoError(t, err, "failed to add routev1 scheme") return fake.NewClientBuilder(). WithScheme(scheme). WithObjects(obj...). diff --git a/services/provider/server/server.go b/services/provider/server/server.go index 1d005fba11..f3f69dd7af 100644 --- a/services/provider/server/server.go +++ b/services/provider/server/server.go @@ -19,11 +19,16 @@ import ( "time" "github.com/blang/semver/v4" + nbapis "github.com/noobaa/noobaa-operator/v5/pkg/apis" + nbv1 "github.com/noobaa/noobaa-operator/v5/pkg/apis/noobaa/v1alpha1" quotav1 "github.com/openshift/api/quota/v1" + routev1 "github.com/openshift/api/route/v1" + opv1a1 "github.com/operator-framework/api/pkg/operators/v1alpha1" ocsv1 "github.com/red-hat-storage/ocs-operator/api/v4/v1" ocsv1alpha1 "github.com/red-hat-storage/ocs-operator/api/v4/v1alpha1" controllers "github.com/red-hat-storage/ocs-operator/v4/controllers/storageconsumer" "github.com/red-hat-storage/ocs-operator/v4/controllers/util" + "github.com/red-hat-storage/ocs-operator/v4/services" pb "github.com/red-hat-storage/ocs-operator/v4/services/provider/pb" ocsVersion "github.com/red-hat-storage/ocs-operator/v4/version" rookCephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" @@ -31,8 +36,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" csiopv1a1 "github.com/ceph/ceph-csi-operator/api/v1alpha1" - opv1a1 "github.com/operator-framework/api/pkg/operators/v1alpha1" - "github.com/red-hat-storage/ocs-operator/v4/services" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" @@ -148,7 +151,10 @@ func (s *OCSProviderServer) AcknowledgeOnboarding(ctx context.Context, req *pb.A } return nil, status.Errorf(codes.Internal, "Failed to update the storageConsumer. %v", err) } - + // create noobaa account CR + if err := s.consumerManager.CreateNoobaaAccount(ctx, req.StorageConsumerUUID); err != nil { + return nil, status.Errorf(codes.Internal, "Failed to create noobaa account for storageconsumer. %v", err) + } return &pb.AcknowledgeOnboardingResponse{}, nil } @@ -198,12 +204,15 @@ func (s *OCSProviderServer) GetStorageConfig(ctx context.Context, req *pb.Storag // OffboardConsumer RPC call to delete the StorageConsumer CR func (s *OCSProviderServer) OffboardConsumer(ctx context.Context, req *pb.OffboardConsumerRequest) (*pb.OffboardConsumerResponse, error) { - - err := s.consumerManager.Delete(ctx, req.StorageConsumerUUID) + // remove noobaa account + err := s.consumerManager.DeleteNoobaaAccount(ctx, req.StorageConsumerUUID) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to delete noobaaAccount resource with the provided UUID. %v", err) + } + err = s.consumerManager.Delete(ctx, req.StorageConsumerUUID) if err != nil { return nil, status.Errorf(codes.Internal, "failed to delete storageConsumer resource with the provided UUID. %v", err) } - return &pb.OffboardConsumerResponse{}, nil } @@ -254,6 +263,14 @@ func newClient() (client.Client, error) { if err != nil { return nil, fmt.Errorf("failed to add ocsv1 to scheme. %v", err) } + err = routev1.AddToScheme(scheme) + if err != nil { + return nil, fmt.Errorf("failed to add routev1 to scheme. %v", err) + } + err = nbapis.AddToScheme(scheme) + if err != nil { + return nil, fmt.Errorf("failed to add nbapis to scheme. %v", err) + } config, err := config.GetConfig() if err != nil { @@ -427,6 +444,59 @@ func (s *OCSProviderServer) getExternalResources(ctx context.Context, consumerRe } + // Fetch noobaa remote secret and management address and append to extResources + noobaaOperatorSecret := &v1.Secret{} + clusterID := strings.TrimPrefix(consumerResource.Name, "storageconsumer-") + if clusterID != "" && len(clusterID) == 0 { + return nil, fmt.Errorf("failed to get clusterID from consumerResource Name: %s %v", consumerResource.Name, err) + } + + noobaaOperatorSecretName := fmt.Sprintf("noobaa-remote-join-secret-%s", clusterID) + err = s.client.Get(ctx, types.NamespacedName{Name: noobaaOperatorSecretName, Namespace: s.namespace}, noobaaOperatorSecret) + if err != nil { + return nil, fmt.Errorf("failed to get %s secret. %v", noobaaOperatorSecretName, err) + } + + authToken, ok := noobaaOperatorSecret.Data["auth_token"] + if !ok || len(authToken) == 0 { + return nil, fmt.Errorf("auth_token not found in %s secret", noobaaOperatorSecretName) + } + + noobaMgmtRoute := &routev1.Route{} + err = s.client.Get(ctx, types.NamespacedName{Name: "noobaa-mgmt", Namespace: s.namespace}, noobaMgmtRoute) + if err != nil { + return nil, fmt.Errorf("failed to get noobaa-mgmt route. %v", err) + } + if noobaMgmtRoute.Status.Ingress == nil || len(noobaMgmtRoute.Status.Ingress) == 0 { + return nil, fmt.Errorf("no Ingress available in noobaa-mgmt route") + } + + noobaaMgmtAddress := noobaMgmtRoute.Status.Ingress[0].Host + if noobaaMgmtAddress == "" { + return nil, fmt.Errorf("no Host found in noobaa-mgmt route Ingress") + } + joinSecret := &corev1.Secret{ + Data: map[string][]byte{ + "auth_token": authToken, + "mgmt_addr": []byte(noobaaMgmtAddress), + }, + } + extR = append(extR, &pb.ExternalResource{ + Name: "noobaa-remote-join-secret", + Kind: "Secret", + Data: mustMarshal(joinSecret), + }) + + noobaaSpec := &nbv1.NooBaaSpec{ + JoinSecret: &v1.SecretReference{ + Name: "noobaa-remote-join-secret", + }, + } + extR = append(extR, &pb.ExternalResource{ + Name: "noobaa-remote", + Kind: "Noobaa", + Data: mustMarshal(noobaaSpec), + }) return extR, nil } diff --git a/services/provider/server/server_test.go b/services/provider/server/server_test.go index b02d274d39..e841929f30 100644 --- a/services/provider/server/server_test.go +++ b/services/provider/server/server_test.go @@ -8,7 +8,9 @@ import ( "testing" csiopv1a1 "github.com/ceph/ceph-csi-operator/api/v1alpha1" + nbv1 "github.com/noobaa/noobaa-operator/v5/pkg/apis/noobaa/v1alpha1" quotav1 "github.com/openshift/api/quota/v1" + routev1 "github.com/openshift/api/route/v1" opv1a1 "github.com/operator-framework/api/pkg/operators/v1alpha1" ocsv1 "github.com/red-hat-storage/ocs-operator/api/v4/v1" ocsv1alpha1 "github.com/red-hat-storage/ocs-operator/api/v4/v1alpha1" @@ -56,49 +58,71 @@ var ocsSubscriptionSpec = &opv1a1.SubscriptionSpec{ Channel: "1.0", Package: "ocs-operator", } +var noobaaSpec = &nbv1.NooBaaSpec{ + JoinSecret: &v1.SecretReference{ + Name: "noobaa-remote-join-secret", + }, +} + +var joinSecret = &v1.Secret{ + Data: map[string][]byte{ + "auth_token": []byte("authToken"), + "mgmt_addr": []byte("noobaaMgmtAddress"), + }, +} var mockExtR = map[string]*externalResource{ "rook-ceph-mon-endpoints": { Name: "rook-ceph-mon-endpoints", Kind: "ConfigMap", - Data: map[string]string{ + Data: mustMarshal(map[string]string{ "data": "a=10.99.45.27:6789", "maxMonId": "0", "mapping": "{}", - }, + }), }, "rook-ceph-mon": { Name: "rook-ceph-mon", Kind: "Secret", - Data: map[string]string{ + Data: mustMarshal(map[string]string{ "ceph-username": "client.995e66248ad3e8642de868f461cdd827", "fsid": "b88c2d78-9de9-4227-9313-a63f62f78743", "mon-secret": "mon-secret", "ceph-secret": "AQADw/hhqBOcORAAJY3fKIvte++L/zYhASjYPQ==", - }, + }), }, "monitoring-endpoint": { Name: "monitoring-endpoint", Kind: "CephCluster", - Data: map[string]string{ + Data: mustMarshal(map[string]string{ "MonitoringEndpoint": "10.105.164.231", "MonitoringPort": "9283", - }, + }), }, "rook-ceph-client-995e66248ad3e8642de868f461cdd827": { Name: "rook-ceph-client-995e66248ad3e8642de868f461cdd827", Kind: "Secret", - Data: map[string]string{ + Data: mustMarshal(map[string]string{ "userID": "995e66248ad3e8642de868f461cdd827", "userKey": "AQADw/hhqBOcORAAJY3fKIvte++L/zYhASjYPQ==", - }, + }), }, "QuotaForConsumer": { Name: "QuotaForConsumer", Kind: "ClusterResourceQuota", - Data: map[string]string{ + Data: mustMarshal(map[string]string{ "QuotaForConsumer": fmt.Sprintf("%+v\n", clusterResourceQuotaSpec), - }, + }), + }, + "noobaa-remote-join-secret": { + Name: "noobaa-remote-join-secret", + Kind: "Secret", + Data: mustMarshal(joinSecret), + }, + "noobaa-remote": { + Name: "noobaa-remote", + Kind: "Noobaa", + Data: mustMarshal(noobaaSpec), }, "monitor-endpoints": { Name: "monitor-endpoints", @@ -264,9 +288,34 @@ func TestGetExternalResources(t *testing.T) { }, } + noobaaRemoteJoinSecretConsumer := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "noobaa-remote-join-secret-consumer", Namespace: server.namespace}, + Data: map[string][]byte{ + "auth_token": []byte("authToken"), + }, + } + + noobaaRemoteJoinSecretConsumer6 := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "noobaa-remote-join-secret-consumer6", Namespace: server.namespace}, + Data: map[string][]byte{ + "auth_token": []byte("authToken"), + }, + } + + noobaaMgmtRoute := &routev1.Route{ + ObjectMeta: metav1.ObjectMeta{Name: "noobaa-mgmt", Namespace: server.namespace}, + Status: routev1.RouteStatus{ + Ingress: []routev1.RouteIngress{{Host: "noobaaMgmtAddress"}}, + }, + } + assert.NoError(t, client.Create(ctx, cephClient)) assert.NoError(t, client.Create(ctx, secret)) + assert.NoError(t, client.Create(ctx, noobaaRemoteJoinSecretConsumer)) + assert.NoError(t, client.Create(ctx, noobaaRemoteJoinSecretConsumer6)) + assert.NoError(t, client.Create(ctx, noobaaMgmtRoute)) + monCm, monSc := createMonConfigMapAndSecret(server) assert.NoError(t, client.Create(ctx, monCm)) assert.NoError(t, client.Create(ctx, monSc)) @@ -290,9 +339,18 @@ func TestGetExternalResources(t *testing.T) { mockResoruce, ok := mockExtR[extResource.Name] assert.True(t, ok) - data, err := json.Marshal(mockResoruce.Data) assert.NoError(t, err) - assert.Equal(t, string(extResource.Data), string(data)) + if extResource.Kind == "Noobaa" { + var extNoobaaSpec, mockNoobaaSpec nbv1.NooBaaSpec + err = json.Unmarshal(extResource.Data, &extNoobaaSpec) + assert.NoError(t, err) + err = json.Unmarshal([]byte(mockResoruce.Data), &mockNoobaaSpec) + assert.NoError(t, err) + + assert.Equal(t, mockNoobaaSpec.JoinSecret, extNoobaaSpec.JoinSecret) + } else { + assert.Equal(t, extResource.Data, mockResoruce.Data) + } assert.Equal(t, extResource.Kind, mockResoruce.Kind) assert.Equal(t, extResource.Name, mockResoruce.Name) } @@ -310,7 +368,6 @@ func TestGetExternalResources(t *testing.T) { mockResoruce, ok := mockExtR[extResource.Name] assert.True(t, ok) - data, err := json.Marshal(mockResoruce.Data) assert.NoError(t, err) if extResource.Kind == "ClusterResourceQuota" { var clusterResourceQuotaSpec quotav1.ClusterResourceQuotaSpec @@ -318,8 +375,16 @@ func TestGetExternalResources(t *testing.T) { assert.NoError(t, err) quantity, _ := resource.ParseQuantity("10240G") assert.Equal(t, clusterResourceQuotaSpec.Quota.Hard["requests.storage"], quantity) + } else if extResource.Kind == "Noobaa" { + var extNoobaaSpec, mockNoobaaSpec nbv1.NooBaaSpec + err = json.Unmarshal(extResource.Data, &extNoobaaSpec) + assert.NoError(t, err) + err = json.Unmarshal([]byte(mockResoruce.Data), &mockNoobaaSpec) + assert.NoError(t, err) + + assert.Equal(t, mockNoobaaSpec.JoinSecret, extNoobaaSpec.JoinSecret) } else { - assert.Equal(t, string(extResource.Data), string(data)) + assert.Equal(t, extResource.Data, mockResoruce.Data) } assert.Equal(t, extResource.Kind, mockResoruce.Kind) @@ -588,7 +653,7 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) { "ceph-rbd-storageclass": { Name: "ceph-rbd", Kind: "StorageClass", - Data: map[string]string{ + Data: mustMarshal(map[string]string{ "clusterID": serverNamespace, "pool": "cephblockpool", "radosnamespace": "cephradosnamespace", @@ -598,37 +663,37 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) { "csi.storage.k8s.io/provisioner-secret-name": "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c", "csi.storage.k8s.io/node-stage-secret-name": "ceph-client-node-8d40b6be71600457b5dec219d2ce2d4c", "csi.storage.k8s.io/controller-expand-secret-name": "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c", - }, + }), }, "ceph-rbd-volumesnapshotclass": { Name: "ceph-rbd", Kind: "VolumeSnapshotClass", - Data: map[string]string{ + Data: mustMarshal(map[string]string{ "csi.storage.k8s.io/snapshotter-secret-name": "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c", - }, + }), }, "ceph-rbd-volumegroupsnapshotclass": { Name: "ceph-rbd", Kind: "VolumeGroupSnapshotClass", - Data: map[string]string{ + Data: mustMarshal(map[string]string{ "csi.storage.k8s.io/group-snapshotter-secret-name": "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c", - }, + }), }, "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c": { Name: "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c", Kind: "Secret", - Data: map[string]string{ + Data: mustMarshal(map[string]string{ "userID": "3de200d5c23524a4612bde1fdbeb717e", "userKey": "AQADw/hhqBOcORAAJY3fKIvte++L/zYhASjYPQ==", - }, + }), }, "ceph-client-node-8d40b6be71600457b5dec219d2ce2d4c": { Name: "ceph-client-node-8d40b6be71600457b5dec219d2ce2d4c", Kind: "Secret", - Data: map[string]string{ + Data: mustMarshal(map[string]string{ "userID": "995e66248ad3e8642de868f461cdd827", "userKey": "AQADw/hhqBOcORAAJY3fKIvte++L/zYhASjYPQ==", - }, + }), }, "ceph-rbd-clientprofile": { Name: "ceph-rbd", @@ -645,7 +710,7 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) { "cephfs-storageclass": { Name: "cephfs", Kind: "StorageClass", - Data: map[string]string{ + Data: mustMarshal(map[string]string{ "clusterID": "8d26c7378c1b0ec9c2455d1c3601c4cd", "fsName": "myfs", "subvolumegroupname": "cephFilesystemSubVolumeGroup", @@ -653,46 +718,46 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) { "csi.storage.k8s.io/provisioner-secret-name": "ceph-client-provisioner-0e8555e6556f70d23a61675af44e880c", "csi.storage.k8s.io/node-stage-secret-name": "ceph-client-node-0e8555e6556f70d23a61675af44e880c", "csi.storage.k8s.io/controller-expand-secret-name": "ceph-client-provisioner-0e8555e6556f70d23a61675af44e880c", - }, + }), }, "cephfs-volumesnapshotclass": { Name: "cephfs", Kind: "VolumeSnapshotClass", - Data: map[string]string{ + Data: mustMarshal(map[string]string{ "csi.storage.k8s.io/snapshotter-secret-name": "ceph-client-provisioner-0e8555e6556f70d23a61675af44e880c", - }, + }), }, "cephfs-volumegroupsnapshotclass": { Name: "cephfs", Kind: "VolumeGroupSnapshotClass", - Data: map[string]string{ + Data: mustMarshal(map[string]string{ "csi.storage.k8s.io/group-snapshotter-secret-name": "ceph-client-provisioner-0e8555e6556f70d23a61675af44e880c", - }, + }), }, "ceph-client-provisioner-0e8555e6556f70d23a61675af44e880c": { Name: "ceph-client-provisioner-0e8555e6556f70d23a61675af44e880c", Kind: "Secret", - Data: map[string]string{ + Data: mustMarshal(map[string]string{ "adminID": "4ffcb503ff8044c8699dac415f82d604", "adminKey": "AQADw/hhqBOcORAAJY3fKIvte++L/zYhASjYPQ==", - }, + }), }, "ceph-client-node-0e8555e6556f70d23a61675af44e880c": { Name: "ceph-client-node-0e8555e6556f70d23a61675af44e880c", Kind: "Secret", - Data: map[string]string{ + Data: mustMarshal(map[string]string{ "adminID": "1b042fcc8812fe4203689eec38fdfbfa", "adminKey": "AQADw/hhqBOcORAAJY3fKIvte++L/zYhASjYPQ==", - }, + }), }, "cephFilesystemSubVolumeGroup": { Name: "cephFilesystemSubVolumeGroup", Kind: "CephFilesystemSubVolumeGroup", - Data: map[string]string{ + Data: mustMarshal(map[string]string{ "filesystemName": "myfs", - }, + }), }, "cephfs-clientprofile": { @@ -1004,9 +1069,8 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) { mockResoruce, ok := mockBlockPoolClaimExtR[name] assert.True(t, ok) - data, err := json.Marshal(mockResoruce.Data) assert.NoError(t, err) - assert.Equal(t, string(extResource.Data), string(data)) + assert.Equal(t, extResource.Data, mockResoruce.Data) assert.Equal(t, extResource.Kind, mockResoruce.Kind) assert.Equal(t, extResource.Name, mockResoruce.Name) } @@ -1035,9 +1099,8 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) { mockResoruce, ok := mockShareFilesystemClaimExtR[name] assert.True(t, ok) - data, err := json.Marshal(mockResoruce.Data) assert.NoError(t, err) - assert.Equal(t, string(extResource.Data), string(data)) + assert.Equal(t, extResource.Data, mockResoruce.Data) assert.Equal(t, extResource.Kind, mockResoruce.Kind) assert.Equal(t, extResource.Name, mockResoruce.Name) } From da969b8fc243d677c958c10c6524daf265f586b5 Mon Sep 17 00:00:00 2001 From: Kaustav Majumder Date: Mon, 22 Jul 2024 22:21:50 +0530 Subject: [PATCH 2/2] generated changes Signed-off-by: Kaustav Majumder --- config/rbac/role.yaml | 1 + controllers/storageconsumer/consumer_test.go | 53 ++++++++- .../storageconsumer_controller.go | 32 +++++ deploy/csv-templates/ocs-operator.csv.yaml.in | 1 + .../ocs-operator.clusterserviceversion.yaml | 1 + .../ocs-operator/manifests/provider-role.yaml | 7 ++ rbac/provider-role.yaml | 9 -- services/provider/server/consumer.go | 47 -------- services/provider/server/server.go | 59 ++++------ services/provider/server/server_test.go | 109 +++++++++--------- 10 files changed, 170 insertions(+), 149 deletions(-) diff --git a/config/rbac/role.yaml b/config/rbac/role.yaml index a8cb9903ae..454dfb60d9 100644 --- a/config/rbac/role.yaml +++ b/config/rbac/role.yaml @@ -188,6 +188,7 @@ rules: - apiGroups: - noobaa.io resources: + - noobaaaccounts - noobaas verbs: - create diff --git a/controllers/storageconsumer/consumer_test.go b/controllers/storageconsumer/consumer_test.go index 51abe8391c..d1c5ad284b 100644 --- a/controllers/storageconsumer/consumer_test.go +++ b/controllers/storageconsumer/consumer_test.go @@ -19,6 +19,9 @@ package controllers import ( "testing" + noobaaApis "github.com/noobaa/noobaa-operator/v5/pkg/apis" + "github.com/noobaa/noobaa-operator/v5/pkg/apis/noobaa/v1alpha1" + routev1 "github.com/openshift/api/route/v1" v1 "github.com/red-hat-storage/ocs-operator/api/v4/v1" ocsv1alpha1 "github.com/red-hat-storage/ocs-operator/api/v4/v1alpha1" "github.com/red-hat-storage/ocs-operator/v4/controllers/util" @@ -46,7 +49,15 @@ func createFakeScheme(t *testing.T) *runtime.Scheme { err = rookCephv1.AddToScheme(scheme) if err != nil { - assert.Fail(t, "failed to add rookCephv1scheme") + assert.Fail(t, "failed to add rookCephv1 scheme") + } + err = routev1.AddToScheme(scheme) + if err != nil { + assert.Fail(t, "failed to add routev1 scheme") + } + err = noobaaApis.AddToScheme(scheme) + if err != nil { + assert.Fail(t, "failed to add nbapis scheme") } return scheme @@ -90,7 +101,23 @@ func TestCephName(t *testing.T) { Name: "cephfs", Phase: "Ready", }, + { + Kind: "NooBaaAccount", + Name: "consumer-acc", + Phase: "Ready", + }, }, + Client: ocsv1alpha1.ClientStatus{ + ClusterID: "consumer", + }, + }, + } + r.noobaaAccount = &v1alpha1.NooBaaAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "consumer-acc", + }, + Status: v1alpha1.NooBaaAccountStatus{ + Phase: v1alpha1.NooBaaAccountPhaseReady, }, } _, err := r.reconcilePhases() @@ -102,6 +129,11 @@ func TestCephName(t *testing.T) { Name: "healthchecker", Phase: "Ready", }, + { + Kind: "NooBaaAccount", + Name: "consumer-acc", + Phase: "Ready", + }, } assert.Equal(t, r.storageConsumer.Status.CephResources, want) @@ -138,9 +170,23 @@ func TestCephName(t *testing.T) { Name: "healthchecker", Phase: "Error", }, + { + Kind: "NooBaaAccount", + Name: "consumer-acc", + Phase: "Error", + }, }, }, } + r.noobaaAccount = &v1alpha1.NooBaaAccount{ + ObjectMeta: metav1.ObjectMeta{ + Name: "consumer-acc", + }, + Status: v1alpha1.NooBaaAccountStatus{ + Phase: v1alpha1.NooBaaAccountPhaseRejected, + }, + } + _, err = r.reconcilePhases() assert.NoError(t, err) @@ -150,6 +196,11 @@ func TestCephName(t *testing.T) { Name: "healthchecker", Phase: "Error", }, + { + Kind: "NooBaaAccount", + Name: "consumer-acc", + Phase: "Rejected", + }, } assert.Equal(t, r.storageConsumer.Status.CephResources, want) } diff --git a/controllers/storageconsumer/storageconsumer_controller.go b/controllers/storageconsumer/storageconsumer_controller.go index 97de086029..7237466f2e 100644 --- a/controllers/storageconsumer/storageconsumer_controller.go +++ b/controllers/storageconsumer/storageconsumer_controller.go @@ -21,9 +21,11 @@ import ( "crypto/md5" "encoding/hex" "encoding/json" + "fmt" "github.com/go-logr/logr" "github.com/red-hat-storage/ocs-operator/api/v4/v1alpha1" + "github.com/red-hat-storage/ocs-operator/v4/controllers/util" rookCephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -36,6 +38,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/handler" "sigs.k8s.io/controller-runtime/pkg/predicate" + nbv1 "github.com/noobaa/noobaa-operator/v5/pkg/apis/noobaa/v1alpha1" ocsv1alpha1 "github.com/red-hat-storage/ocs-operator/api/v4/v1alpha1" "sigs.k8s.io/controller-runtime/pkg/reconcile" ) @@ -59,12 +62,14 @@ type StorageConsumerReconciler struct { storageConsumer *ocsv1alpha1.StorageConsumer cephClientHealthChecker *rookCephv1.CephClient namespace string + noobaaAccount *nbv1.NooBaaAccount } //+kubebuilder:rbac:groups=ocs.openshift.io,resources=storageconsumers,verbs=get;list;watch;create;update;patch;delete //+kubebuilder:rbac:groups=ceph.rook.io,resources=cephclients,verbs=get;list;watch;create;update;delete //+kubebuilder:rbac:groups=ocs.openshift.io,resources=storageconsumers/status,verbs=get;update;patch // +kubebuilder:rbac:groups=ocs.openshift.io,resources=storagerequests,verbs=get;list; +// +kubebuilder:rbac:groups=noobaa.io,resources=noobaaaccounts,verbs=get;list;watch;create;update;delete // Reconcile reads that state of the cluster for a StorageConsumer object and makes changes based on the state read // and what is in the StorageConsumer.Spec @@ -127,6 +132,10 @@ func (r *StorageConsumerReconciler) initReconciler(request reconcile.Request) { r.cephClientHealthChecker = &rookCephv1.CephClient{} r.cephClientHealthChecker.Name = GenerateHashForCephClient(r.storageConsumer.Name, "global") r.cephClientHealthChecker.Namespace = r.namespace + + r.noobaaAccount = &nbv1.NooBaaAccount{} + r.noobaaAccount.Name = "noobaa-remote-" + r.storageConsumer.Name + r.noobaaAccount.Namespace = r.storageConsumer.Namespace } func (r *StorageConsumerReconciler) reconcilePhases() (reconcile.Result, error) { @@ -145,6 +154,10 @@ func (r *StorageConsumerReconciler) reconcilePhases() (reconcile.Result, error) return reconcile.Result{}, err } + if err := r.reconcileNoobaaAccount(); err != nil { + return reconcile.Result{}, err + } + cephResourcesReady := true for _, cephResource := range r.storageConsumer.Status.CephResources { if cephResource.Phase != "Ready" { @@ -205,6 +218,25 @@ func (r *StorageConsumerReconciler) reconcileCephClientHealthChecker() error { return nil } +func (r *StorageConsumerReconciler) reconcileNoobaaAccount() error { + _, err := ctrl.CreateOrUpdate(r.ctx, r.Client, r.noobaaAccount, func() error { + if err := r.own(r.noobaaAccount); err != nil { + return err + } + // the following annotation will enable noobaa-operator to create a auth_token secret based on this account + util.AddAnnotation(r.noobaaAccount, "remote-operator", "true") + return nil + }) + if err != nil { + return fmt.Errorf("failed to create noobaa account for storageConsumer %v: %v", r.storageConsumer.Name, err) + } + + phase := string(r.noobaaAccount.Status.Phase) + r.setCephResourceStatus(r.noobaaAccount.Name, "NooBaaAccount", phase, nil) + + return nil +} + func (r *StorageConsumerReconciler) setCephResourceStatus(name string, kind string, phase string, cephClients map[string]string) { cephResourceSpec := ocsv1alpha1.CephResourcesSpec{ Name: name, diff --git a/deploy/csv-templates/ocs-operator.csv.yaml.in b/deploy/csv-templates/ocs-operator.csv.yaml.in index eabea619a9..f553b22fea 100644 --- a/deploy/csv-templates/ocs-operator.csv.yaml.in +++ b/deploy/csv-templates/ocs-operator.csv.yaml.in @@ -359,6 +359,7 @@ spec: - apiGroups: - noobaa.io resources: + - noobaaaccounts - noobaas verbs: - create diff --git a/deploy/ocs-operator/manifests/ocs-operator.clusterserviceversion.yaml b/deploy/ocs-operator/manifests/ocs-operator.clusterserviceversion.yaml index 09b660ec8c..d78c5b2be3 100644 --- a/deploy/ocs-operator/manifests/ocs-operator.clusterserviceversion.yaml +++ b/deploy/ocs-operator/manifests/ocs-operator.clusterserviceversion.yaml @@ -368,6 +368,7 @@ spec: - apiGroups: - noobaa.io resources: + - noobaaaccounts - noobaas verbs: - create diff --git a/deploy/ocs-operator/manifests/provider-role.yaml b/deploy/ocs-operator/manifests/provider-role.yaml index aea3ada2e5..b1cb627919 100644 --- a/deploy/ocs-operator/manifests/provider-role.yaml +++ b/deploy/ocs-operator/manifests/provider-role.yaml @@ -68,3 +68,10 @@ rules: verbs: - get - list +- apiGroups: + - route.openshift.io + resources: + - routes + verbs: + - get + - list diff --git a/rbac/provider-role.yaml b/rbac/provider-role.yaml index 260b5d135a..b1cb627919 100644 --- a/rbac/provider-role.yaml +++ b/rbac/provider-role.yaml @@ -68,15 +68,6 @@ rules: verbs: - get - list -- apiGroups: - - noobaa.io - resources: - - noobaaaccounts - verbs: - - get - - list - - create - - delete - apiGroups: - route.openshift.io resources: diff --git a/services/provider/server/consumer.go b/services/provider/server/consumer.go index 9415b8f014..6e050257fd 100644 --- a/services/provider/server/consumer.go +++ b/services/provider/server/consumer.go @@ -4,12 +4,9 @@ import ( "context" "errors" "fmt" - "strings" "sync" - nbv1 "github.com/noobaa/noobaa-operator/v5/pkg/apis/noobaa/v1alpha1" ocsv1alpha1 "github.com/red-hat-storage/ocs-operator/api/v4/v1alpha1" - "github.com/red-hat-storage/ocs-operator/v4/controllers/util" ifaces "github.com/red-hat-storage/ocs-operator/v4/services/provider/interfaces" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -229,47 +226,3 @@ func (c *ocsConsumerManager) UpdateConsumerStatus(ctx context.Context, id string klog.Infof("successfully updated Status for StorageConsumer %v", consumerObj.Name) return nil } - -func (c *ocsConsumerManager) CreateNoobaaAccount(ctx context.Context, id string) error { - - consumerObj, err := c.Get(ctx, id) - if err != nil { - return err - } - consumerClusterID := strings.TrimPrefix(consumerObj.Name, "storageconsumer-") - if consumerClusterID != "" && len(consumerClusterID) == 0 { - return fmt.Errorf("failed to get clusterID from consumerResource Name: %s %v", consumerObj.Name, err) - } - - noobaaAccountName := fmt.Sprintf("noobaa-remote-%s", consumerClusterID) - nbAccountObj := &nbv1.NooBaaAccount{} - nbAccountObj.Name = noobaaAccountName - nbAccountObj.Namespace = consumerObj.Namespace - // the following annotation will enable noobaa-operator to create a auth_token secret based on this account - util.AddAnnotation(nbAccountObj, "remote-operator", "true") - - err = c.client.Create(ctx, nbAccountObj) - if err != nil { - return fmt.Errorf("failed to create noobaa account for storageConsumer %v: %v", consumerObj.Name, err) - } - return nil -} - -func (c *ocsConsumerManager) DeleteNoobaaAccount(ctx context.Context, id string) error { - consumerObj, err := c.Get(ctx, id) - if err != nil { - return err - } - clusterID := strings.TrimPrefix(consumerObj.Name, "storageconsumer-") - if clusterID != "" && len(clusterID) == 0 { - return fmt.Errorf("failed to get clusterID from consumerResource Name: %s %v", consumerObj.Name, err) - } - noobaaAccountName := fmt.Sprintf("noobaa-remote-%s", clusterID) - nbAccountObj := &nbv1.NooBaaAccount{} - nbAccountObj.Name = noobaaAccountName - nbAccountObj.Namespace = consumerObj.Namespace - if err := c.client.Delete(ctx, nbAccountObj); err != nil { - return fmt.Errorf("failed to delete Noobaa account %q. %v", nbAccountObj.Name, err) - } - return nil -} diff --git a/services/provider/server/server.go b/services/provider/server/server.go index f3f69dd7af..a41b934f59 100644 --- a/services/provider/server/server.go +++ b/services/provider/server/server.go @@ -19,7 +19,6 @@ import ( "time" "github.com/blang/semver/v4" - nbapis "github.com/noobaa/noobaa-operator/v5/pkg/apis" nbv1 "github.com/noobaa/noobaa-operator/v5/pkg/apis/noobaa/v1alpha1" quotav1 "github.com/openshift/api/quota/v1" routev1 "github.com/openshift/api/route/v1" @@ -151,10 +150,6 @@ func (s *OCSProviderServer) AcknowledgeOnboarding(ctx context.Context, req *pb.A } return nil, status.Errorf(codes.Internal, "Failed to update the storageConsumer. %v", err) } - // create noobaa account CR - if err := s.consumerManager.CreateNoobaaAccount(ctx, req.StorageConsumerUUID); err != nil { - return nil, status.Errorf(codes.Internal, "Failed to create noobaa account for storageconsumer. %v", err) - } return &pb.AcknowledgeOnboardingResponse{}, nil } @@ -204,12 +199,7 @@ func (s *OCSProviderServer) GetStorageConfig(ctx context.Context, req *pb.Storag // OffboardConsumer RPC call to delete the StorageConsumer CR func (s *OCSProviderServer) OffboardConsumer(ctx context.Context, req *pb.OffboardConsumerRequest) (*pb.OffboardConsumerResponse, error) { - // remove noobaa account - err := s.consumerManager.DeleteNoobaaAccount(ctx, req.StorageConsumerUUID) - if err != nil { - return nil, status.Errorf(codes.Internal, "failed to delete noobaaAccount resource with the provided UUID. %v", err) - } - err = s.consumerManager.Delete(ctx, req.StorageConsumerUUID) + err := s.consumerManager.Delete(ctx, req.StorageConsumerUUID) if err != nil { return nil, status.Errorf(codes.Internal, "failed to delete storageConsumer resource with the provided UUID. %v", err) } @@ -267,10 +257,6 @@ func newClient() (client.Client, error) { if err != nil { return nil, fmt.Errorf("failed to add routev1 to scheme. %v", err) } - err = nbapis.AddToScheme(scheme) - if err != nil { - return nil, fmt.Errorf("failed to add nbapis to scheme. %v", err) - } config, err := config.GetConfig() if err != nil { @@ -445,26 +431,25 @@ func (s *OCSProviderServer) getExternalResources(ctx context.Context, consumerRe } // Fetch noobaa remote secret and management address and append to extResources + consumerName := consumerResource.Name noobaaOperatorSecret := &v1.Secret{} - clusterID := strings.TrimPrefix(consumerResource.Name, "storageconsumer-") - if clusterID != "" && len(clusterID) == 0 { - return nil, fmt.Errorf("failed to get clusterID from consumerResource Name: %s %v", consumerResource.Name, err) - } + noobaaOperatorSecret.Name = fmt.Sprintf("noobaa-account-%s", consumerName) + noobaaOperatorSecret.Namespace = s.namespace - noobaaOperatorSecretName := fmt.Sprintf("noobaa-remote-join-secret-%s", clusterID) - err = s.client.Get(ctx, types.NamespacedName{Name: noobaaOperatorSecretName, Namespace: s.namespace}, noobaaOperatorSecret) - if err != nil { - return nil, fmt.Errorf("failed to get %s secret. %v", noobaaOperatorSecretName, err) + if err := s.client.Get(ctx, client.ObjectKeyFromObject(noobaaOperatorSecret), noobaaOperatorSecret); err != nil { + return nil, fmt.Errorf("failed to get %s secret. %v", noobaaOperatorSecret.Name, err) } authToken, ok := noobaaOperatorSecret.Data["auth_token"] if !ok || len(authToken) == 0 { - return nil, fmt.Errorf("auth_token not found in %s secret", noobaaOperatorSecretName) + return nil, fmt.Errorf("auth_token not found in %s secret", noobaaOperatorSecret.Name) } noobaMgmtRoute := &routev1.Route{} - err = s.client.Get(ctx, types.NamespacedName{Name: "noobaa-mgmt", Namespace: s.namespace}, noobaMgmtRoute) - if err != nil { + noobaMgmtRoute.Name = "noobaa-mgmt" + noobaMgmtRoute.Namespace = s.namespace + + if err = s.client.Get(ctx, client.ObjectKeyFromObject(noobaMgmtRoute), noobaMgmtRoute); err != nil { return nil, fmt.Errorf("failed to get noobaa-mgmt route. %v", err) } if noobaMgmtRoute.Status.Ingress == nil || len(noobaMgmtRoute.Status.Ingress) == 0 { @@ -475,27 +460,23 @@ func (s *OCSProviderServer) getExternalResources(ctx context.Context, consumerRe if noobaaMgmtAddress == "" { return nil, fmt.Errorf("no Host found in noobaa-mgmt route Ingress") } - joinSecret := &corev1.Secret{ - Data: map[string][]byte{ - "auth_token": authToken, - "mgmt_addr": []byte(noobaaMgmtAddress), - }, - } extR = append(extR, &pb.ExternalResource{ Name: "noobaa-remote-join-secret", Kind: "Secret", - Data: mustMarshal(joinSecret), + Data: mustMarshal(map[string][]byte{ + "auth_token": authToken, + "mgmt_addr": []byte(noobaaMgmtAddress), + }), }) - noobaaSpec := &nbv1.NooBaaSpec{ - JoinSecret: &v1.SecretReference{ - Name: "noobaa-remote-join-secret", - }, - } extR = append(extR, &pb.ExternalResource{ Name: "noobaa-remote", Kind: "Noobaa", - Data: mustMarshal(noobaaSpec), + Data: mustMarshal(&nbv1.NooBaaSpec{ + JoinSecret: &v1.SecretReference{ + Name: "noobaa-remote-join-secret", + }, + }), }) return extR, nil } diff --git a/services/provider/server/server_test.go b/services/provider/server/server_test.go index e841929f30..f103724990 100644 --- a/services/provider/server/server_test.go +++ b/services/provider/server/server_test.go @@ -64,65 +64,63 @@ var noobaaSpec = &nbv1.NooBaaSpec{ }, } -var joinSecret = &v1.Secret{ - Data: map[string][]byte{ - "auth_token": []byte("authToken"), - "mgmt_addr": []byte("noobaaMgmtAddress"), - }, +var joinSecret = map[string][]byte{ + "auth_token": []byte("authToken"), + "mgmt_addr": []byte("noobaaMgmtAddress"), } var mockExtR = map[string]*externalResource{ "rook-ceph-mon-endpoints": { Name: "rook-ceph-mon-endpoints", Kind: "ConfigMap", - Data: mustMarshal(map[string]string{ + Data: map[string]string{ "data": "a=10.99.45.27:6789", "maxMonId": "0", "mapping": "{}", - }), + }, }, "rook-ceph-mon": { Name: "rook-ceph-mon", Kind: "Secret", - Data: mustMarshal(map[string]string{ + Data: map[string]string{ "ceph-username": "client.995e66248ad3e8642de868f461cdd827", "fsid": "b88c2d78-9de9-4227-9313-a63f62f78743", "mon-secret": "mon-secret", "ceph-secret": "AQADw/hhqBOcORAAJY3fKIvte++L/zYhASjYPQ==", - }), + }, }, "monitoring-endpoint": { Name: "monitoring-endpoint", Kind: "CephCluster", - Data: mustMarshal(map[string]string{ + Data: map[string]string{ "MonitoringEndpoint": "10.105.164.231", "MonitoringPort": "9283", - }), + }, }, "rook-ceph-client-995e66248ad3e8642de868f461cdd827": { Name: "rook-ceph-client-995e66248ad3e8642de868f461cdd827", Kind: "Secret", - Data: mustMarshal(map[string]string{ + Data: map[string]string{ "userID": "995e66248ad3e8642de868f461cdd827", "userKey": "AQADw/hhqBOcORAAJY3fKIvte++L/zYhASjYPQ==", - }), + }, }, "QuotaForConsumer": { Name: "QuotaForConsumer", Kind: "ClusterResourceQuota", - Data: mustMarshal(map[string]string{ + Data: map[string]string{ "QuotaForConsumer": fmt.Sprintf("%+v\n", clusterResourceQuotaSpec), - }), + }, }, "noobaa-remote-join-secret": { Name: "noobaa-remote-join-secret", Kind: "Secret", - Data: mustMarshal(joinSecret), + Data: joinSecret, }, "noobaa-remote": { Name: "noobaa-remote", Kind: "Noobaa", - Data: mustMarshal(noobaaSpec), + Data: noobaaSpec, }, "monitor-endpoints": { Name: "monitor-endpoints", @@ -289,14 +287,14 @@ func TestGetExternalResources(t *testing.T) { } noobaaRemoteJoinSecretConsumer := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{Name: "noobaa-remote-join-secret-consumer", Namespace: server.namespace}, + ObjectMeta: metav1.ObjectMeta{Name: "noobaa-account-consumer", Namespace: server.namespace}, Data: map[string][]byte{ "auth_token": []byte("authToken"), }, } noobaaRemoteJoinSecretConsumer6 := &v1.Secret{ - ObjectMeta: metav1.ObjectMeta{Name: "noobaa-remote-join-secret-consumer6", Namespace: server.namespace}, + ObjectMeta: metav1.ObjectMeta{Name: "noobaa-account-consumer6", Namespace: server.namespace}, Data: map[string][]byte{ "auth_token": []byte("authToken"), }, @@ -339,17 +337,19 @@ func TestGetExternalResources(t *testing.T) { mockResoruce, ok := mockExtR[extResource.Name] assert.True(t, ok) - assert.NoError(t, err) if extResource.Kind == "Noobaa" { var extNoobaaSpec, mockNoobaaSpec nbv1.NooBaaSpec err = json.Unmarshal(extResource.Data, &extNoobaaSpec) assert.NoError(t, err) - err = json.Unmarshal([]byte(mockResoruce.Data), &mockNoobaaSpec) + data, err := json.Marshal(mockResoruce.Data) assert.NoError(t, err) - - assert.Equal(t, mockNoobaaSpec.JoinSecret, extNoobaaSpec.JoinSecret) + err = json.Unmarshal(data, &mockNoobaaSpec) + assert.NoError(t, err) + assert.Equal(t, extNoobaaSpec.JoinSecret, mockNoobaaSpec.JoinSecret) } else { - assert.Equal(t, extResource.Data, mockResoruce.Data) + data, err := json.Marshal(mockResoruce.Data) + assert.NoError(t, err) + assert.Equal(t, string(extResource.Data), string(data)) } assert.Equal(t, extResource.Kind, mockResoruce.Kind) assert.Equal(t, extResource.Name, mockResoruce.Name) @@ -368,7 +368,6 @@ func TestGetExternalResources(t *testing.T) { mockResoruce, ok := mockExtR[extResource.Name] assert.True(t, ok) - assert.NoError(t, err) if extResource.Kind == "ClusterResourceQuota" { var clusterResourceQuotaSpec quotav1.ClusterResourceQuotaSpec err = json.Unmarshal([]byte(extResource.Data), &clusterResourceQuotaSpec) @@ -379,12 +378,15 @@ func TestGetExternalResources(t *testing.T) { var extNoobaaSpec, mockNoobaaSpec nbv1.NooBaaSpec err = json.Unmarshal(extResource.Data, &extNoobaaSpec) assert.NoError(t, err) - err = json.Unmarshal([]byte(mockResoruce.Data), &mockNoobaaSpec) + data, err := json.Marshal(mockResoruce.Data) + assert.NoError(t, err) + err = json.Unmarshal(data, &mockNoobaaSpec) assert.NoError(t, err) - assert.Equal(t, mockNoobaaSpec.JoinSecret, extNoobaaSpec.JoinSecret) } else { - assert.Equal(t, extResource.Data, mockResoruce.Data) + data, err := json.Marshal(mockResoruce.Data) + assert.NoError(t, err) + assert.Equal(t, string(extResource.Data), string(data)) } assert.Equal(t, extResource.Kind, mockResoruce.Kind) @@ -653,7 +655,7 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) { "ceph-rbd-storageclass": { Name: "ceph-rbd", Kind: "StorageClass", - Data: mustMarshal(map[string]string{ + Data: map[string]string{ "clusterID": serverNamespace, "pool": "cephblockpool", "radosnamespace": "cephradosnamespace", @@ -663,37 +665,37 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) { "csi.storage.k8s.io/provisioner-secret-name": "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c", "csi.storage.k8s.io/node-stage-secret-name": "ceph-client-node-8d40b6be71600457b5dec219d2ce2d4c", "csi.storage.k8s.io/controller-expand-secret-name": "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c", - }), + }, }, "ceph-rbd-volumesnapshotclass": { Name: "ceph-rbd", Kind: "VolumeSnapshotClass", - Data: mustMarshal(map[string]string{ + Data: map[string]string{ "csi.storage.k8s.io/snapshotter-secret-name": "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c", - }), + }, }, "ceph-rbd-volumegroupsnapshotclass": { Name: "ceph-rbd", Kind: "VolumeGroupSnapshotClass", - Data: mustMarshal(map[string]string{ + Data: map[string]string{ "csi.storage.k8s.io/group-snapshotter-secret-name": "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c", - }), + }, }, "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c": { Name: "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c", Kind: "Secret", - Data: mustMarshal(map[string]string{ + Data: map[string]string{ "userID": "3de200d5c23524a4612bde1fdbeb717e", "userKey": "AQADw/hhqBOcORAAJY3fKIvte++L/zYhASjYPQ==", - }), + }, }, "ceph-client-node-8d40b6be71600457b5dec219d2ce2d4c": { Name: "ceph-client-node-8d40b6be71600457b5dec219d2ce2d4c", Kind: "Secret", - Data: mustMarshal(map[string]string{ + Data: map[string]string{ "userID": "995e66248ad3e8642de868f461cdd827", "userKey": "AQADw/hhqBOcORAAJY3fKIvte++L/zYhASjYPQ==", - }), + }, }, "ceph-rbd-clientprofile": { Name: "ceph-rbd", @@ -710,7 +712,7 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) { "cephfs-storageclass": { Name: "cephfs", Kind: "StorageClass", - Data: mustMarshal(map[string]string{ + Data: map[string]string{ "clusterID": "8d26c7378c1b0ec9c2455d1c3601c4cd", "fsName": "myfs", "subvolumegroupname": "cephFilesystemSubVolumeGroup", @@ -718,46 +720,46 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) { "csi.storage.k8s.io/provisioner-secret-name": "ceph-client-provisioner-0e8555e6556f70d23a61675af44e880c", "csi.storage.k8s.io/node-stage-secret-name": "ceph-client-node-0e8555e6556f70d23a61675af44e880c", "csi.storage.k8s.io/controller-expand-secret-name": "ceph-client-provisioner-0e8555e6556f70d23a61675af44e880c", - }), + }, }, "cephfs-volumesnapshotclass": { Name: "cephfs", Kind: "VolumeSnapshotClass", - Data: mustMarshal(map[string]string{ + Data: map[string]string{ "csi.storage.k8s.io/snapshotter-secret-name": "ceph-client-provisioner-0e8555e6556f70d23a61675af44e880c", - }), + }, }, "cephfs-volumegroupsnapshotclass": { Name: "cephfs", Kind: "VolumeGroupSnapshotClass", - Data: mustMarshal(map[string]string{ + Data: map[string]string{ "csi.storage.k8s.io/group-snapshotter-secret-name": "ceph-client-provisioner-0e8555e6556f70d23a61675af44e880c", - }), + }, }, "ceph-client-provisioner-0e8555e6556f70d23a61675af44e880c": { Name: "ceph-client-provisioner-0e8555e6556f70d23a61675af44e880c", Kind: "Secret", - Data: mustMarshal(map[string]string{ + Data: map[string]string{ "adminID": "4ffcb503ff8044c8699dac415f82d604", "adminKey": "AQADw/hhqBOcORAAJY3fKIvte++L/zYhASjYPQ==", - }), + }, }, "ceph-client-node-0e8555e6556f70d23a61675af44e880c": { Name: "ceph-client-node-0e8555e6556f70d23a61675af44e880c", Kind: "Secret", - Data: mustMarshal(map[string]string{ + Data: map[string]string{ "adminID": "1b042fcc8812fe4203689eec38fdfbfa", "adminKey": "AQADw/hhqBOcORAAJY3fKIvte++L/zYhASjYPQ==", - }), + }, }, "cephFilesystemSubVolumeGroup": { Name: "cephFilesystemSubVolumeGroup", Kind: "CephFilesystemSubVolumeGroup", - Data: mustMarshal(map[string]string{ + Data: map[string]string{ "filesystemName": "myfs", - }), + }, }, "cephfs-clientprofile": { @@ -1069,8 +1071,9 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) { mockResoruce, ok := mockBlockPoolClaimExtR[name] assert.True(t, ok) + data, err := json.Marshal(mockResoruce.Data) assert.NoError(t, err) - assert.Equal(t, extResource.Data, mockResoruce.Data) + assert.Equal(t, string(extResource.Data), string(data)) assert.Equal(t, extResource.Kind, mockResoruce.Kind) assert.Equal(t, extResource.Name, mockResoruce.Name) } @@ -1098,9 +1101,9 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) { } mockResoruce, ok := mockShareFilesystemClaimExtR[name] assert.True(t, ok) - + data, err := json.Marshal(mockResoruce.Data) assert.NoError(t, err) - assert.Equal(t, extResource.Data, mockResoruce.Data) + assert.Equal(t, string(extResource.Data), string(data)) assert.Equal(t, extResource.Kind, mockResoruce.Kind) assert.Equal(t, extResource.Name, mockResoruce.Name) }