diff --git a/rbac/provider-role.yaml b/rbac/provider-role.yaml index aea3ada2e5..260b5d135a 100644 --- a/rbac/provider-role.yaml +++ b/rbac/provider-role.yaml @@ -68,3 +68,19 @@ rules: verbs: - get - list +- apiGroups: + - noobaa.io + resources: + - noobaaaccounts + verbs: + - get + - list + - create + - delete +- apiGroups: + - route.openshift.io + resources: + - routes + verbs: + - get + - list diff --git a/services/provider/server/consumer.go b/services/provider/server/consumer.go index 6e050257fd..9415b8f014 100644 --- a/services/provider/server/consumer.go +++ b/services/provider/server/consumer.go @@ -4,9 +4,12 @@ import ( "context" "errors" "fmt" + "strings" "sync" + nbv1 "github.com/noobaa/noobaa-operator/v5/pkg/apis/noobaa/v1alpha1" ocsv1alpha1 "github.com/red-hat-storage/ocs-operator/api/v4/v1alpha1" + "github.com/red-hat-storage/ocs-operator/v4/controllers/util" ifaces "github.com/red-hat-storage/ocs-operator/v4/services/provider/interfaces" kerrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -226,3 +229,47 @@ func (c *ocsConsumerManager) UpdateConsumerStatus(ctx context.Context, id string klog.Infof("successfully updated Status for StorageConsumer %v", consumerObj.Name) return nil } + +func (c *ocsConsumerManager) CreateNoobaaAccount(ctx context.Context, id string) error { + + consumerObj, err := c.Get(ctx, id) + if err != nil { + return err + } + consumerClusterID := strings.TrimPrefix(consumerObj.Name, "storageconsumer-") + if consumerClusterID != "" && len(consumerClusterID) == 0 { + return fmt.Errorf("failed to get clusterID from consumerResource Name: %s %v", consumerObj.Name, err) + } + + noobaaAccountName := fmt.Sprintf("noobaa-remote-%s", consumerClusterID) + nbAccountObj := &nbv1.NooBaaAccount{} + nbAccountObj.Name = noobaaAccountName + nbAccountObj.Namespace = consumerObj.Namespace + // the following annotation will enable noobaa-operator to create a auth_token secret based on this account + util.AddAnnotation(nbAccountObj, "remote-operator", "true") + + err = c.client.Create(ctx, nbAccountObj) + if err != nil { + return fmt.Errorf("failed to create noobaa account for storageConsumer %v: %v", consumerObj.Name, err) + } + return nil +} + +func (c *ocsConsumerManager) DeleteNoobaaAccount(ctx context.Context, id string) error { + consumerObj, err := c.Get(ctx, id) + if err != nil { + return err + } + clusterID := strings.TrimPrefix(consumerObj.Name, "storageconsumer-") + if clusterID != "" && len(clusterID) == 0 { + return fmt.Errorf("failed to get clusterID from consumerResource Name: %s %v", consumerObj.Name, err) + } + noobaaAccountName := fmt.Sprintf("noobaa-remote-%s", clusterID) + nbAccountObj := &nbv1.NooBaaAccount{} + nbAccountObj.Name = noobaaAccountName + nbAccountObj.Namespace = consumerObj.Namespace + if err := c.client.Delete(ctx, nbAccountObj); err != nil { + return fmt.Errorf("failed to delete Noobaa account %q. %v", nbAccountObj.Name, err) + } + return nil +} diff --git a/services/provider/server/consumer_test.go b/services/provider/server/consumer_test.go index ad0f30acb8..f3b60293bc 100644 --- a/services/provider/server/consumer_test.go +++ b/services/provider/server/consumer_test.go @@ -4,6 +4,7 @@ import ( "context" "testing" + routev1 "github.com/openshift/api/route/v1" opv1a1 "github.com/operator-framework/api/pkg/operators/v1alpha1" api "github.com/red-hat-storage/ocs-operator/api/v4/v1" ocsv1alpha1 "github.com/red-hat-storage/ocs-operator/api/v4/v1alpha1" @@ -13,7 +14,6 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "sigs.k8s.io/controller-runtime/pkg/client" "sigs.k8s.io/controller-runtime/pkg/client/fake" ) @@ -67,6 +67,8 @@ func newFakeClient(t *testing.T, obj ...client.Object) client.Client { err = opv1a1.AddToScheme(scheme) assert.NoError(t, err, "failed to add opv1a1 scheme") + err = routev1.AddToScheme(scheme) + assert.NoError(t, err, "failed to add routev1 scheme") return fake.NewClientBuilder(). WithScheme(scheme). WithObjects(obj...). diff --git a/services/provider/server/server.go b/services/provider/server/server.go index 1d005fba11..f3f69dd7af 100644 --- a/services/provider/server/server.go +++ b/services/provider/server/server.go @@ -19,11 +19,16 @@ import ( "time" "github.com/blang/semver/v4" + nbapis "github.com/noobaa/noobaa-operator/v5/pkg/apis" + nbv1 "github.com/noobaa/noobaa-operator/v5/pkg/apis/noobaa/v1alpha1" quotav1 "github.com/openshift/api/quota/v1" + routev1 "github.com/openshift/api/route/v1" + opv1a1 "github.com/operator-framework/api/pkg/operators/v1alpha1" ocsv1 "github.com/red-hat-storage/ocs-operator/api/v4/v1" ocsv1alpha1 "github.com/red-hat-storage/ocs-operator/api/v4/v1alpha1" controllers "github.com/red-hat-storage/ocs-operator/v4/controllers/storageconsumer" "github.com/red-hat-storage/ocs-operator/v4/controllers/util" + "github.com/red-hat-storage/ocs-operator/v4/services" pb "github.com/red-hat-storage/ocs-operator/v4/services/provider/pb" ocsVersion "github.com/red-hat-storage/ocs-operator/v4/version" rookCephv1 "github.com/rook/rook/pkg/apis/ceph.rook.io/v1" @@ -31,8 +36,6 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" csiopv1a1 "github.com/ceph/ceph-csi-operator/api/v1alpha1" - opv1a1 "github.com/operator-framework/api/pkg/operators/v1alpha1" - "github.com/red-hat-storage/ocs-operator/v4/services" "google.golang.org/grpc" "google.golang.org/grpc/codes" "google.golang.org/grpc/credentials" @@ -148,7 +151,10 @@ func (s *OCSProviderServer) AcknowledgeOnboarding(ctx context.Context, req *pb.A } return nil, status.Errorf(codes.Internal, "Failed to update the storageConsumer. %v", err) } - + // create noobaa account CR + if err := s.consumerManager.CreateNoobaaAccount(ctx, req.StorageConsumerUUID); err != nil { + return nil, status.Errorf(codes.Internal, "Failed to create noobaa account for storageconsumer. %v", err) + } return &pb.AcknowledgeOnboardingResponse{}, nil } @@ -198,12 +204,15 @@ func (s *OCSProviderServer) GetStorageConfig(ctx context.Context, req *pb.Storag // OffboardConsumer RPC call to delete the StorageConsumer CR func (s *OCSProviderServer) OffboardConsumer(ctx context.Context, req *pb.OffboardConsumerRequest) (*pb.OffboardConsumerResponse, error) { - - err := s.consumerManager.Delete(ctx, req.StorageConsumerUUID) + // remove noobaa account + err := s.consumerManager.DeleteNoobaaAccount(ctx, req.StorageConsumerUUID) + if err != nil { + return nil, status.Errorf(codes.Internal, "failed to delete noobaaAccount resource with the provided UUID. %v", err) + } + err = s.consumerManager.Delete(ctx, req.StorageConsumerUUID) if err != nil { return nil, status.Errorf(codes.Internal, "failed to delete storageConsumer resource with the provided UUID. %v", err) } - return &pb.OffboardConsumerResponse{}, nil } @@ -254,6 +263,14 @@ func newClient() (client.Client, error) { if err != nil { return nil, fmt.Errorf("failed to add ocsv1 to scheme. %v", err) } + err = routev1.AddToScheme(scheme) + if err != nil { + return nil, fmt.Errorf("failed to add routev1 to scheme. %v", err) + } + err = nbapis.AddToScheme(scheme) + if err != nil { + return nil, fmt.Errorf("failed to add nbapis to scheme. %v", err) + } config, err := config.GetConfig() if err != nil { @@ -427,6 +444,59 @@ func (s *OCSProviderServer) getExternalResources(ctx context.Context, consumerRe } + // Fetch noobaa remote secret and management address and append to extResources + noobaaOperatorSecret := &v1.Secret{} + clusterID := strings.TrimPrefix(consumerResource.Name, "storageconsumer-") + if clusterID != "" && len(clusterID) == 0 { + return nil, fmt.Errorf("failed to get clusterID from consumerResource Name: %s %v", consumerResource.Name, err) + } + + noobaaOperatorSecretName := fmt.Sprintf("noobaa-remote-join-secret-%s", clusterID) + err = s.client.Get(ctx, types.NamespacedName{Name: noobaaOperatorSecretName, Namespace: s.namespace}, noobaaOperatorSecret) + if err != nil { + return nil, fmt.Errorf("failed to get %s secret. %v", noobaaOperatorSecretName, err) + } + + authToken, ok := noobaaOperatorSecret.Data["auth_token"] + if !ok || len(authToken) == 0 { + return nil, fmt.Errorf("auth_token not found in %s secret", noobaaOperatorSecretName) + } + + noobaMgmtRoute := &routev1.Route{} + err = s.client.Get(ctx, types.NamespacedName{Name: "noobaa-mgmt", Namespace: s.namespace}, noobaMgmtRoute) + if err != nil { + return nil, fmt.Errorf("failed to get noobaa-mgmt route. %v", err) + } + if noobaMgmtRoute.Status.Ingress == nil || len(noobaMgmtRoute.Status.Ingress) == 0 { + return nil, fmt.Errorf("no Ingress available in noobaa-mgmt route") + } + + noobaaMgmtAddress := noobaMgmtRoute.Status.Ingress[0].Host + if noobaaMgmtAddress == "" { + return nil, fmt.Errorf("no Host found in noobaa-mgmt route Ingress") + } + joinSecret := &corev1.Secret{ + Data: map[string][]byte{ + "auth_token": authToken, + "mgmt_addr": []byte(noobaaMgmtAddress), + }, + } + extR = append(extR, &pb.ExternalResource{ + Name: "noobaa-remote-join-secret", + Kind: "Secret", + Data: mustMarshal(joinSecret), + }) + + noobaaSpec := &nbv1.NooBaaSpec{ + JoinSecret: &v1.SecretReference{ + Name: "noobaa-remote-join-secret", + }, + } + extR = append(extR, &pb.ExternalResource{ + Name: "noobaa-remote", + Kind: "Noobaa", + Data: mustMarshal(noobaaSpec), + }) return extR, nil } diff --git a/services/provider/server/server_test.go b/services/provider/server/server_test.go index b02d274d39..e841929f30 100644 --- a/services/provider/server/server_test.go +++ b/services/provider/server/server_test.go @@ -8,7 +8,9 @@ import ( "testing" csiopv1a1 "github.com/ceph/ceph-csi-operator/api/v1alpha1" + nbv1 "github.com/noobaa/noobaa-operator/v5/pkg/apis/noobaa/v1alpha1" quotav1 "github.com/openshift/api/quota/v1" + routev1 "github.com/openshift/api/route/v1" opv1a1 "github.com/operator-framework/api/pkg/operators/v1alpha1" ocsv1 "github.com/red-hat-storage/ocs-operator/api/v4/v1" ocsv1alpha1 "github.com/red-hat-storage/ocs-operator/api/v4/v1alpha1" @@ -56,49 +58,71 @@ var ocsSubscriptionSpec = &opv1a1.SubscriptionSpec{ Channel: "1.0", Package: "ocs-operator", } +var noobaaSpec = &nbv1.NooBaaSpec{ + JoinSecret: &v1.SecretReference{ + Name: "noobaa-remote-join-secret", + }, +} + +var joinSecret = &v1.Secret{ + Data: map[string][]byte{ + "auth_token": []byte("authToken"), + "mgmt_addr": []byte("noobaaMgmtAddress"), + }, +} var mockExtR = map[string]*externalResource{ "rook-ceph-mon-endpoints": { Name: "rook-ceph-mon-endpoints", Kind: "ConfigMap", - Data: map[string]string{ + Data: mustMarshal(map[string]string{ "data": "a=10.99.45.27:6789", "maxMonId": "0", "mapping": "{}", - }, + }), }, "rook-ceph-mon": { Name: "rook-ceph-mon", Kind: "Secret", - Data: map[string]string{ + Data: mustMarshal(map[string]string{ "ceph-username": "client.995e66248ad3e8642de868f461cdd827", "fsid": "b88c2d78-9de9-4227-9313-a63f62f78743", "mon-secret": "mon-secret", "ceph-secret": "AQADw/hhqBOcORAAJY3fKIvte++L/zYhASjYPQ==", - }, + }), }, "monitoring-endpoint": { Name: "monitoring-endpoint", Kind: "CephCluster", - Data: map[string]string{ + Data: mustMarshal(map[string]string{ "MonitoringEndpoint": "10.105.164.231", "MonitoringPort": "9283", - }, + }), }, "rook-ceph-client-995e66248ad3e8642de868f461cdd827": { Name: "rook-ceph-client-995e66248ad3e8642de868f461cdd827", Kind: "Secret", - Data: map[string]string{ + Data: mustMarshal(map[string]string{ "userID": "995e66248ad3e8642de868f461cdd827", "userKey": "AQADw/hhqBOcORAAJY3fKIvte++L/zYhASjYPQ==", - }, + }), }, "QuotaForConsumer": { Name: "QuotaForConsumer", Kind: "ClusterResourceQuota", - Data: map[string]string{ + Data: mustMarshal(map[string]string{ "QuotaForConsumer": fmt.Sprintf("%+v\n", clusterResourceQuotaSpec), - }, + }), + }, + "noobaa-remote-join-secret": { + Name: "noobaa-remote-join-secret", + Kind: "Secret", + Data: mustMarshal(joinSecret), + }, + "noobaa-remote": { + Name: "noobaa-remote", + Kind: "Noobaa", + Data: mustMarshal(noobaaSpec), }, "monitor-endpoints": { Name: "monitor-endpoints", @@ -264,9 +288,34 @@ func TestGetExternalResources(t *testing.T) { }, } + noobaaRemoteJoinSecretConsumer := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "noobaa-remote-join-secret-consumer", Namespace: server.namespace}, + Data: map[string][]byte{ + "auth_token": []byte("authToken"), + }, + } + + noobaaRemoteJoinSecretConsumer6 := &v1.Secret{ + ObjectMeta: metav1.ObjectMeta{Name: "noobaa-remote-join-secret-consumer6", Namespace: server.namespace}, + Data: map[string][]byte{ + "auth_token": []byte("authToken"), + }, + } + + noobaaMgmtRoute := &routev1.Route{ + ObjectMeta: metav1.ObjectMeta{Name: "noobaa-mgmt", Namespace: server.namespace}, + Status: routev1.RouteStatus{ + Ingress: []routev1.RouteIngress{{Host: "noobaaMgmtAddress"}}, + }, + } + assert.NoError(t, client.Create(ctx, cephClient)) assert.NoError(t, client.Create(ctx, secret)) + assert.NoError(t, client.Create(ctx, noobaaRemoteJoinSecretConsumer)) + assert.NoError(t, client.Create(ctx, noobaaRemoteJoinSecretConsumer6)) + assert.NoError(t, client.Create(ctx, noobaaMgmtRoute)) + monCm, monSc := createMonConfigMapAndSecret(server) assert.NoError(t, client.Create(ctx, monCm)) assert.NoError(t, client.Create(ctx, monSc)) @@ -290,9 +339,18 @@ func TestGetExternalResources(t *testing.T) { mockResoruce, ok := mockExtR[extResource.Name] assert.True(t, ok) - data, err := json.Marshal(mockResoruce.Data) assert.NoError(t, err) - assert.Equal(t, string(extResource.Data), string(data)) + if extResource.Kind == "Noobaa" { + var extNoobaaSpec, mockNoobaaSpec nbv1.NooBaaSpec + err = json.Unmarshal(extResource.Data, &extNoobaaSpec) + assert.NoError(t, err) + err = json.Unmarshal([]byte(mockResoruce.Data), &mockNoobaaSpec) + assert.NoError(t, err) + + assert.Equal(t, mockNoobaaSpec.JoinSecret, extNoobaaSpec.JoinSecret) + } else { + assert.Equal(t, extResource.Data, mockResoruce.Data) + } assert.Equal(t, extResource.Kind, mockResoruce.Kind) assert.Equal(t, extResource.Name, mockResoruce.Name) } @@ -310,7 +368,6 @@ func TestGetExternalResources(t *testing.T) { mockResoruce, ok := mockExtR[extResource.Name] assert.True(t, ok) - data, err := json.Marshal(mockResoruce.Data) assert.NoError(t, err) if extResource.Kind == "ClusterResourceQuota" { var clusterResourceQuotaSpec quotav1.ClusterResourceQuotaSpec @@ -318,8 +375,16 @@ func TestGetExternalResources(t *testing.T) { assert.NoError(t, err) quantity, _ := resource.ParseQuantity("10240G") assert.Equal(t, clusterResourceQuotaSpec.Quota.Hard["requests.storage"], quantity) + } else if extResource.Kind == "Noobaa" { + var extNoobaaSpec, mockNoobaaSpec nbv1.NooBaaSpec + err = json.Unmarshal(extResource.Data, &extNoobaaSpec) + assert.NoError(t, err) + err = json.Unmarshal([]byte(mockResoruce.Data), &mockNoobaaSpec) + assert.NoError(t, err) + + assert.Equal(t, mockNoobaaSpec.JoinSecret, extNoobaaSpec.JoinSecret) } else { - assert.Equal(t, string(extResource.Data), string(data)) + assert.Equal(t, extResource.Data, mockResoruce.Data) } assert.Equal(t, extResource.Kind, mockResoruce.Kind) @@ -588,7 +653,7 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) { "ceph-rbd-storageclass": { Name: "ceph-rbd", Kind: "StorageClass", - Data: map[string]string{ + Data: mustMarshal(map[string]string{ "clusterID": serverNamespace, "pool": "cephblockpool", "radosnamespace": "cephradosnamespace", @@ -598,37 +663,37 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) { "csi.storage.k8s.io/provisioner-secret-name": "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c", "csi.storage.k8s.io/node-stage-secret-name": "ceph-client-node-8d40b6be71600457b5dec219d2ce2d4c", "csi.storage.k8s.io/controller-expand-secret-name": "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c", - }, + }), }, "ceph-rbd-volumesnapshotclass": { Name: "ceph-rbd", Kind: "VolumeSnapshotClass", - Data: map[string]string{ + Data: mustMarshal(map[string]string{ "csi.storage.k8s.io/snapshotter-secret-name": "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c", - }, + }), }, "ceph-rbd-volumegroupsnapshotclass": { Name: "ceph-rbd", Kind: "VolumeGroupSnapshotClass", - Data: map[string]string{ + Data: mustMarshal(map[string]string{ "csi.storage.k8s.io/group-snapshotter-secret-name": "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c", - }, + }), }, "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c": { Name: "ceph-client-provisioner-8d40b6be71600457b5dec219d2ce2d4c", Kind: "Secret", - Data: map[string]string{ + Data: mustMarshal(map[string]string{ "userID": "3de200d5c23524a4612bde1fdbeb717e", "userKey": "AQADw/hhqBOcORAAJY3fKIvte++L/zYhASjYPQ==", - }, + }), }, "ceph-client-node-8d40b6be71600457b5dec219d2ce2d4c": { Name: "ceph-client-node-8d40b6be71600457b5dec219d2ce2d4c", Kind: "Secret", - Data: map[string]string{ + Data: mustMarshal(map[string]string{ "userID": "995e66248ad3e8642de868f461cdd827", "userKey": "AQADw/hhqBOcORAAJY3fKIvte++L/zYhASjYPQ==", - }, + }), }, "ceph-rbd-clientprofile": { Name: "ceph-rbd", @@ -645,7 +710,7 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) { "cephfs-storageclass": { Name: "cephfs", Kind: "StorageClass", - Data: map[string]string{ + Data: mustMarshal(map[string]string{ "clusterID": "8d26c7378c1b0ec9c2455d1c3601c4cd", "fsName": "myfs", "subvolumegroupname": "cephFilesystemSubVolumeGroup", @@ -653,46 +718,46 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) { "csi.storage.k8s.io/provisioner-secret-name": "ceph-client-provisioner-0e8555e6556f70d23a61675af44e880c", "csi.storage.k8s.io/node-stage-secret-name": "ceph-client-node-0e8555e6556f70d23a61675af44e880c", "csi.storage.k8s.io/controller-expand-secret-name": "ceph-client-provisioner-0e8555e6556f70d23a61675af44e880c", - }, + }), }, "cephfs-volumesnapshotclass": { Name: "cephfs", Kind: "VolumeSnapshotClass", - Data: map[string]string{ + Data: mustMarshal(map[string]string{ "csi.storage.k8s.io/snapshotter-secret-name": "ceph-client-provisioner-0e8555e6556f70d23a61675af44e880c", - }, + }), }, "cephfs-volumegroupsnapshotclass": { Name: "cephfs", Kind: "VolumeGroupSnapshotClass", - Data: map[string]string{ + Data: mustMarshal(map[string]string{ "csi.storage.k8s.io/group-snapshotter-secret-name": "ceph-client-provisioner-0e8555e6556f70d23a61675af44e880c", - }, + }), }, "ceph-client-provisioner-0e8555e6556f70d23a61675af44e880c": { Name: "ceph-client-provisioner-0e8555e6556f70d23a61675af44e880c", Kind: "Secret", - Data: map[string]string{ + Data: mustMarshal(map[string]string{ "adminID": "4ffcb503ff8044c8699dac415f82d604", "adminKey": "AQADw/hhqBOcORAAJY3fKIvte++L/zYhASjYPQ==", - }, + }), }, "ceph-client-node-0e8555e6556f70d23a61675af44e880c": { Name: "ceph-client-node-0e8555e6556f70d23a61675af44e880c", Kind: "Secret", - Data: map[string]string{ + Data: mustMarshal(map[string]string{ "adminID": "1b042fcc8812fe4203689eec38fdfbfa", "adminKey": "AQADw/hhqBOcORAAJY3fKIvte++L/zYhASjYPQ==", - }, + }), }, "cephFilesystemSubVolumeGroup": { Name: "cephFilesystemSubVolumeGroup", Kind: "CephFilesystemSubVolumeGroup", - Data: map[string]string{ + Data: mustMarshal(map[string]string{ "filesystemName": "myfs", - }, + }), }, "cephfs-clientprofile": { @@ -1004,9 +1069,8 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) { mockResoruce, ok := mockBlockPoolClaimExtR[name] assert.True(t, ok) - data, err := json.Marshal(mockResoruce.Data) assert.NoError(t, err) - assert.Equal(t, string(extResource.Data), string(data)) + assert.Equal(t, extResource.Data, mockResoruce.Data) assert.Equal(t, extResource.Kind, mockResoruce.Kind) assert.Equal(t, extResource.Name, mockResoruce.Name) } @@ -1035,9 +1099,8 @@ func TestOCSProviderServerGetStorageClaimConfig(t *testing.T) { mockResoruce, ok := mockShareFilesystemClaimExtR[name] assert.True(t, ok) - data, err := json.Marshal(mockResoruce.Data) assert.NoError(t, err) - assert.Equal(t, string(extResource.Data), string(data)) + assert.Equal(t, extResource.Data, mockResoruce.Data) assert.Equal(t, extResource.Kind, mockResoruce.Kind) assert.Equal(t, extResource.Name, mockResoruce.Name) }