From 0d019087075527b596dd611ef6ab812d8db7d269 Mon Sep 17 00:00:00 2001 From: cabbet Date: Fri, 20 Dec 2024 14:48:53 +0800 Subject: [PATCH] Update v4.6.0 --- CHANGELOG.md | 13 + Makefile | 38 +- build.sh | 15 +- cli/client/client_discover.go | 2 +- cli/client/client_helper.go | 6 +- cli/client/kube_client.go | 2 +- cli/client/kubectl_options.go | 2 +- cli/cmd/collect.go | 2 +- cli/cmd/collect_logs.go | 8 +- cli/cmd/create.go | 2 +- cli/cmd/create_backend.go | 10 +- cli/cmd/create_cert.go | 10 +- cli/cmd/delete.go | 2 +- cli/cmd/delete_backend.go | 10 +- cli/cmd/delete_cert.go | 8 +- cli/cmd/get.go | 2 +- cli/cmd/get_backend.go | 10 +- cli/cmd/get_cert.go | 8 +- cli/cmd/options/flag_options.go | 4 +- cli/cmd/root.go | 8 +- cli/cmd/update.go | 2 +- cli/cmd/update_backend.go | 10 +- cli/cmd/update_cert.go | 8 +- cli/cmd/version.go | 2 +- cli/config/config.go | 4 +- cli/helper/async_task.go | 2 +- cli/helper/command_helper.go | 2 +- cli/helper/goroutine_limit.go | 2 +- cli/helper/utils.go | 2 +- cli/main.go | 2 +- cli/resources/backend.go | 10 +- cli/resources/backend_helper.go | 4 +- cli/resources/cert.go | 10 +- cli/resources/file_logs_factory.go | 6 +- cli/resources/file_logs_factory_test.go | 6 +- cli/resources/logs.go | 8 +- cli/resources/logs_helper.go | 6 +- cli/resources/resource.go | 2 +- cli/resources/validate.go | 4 +- cmd/huawei-csi-extender/main.go | 18 +- cmd/storage-backend-controller/main.go | 16 +- cmd/storage-backend-sidecar/main.go | 20 +- connector/connector.go | 2 +- connector/connector_test.go | 6 +- connector/connector_utils.go | 8 +- connector/connector_utils_common.go | 6 +- connector/connector_utils_nvme.go | 4 +- connector/connector_utils_test.go | 2 +- connector/connector_utils_ultrapath.go | 4 +- connector/connector_utils_ultrapath_test.go | 2 +- connector/fibrechannel/fc.go | 6 +- connector/fibrechannel/fc_helper.go | 8 +- connector/fibrechannel/fc_ultrapath_helper.go | 6 +- connector/host/host_helper.go | 8 +- connector/host/host_helper_test.go | 12 +- connector/iscsi/iscsi.go | 6 +- connector/iscsi/iscsi_common.go | 6 +- connector/iscsi/iscsi_helper.go | 12 +- connector/local/local.go | 6 +- connector/local/local_helper.go | 6 +- connector/local/local_test.go | 12 +- connector/nfs/nfs.go | 4 +- connector/nfs/nfs_helper.go | 169 +- connector/nfs/nfs_test.go | 8 +- connector/nfsplus/nfs_plus.go | 4 +- connector/nfsplus/nfs_plus_helper.go | 14 +- connector/nvme/nvme.go | 6 +- connector/nvme/nvme_helper.go | 6 +- connector/nvme/nvme_test.go | 10 +- connector/roce/roce.go | 6 +- connector/roce/roce_common.go | 6 +- connector/roce/roce_helper.go | 10 +- connector/ultrapath.go | 4 +- connector/ultrapath_test.go | 2 +- connector/ultrapath_vlun.go | 2 +- connector/utils/lock/lock.go | 6 +- connector/utils/lock/lock_utils.go | 6 +- connector/utils/mount.go | 280 +++ connector/utils/utils.go | 4 +- csi/app/app.go | 4 +- csi/app/config/config.go | 7 +- csi/app/config/config_mock.go | 4 +- csi/app/options/connector.go | 2 +- csi/app/options/extender.go | 2 +- csi/app/options/k8s.go | 4 +- csi/app/options/logging.go | 2 +- csi/app/options/options.go | 2 +- csi/app/options/options_test.go | 2 +- csi/app/options/service.go | 14 +- csi/backend/backend.go | 33 +- csi/backend/backend_get.go | 8 +- csi/backend/backend_test.go | 10 +- csi/backend/cache/backend_cache.go | 4 +- csi/backend/cache/backend_cache_test.go | 8 +- csi/backend/handler/backend_cache_wrapper.go | 10 +- .../handler/backend_cache_wrapper_test.go | 14 +- csi/backend/handler/backend_fetcher.go | 8 +- csi/backend/handler/backend_fetcher_test.go | 6 +- csi/backend/handler/backend_register.go | 8 +- csi/backend/handler/backend_register_test.go | 4 +- csi/backend/handler/backend_selector.go | 8 +- csi/backend/handler/backend_selector_test.go | 4 +- csi/backend/handler/backend_service.go | 8 +- csi/backend/job/backend_sync_job.go | 8 +- csi/backend/model/model.go | 6 +- csi/backend/model/storage_pool.go | 6 +- csi/backend/model/storage_pool_test.go | 8 +- csi/backend/plugin/fusionstorage-nas.go | 36 +- csi/backend/plugin/fusionstorage-san.go | 47 +- csi/backend/plugin/fusionstorage.go | 27 +- csi/backend/plugin/oceandisk-san.go | 302 ++++ csi/backend/plugin/oceandisk.go | 125 ++ csi/backend/plugin/oceanstor-dtree.go | 54 +- csi/backend/plugin/oceanstor-nas.go | 79 +- csi/backend/plugin/oceanstor-nas_test.go | 58 +- csi/backend/plugin/oceanstor-san.go | 81 +- csi/backend/plugin/oceanstor.go | 95 +- csi/backend/plugin/plugin.go | 10 +- csi/backend/plugin/plugin_helper.go | 149 +- csi/backend/plugin/plugin_test.go | 6 +- csi/driver/controller.go | 56 +- csi/driver/controller_helper.go | 94 +- csi/driver/controller_helper_test.go | 85 +- csi/driver/driver.go | 4 +- csi/driver/identity.go | 2 +- csi/driver/node.go | 65 +- csi/garbage_collector.go | 12 +- csi/main.go | 94 +- csi/manage/manager_helper.go | 37 +- csi/manage/manager_helper_test.go | 10 +- csi/manage/nas_manager.go | 10 +- csi/manage/nas_manager_test.go | 2 +- csi/manage/san_manager.go | 22 +- csi/manage/san_manager_test.go | 23 +- csi/manage/types.go | 2 +- csi/provider/backend.go | 12 +- csi/provider/identity.go | 4 +- csi/provider/provider.go | 2 +- csi/provider/volume.go | 10 +- csi/provider/volume_test.go | 14 +- go.mod | 2 +- helm/esdk/Chart.yaml | 4 +- .../esdk/templates/huawei-csi-controller.yaml | 16 +- helm/esdk/values.yaml | 33 +- lib/drcsi/rpc/common.go | 8 +- .../huawei-csi-controller-extender.yaml | 20 +- manual/esdk/deploy/huawei-csi-controller.yaml | 18 +- manual/esdk/deploy/huawei-csi-node.yaml | 6 +- pkg/client/clientset/versioned/clientset.go | 2 +- .../versioned/fake/clientset_generated.go | 8 +- .../clientset/versioned/fake/register.go | 18 +- .../clientset/versioned/scheme/register.go | 18 +- .../v1/fake/fake_storagebackendclaim.go | 3 +- .../v1/fake/fake_storagebackendcontent.go | 3 +- .../xuanwu/v1/fake/fake_volumemodifyclaim.go | 3 +- .../v1/fake/fake_volumemodifycontent.go | 3 +- .../xuanwu/v1/fake/fake_xuanwu_client.go | 4 +- .../typed/xuanwu/v1/storagebackendclaim.go | 5 +- .../typed/xuanwu/v1/storagebackendcontent.go | 5 +- .../typed/xuanwu/v1/volumemodifyclaim.go | 5 +- .../typed/xuanwu/v1/volumemodifycontent.go | 5 +- .../typed/xuanwu/v1/xuanwu_client.go | 5 +- .../informers/externalversions/factory.go | 49 +- .../informers/externalversions/generic.go | 3 +- .../internalinterfaces/factory_interfaces.go | 5 +- .../externalversions/xuanwu/interface.go | 4 +- .../externalversions/xuanwu/v1/interface.go | 2 +- .../xuanwu/v1/storagebackendclaim.go | 11 +- .../xuanwu/v1/storagebackendcontent.go | 11 +- .../xuanwu/v1/volumemodifyclaim.go | 11 +- .../xuanwu/v1/volumemodifycontent.go | 11 +- .../listers/xuanwu/v1/storagebackendclaim.go | 4 +- .../xuanwu/v1/storagebackendcontent.go | 4 +- .../listers/xuanwu/v1/volumemodifyclaim.go | 4 +- .../listers/xuanwu/v1/volumemodifycontent.go | 4 +- pkg/constants/constants.go | 8 +- pkg/constants/storage.go | 50 +- pkg/finalizers/finalizers_test.go | 2 +- pkg/modify/claim_worker.go | 10 +- pkg/modify/claim_worker_test.go | 10 +- pkg/modify/content_worker.go | 8 +- pkg/modify/controller.go | 18 +- pkg/modify/object_worker.go | 2 +- pkg/modify/object_worker_test.go | 2 +- pkg/sidecar/controller/content_delete.go | 6 +- pkg/sidecar/controller/content_sync.go | 8 +- pkg/sidecar/controller/sidecar_controller.go | 16 +- pkg/sidecar/controller/sidecar_handler.go | 8 +- .../controller/claim_delete.go | 8 +- .../controller/claim_delete_test.go | 4 +- pkg/storage-backend/controller/claim_sync.go | 12 +- pkg/storage-backend/controller/configmap.go | 10 +- .../controller/content_delete.go | 4 +- .../controller/content_delete_test.go | 8 +- .../controller/content_sync.go | 8 +- .../controller/content_sync_test.go | 6 +- pkg/storage-backend/controller/controller.go | 12 +- pkg/storage-backend/controller/secret.go | 10 +- pkg/storage-backend/handle/storage_backend.go | 8 +- .../handle/storage_backend_test.go | 4 +- pkg/utils/claim_utils.go | 6 +- pkg/utils/content_utils.go | 6 +- pkg/utils/k8s.go | 12 +- pkg/utils/leader_election.go | 4 +- pkg/utils/notify_utils.go | 2 +- pkg/utils/utils.go | 8 +- pkg/utils/utils_test.go | 6 +- pkg/webhook/utils.go | 73 +- pkg/webhook/validate_webhook.go | 4 +- pkg/webhook/webhook.go | 17 +- pkg/webhook/webhook_cfg.go | 4 +- pkg/webhook/webhook_test.go | 10 +- proto/proto.go | 4 +- proto/proto_test.go | 4 +- storage/fusionstorage/attacher/attacher.go | 14 +- storage/fusionstorage/client/client.go | 57 +- storage/fusionstorage/client/client_host.go | 2 +- storage/fusionstorage/client/client_iscsi.go | 2 +- .../fusionstorage/client/client_namespace.go | 2 +- storage/fusionstorage/client/client_qos.go | 8 +- storage/fusionstorage/client/client_quota.go | 2 +- .../fusionstorage/client/client_snapshot.go | 2 +- storage/fusionstorage/client/client_system.go | 6 +- storage/fusionstorage/client/client_test.go | 6 +- storage/fusionstorage/client/client_volume.go | 2 +- storage/fusionstorage/smartx/smartx.go | 4 +- storage/fusionstorage/utils/utils.go | 13 +- storage/fusionstorage/volume/nas.go | 37 +- storage/fusionstorage/volume/nas_test.go | 6 +- storage/fusionstorage/volume/san.go | 35 +- storage/oceanstor/client/client.go | 1138 ------------- .../oceanstor/volume/creator/snapshot_fs.go | 81 - storage/oceanstorage/api/base.go | 24 + storage/oceanstorage/api/oceandisk.go | 52 + .../base}/attacher/attacher.go | 426 +---- .../base}/attacher/attacher_utils.go | 12 +- .../base}/attacher/attacher_utils_test.go | 4 +- storage/oceanstorage/base/client.go | 280 +++ .../base}/client_applicationtype.go | 14 +- .../client => oceanstorage/base}/client_fc.go | 26 +- .../base}/client_host.go | 32 +- .../base}/client_iscsi.go | 30 +- .../base}/client_mapping.go | 24 +- .../base}/client_qos.go | 70 +- .../base}/client_roce.go | 24 +- .../base}/client_system.go | 95 +- .../base}/client_system_types.go | 3 +- .../oceandisk/attacher/attacher.go | 324 ++++ .../oceandisk/attacher/attacher_test.go | 505 ++++++ .../oceanstorage/oceandisk/client/client.go | 167 ++ .../oceandisk/client/client_namespace.go | 610 +++++++ .../oceandisk/client/client_namespace_test.go | 1513 +++++++++++++++++ .../oceandisk/client/client_restful.go | 502 ++++++ .../oceandisk/client/client_restful_test.go | 427 +++++ .../oceandisk/client/client_test.go | 42 + .../oceanstorage/oceandisk/smartx/smartx.go | 345 ++++ storage/oceanstorage/oceandisk/volume/base.go | 146 ++ .../oceandisk/volume/base_test.go | 55 + storage/oceanstorage/oceandisk/volume/san.go | 321 ++++ .../oceanstorage/oceandisk/volume/san_test.go | 205 +++ .../oceanstor/attacher/attacher.go | 317 ++++ .../oceanstor/attacher/dorado_v6_attacher.go | 47 +- .../oceanstor/attacher/metroAttacher.go | 9 +- .../oceanstor/attacher/oceanstor_attacher.go | 49 +- .../oceanstorage/oceanstor/client/client.go | 452 +++++ .../oceanstor/client/client_clone.go | 16 +- .../oceanstor/client/client_dtree.go | 11 +- .../oceanstor/client/client_filesystem.go | 44 +- .../client/client_filesystem_response.go | 0 .../client/client_filesystem_test.go | 32 +- .../oceanstor/client/client_fssnapshot.go | 15 +- .../oceanstor/client/client_hypermetro.go | 24 +- .../oceanstor/client/client_lif.go | 20 +- .../oceanstor/client/client_lif_test.go | 24 +- .../oceanstor/client/client_lif_types.go | 0 .../oceanstor/client/client_lun.go | 53 +- .../oceanstor/client/client_luncopy.go | 16 +- .../oceanstor/client/client_lunsnapshot.go | 14 +- .../oceanstor/client/client_mock.go | 0 .../oceanstor/client/client_quota.go | 13 +- .../oceanstor/client/client_replication.go | 18 +- .../oceanstor/client/client_restful.go | 569 +++++++ .../oceanstor/client/client_test.go | 10 +- .../oceanstor/client/client_vstore.go | 22 +- .../oceanstor/client/ulrs.go | 0 .../oceanstor/clientv6/clientv6.go | 26 +- .../oceanstor/clientv6/clientv6_test.go | 10 +- .../oceanstor/clientv6/urls.go | 0 .../oceanstor/smartx/smartx.go | 41 +- .../oceanstor/volume/base.go | 42 +- .../oceanstor/volume/codes.go | 0 .../oceanstor/volume/creator/base.go | 37 +- .../oceanstor/volume/creator/clone_fs.go | 33 +- .../oceanstor/volume/creator/creator.go | 30 +- .../oceanstor/volume/creator/filesystem.go | 49 +- .../oceanstor/volume/creator/hypermetro_fs.go | 21 +- .../oceanstor/volume/creator/modify_fs.go | 96 +- .../oceanstor/volume/creator/parameter.go | 174 +- .../volume/creator/parameter_test.go | 7 +- .../oceanstor/volume/creator/snapshot_fs.go | 46 + .../oceanstor/volume/dtree.go | 15 +- .../oceanstor/volume/dtree_test.go | 6 +- .../oceanstor/volume/nas.go | 68 +- .../oceanstor/volume/nas_test.go | 0 .../oceanstor/volume/san.go | 46 +- .../oceanstor/volume/types.go | 0 utils/cert/cert.go | 202 +++ utils/flow/taskflow.go | 10 +- utils/flow/taskflow_test.go | 2 +- utils/flow/transaction_test.go | 2 +- utils/host.go | 2 +- utils/host_test.go | 2 +- utils/k8sutils/k8s_utils.go | 38 +- utils/k8sutils/pv_helper.go | 80 + utils/k8sutils/pvc_helper.go | 120 +- utils/k8sutils/pvc_helper_test.go | 68 +- utils/k8sutils/resource_accessor.go | 114 ++ utils/k8sutils/resource_accessor_test.go | 67 + utils/log/file.go | 2 +- utils/notify/stop_notify.go | 2 +- utils/retry/retry.go | 54 + utils/retry/retry_test.go | 70 + utils/utils.go | 173 +- utils/utils_test.go | 63 +- utils/version/version.go | 2 +- utils/volume.go | 11 +- utils/wwn_helper.go | 2 +- utils/wwn_helper_test.go | 2 +- 328 files changed, 10945 insertions(+), 3889 deletions(-) create mode 100644 connector/utils/mount.go create mode 100644 csi/backend/plugin/oceandisk-san.go create mode 100644 csi/backend/plugin/oceandisk.go delete mode 100644 storage/oceanstor/client/client.go delete mode 100644 storage/oceanstor/volume/creator/snapshot_fs.go create mode 100644 storage/oceanstorage/api/base.go create mode 100644 storage/oceanstorage/api/oceandisk.go rename storage/{oceanstor => oceanstorage/base}/attacher/attacher.go (52%) rename storage/{oceanstor => oceanstorage/base}/attacher/attacher_utils.go (88%) rename storage/{oceanstor => oceanstorage/base}/attacher/attacher_utils_test.go (97%) create mode 100644 storage/oceanstorage/base/client.go rename storage/{oceanstor/client => oceanstorage/base}/client_applicationtype.go (80%) rename storage/{oceanstor/client => oceanstorage/base}/client_fc.go (85%) rename storage/{oceanstor/client => oceanstorage/base}/client_host.go (90%) rename storage/{oceanstor/client => oceanstorage/base}/client_iscsi.go (84%) rename storage/{oceanstor/client => oceanstorage/base}/client_mapping.go (85%) rename storage/{oceanstor/client => oceanstorage/base}/client_qos.go (72%) rename storage/{oceanstor/client => oceanstorage/base}/client_roce.go (88%) rename storage/{oceanstor/client => oceanstorage/base}/client_system.go (64%) rename storage/{oceanstor/client => oceanstorage/base}/client_system_types.go (97%) create mode 100644 storage/oceanstorage/oceandisk/attacher/attacher.go create mode 100644 storage/oceanstorage/oceandisk/attacher/attacher_test.go create mode 100644 storage/oceanstorage/oceandisk/client/client.go create mode 100644 storage/oceanstorage/oceandisk/client/client_namespace.go create mode 100644 storage/oceanstorage/oceandisk/client/client_namespace_test.go create mode 100644 storage/oceanstorage/oceandisk/client/client_restful.go create mode 100644 storage/oceanstorage/oceandisk/client/client_restful_test.go create mode 100644 storage/oceanstorage/oceandisk/client/client_test.go create mode 100644 storage/oceanstorage/oceandisk/smartx/smartx.go create mode 100644 storage/oceanstorage/oceandisk/volume/base.go create mode 100644 storage/oceanstorage/oceandisk/volume/base_test.go create mode 100644 storage/oceanstorage/oceandisk/volume/san.go create mode 100644 storage/oceanstorage/oceandisk/volume/san_test.go create mode 100644 storage/oceanstorage/oceanstor/attacher/attacher.go rename storage/{ => oceanstorage}/oceanstor/attacher/dorado_v6_attacher.go (65%) rename storage/{ => oceanstorage}/oceanstor/attacher/metroAttacher.go (94%) rename storage/{ => oceanstorage}/oceanstor/attacher/oceanstor_attacher.go (73%) create mode 100644 storage/oceanstorage/oceanstor/client/client.go rename storage/{ => oceanstorage}/oceanstor/client/client_clone.go (89%) rename storage/{ => oceanstorage}/oceanstor/client/client_dtree.go (87%) rename storage/{ => oceanstorage}/oceanstor/client/client_filesystem.go (89%) rename storage/{ => oceanstorage}/oceanstor/client/client_filesystem_response.go (100%) rename storage/{ => oceanstorage}/oceanstor/client/client_filesystem_test.go (76%) rename storage/{ => oceanstorage}/oceanstor/client/client_fssnapshot.go (89%) rename storage/{ => oceanstorage}/oceanstor/client/client_hypermetro.go (88%) rename storage/{ => oceanstorage}/oceanstor/client/client_lif.go (75%) rename storage/{ => oceanstorage}/oceanstor/client/client_lif_test.go (82%) rename storage/{ => oceanstorage}/oceanstor/client/client_lif_types.go (100%) rename storage/{ => oceanstorage}/oceanstor/client/client_lun.go (86%) rename storage/{ => oceanstorage}/oceanstor/client/client_luncopy.go (86%) rename storage/{ => oceanstorage}/oceanstor/client/client_lunsnapshot.go (86%) rename storage/{ => oceanstorage}/oceanstor/client/client_mock.go (100%) rename storage/{ => oceanstorage}/oceanstor/client/client_quota.go (87%) rename storage/{ => oceanstorage}/oceanstor/client/client_replication.go (88%) create mode 100644 storage/oceanstorage/oceanstor/client/client_restful.go rename storage/{ => oceanstorage}/oceanstor/client/client_test.go (99%) rename storage/{ => oceanstorage}/oceanstor/client/client_vstore.go (82%) rename storage/{ => oceanstorage}/oceanstor/client/ulrs.go (100%) rename storage/{ => oceanstorage}/oceanstor/clientv6/clientv6.go (69%) rename storage/{ => oceanstorage}/oceanstor/clientv6/clientv6_test.go (88%) rename storage/{ => oceanstorage}/oceanstor/clientv6/urls.go (100%) rename storage/{ => oceanstorage}/oceanstor/smartx/smartx.go (88%) rename storage/{ => oceanstorage}/oceanstor/volume/base.go (86%) rename storage/{ => oceanstorage}/oceanstor/volume/codes.go (100%) rename storage/{ => oceanstorage}/oceanstor/volume/creator/base.go (91%) rename storage/{ => oceanstorage}/oceanstor/volume/creator/clone_fs.go (89%) rename storage/{ => oceanstorage}/oceanstor/volume/creator/creator.go (74%) rename storage/{ => oceanstorage}/oceanstor/volume/creator/filesystem.go (77%) rename storage/{ => oceanstorage}/oceanstor/volume/creator/hypermetro_fs.go (88%) rename storage/{ => oceanstorage}/oceanstor/volume/creator/modify_fs.go (63%) rename storage/{ => oceanstorage}/oceanstor/volume/creator/parameter.go (68%) rename storage/{ => oceanstorage}/oceanstor/volume/creator/parameter_test.go (97%) create mode 100644 storage/oceanstorage/oceanstor/volume/creator/snapshot_fs.go rename storage/{ => oceanstorage}/oceanstor/volume/dtree.go (98%) rename storage/{ => oceanstorage}/oceanstor/volume/dtree_test.go (90%) rename storage/{ => oceanstorage}/oceanstor/volume/nas.go (94%) rename storage/{ => oceanstorage}/oceanstor/volume/nas_test.go (100%) rename storage/{ => oceanstorage}/oceanstor/volume/san.go (97%) rename storage/{ => oceanstorage}/oceanstor/volume/types.go (100%) create mode 100644 utils/cert/cert.go create mode 100644 utils/k8sutils/pv_helper.go create mode 100644 utils/k8sutils/resource_accessor.go create mode 100644 utils/k8sutils/resource_accessor_test.go create mode 100644 utils/retry/retry.go create mode 100644 utils/retry/retry_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 968d9bcd..caabfbf9 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,19 @@ [Releases](https://github.com/Huawei/eSDK_K8S_Plugin/releases) +## v4.6.0 + +- Support Kubernetes 1.31. +- Support Openshift 4.16/4.17. +- Support OceanStor Dorado V700R001C00. +- Supports the PPC64LE CPU architecture of the IBM Power platform. +- Support for Red Hat Enterprise Linux 8.6/8.7/8.8/8.9/8.10/9.4 x86_64. +- Support NFS 4.2 on OceanStor Dorado storage 6.1.8 and later version. +- Support NFS over RDMA on OceanStor Pacific storage 8.2.0 and later version. +- Added `disableVerifyCapacity` parameter in StorageClass whether allow to disable volume capacity verification. +- Added the restriction which is 1~30 on the `maxClientThreads` parameter in the backend. +- Fixed an issue where raw volumes may be misplaced when they are powered off unexpectedly. + ## v4.5.0 - The default synchronization speed of hyper metro pair is changed from the highest speed to the default speed determined by the storage. diff --git a/Makefile b/Makefile index 4c719d68..729fabf2 100644 --- a/Makefile +++ b/Makefile @@ -7,10 +7,6 @@ PLATFORM=PLATFORM # (Optional) [2.5.RC1 2.5.RC2 ...] eSDK Version RELEASE_VER=RELEASE_VER -# (Optional) [TRUE FALSE] Compile Binary Only, Cancel Inline Optimization -ONLY_BIN=ONLY_BIN -# (Optional) [github] Specifies the platform which to build on -BUILD_ON=BUILD_ON export GO111MODULE=on @@ -20,32 +16,18 @@ else export PACKAGE=eSDK_Huawei_Storage_${RELEASE_VER}_Kubernetes_CSI_Plugin_V${VER}_${PLATFORM}_64 endif -# Build process -ifeq (${ONLY_BIN}, TRUE) -all:PREPARE BUILD PACK -# Disable inline optimization -flag = -gcflags "all=-N -l" -binary_flag = -gcflags "all=-N -l" +# Platform [X86, ARM, PPC64LE], default value is [X86] +ifeq (${PLATFORM}, PPC64LE) +arch=ppc64le +else ifeq (${PLATFORM}, ARM) +arch=arm64 else -flag = -ldflags="-s -linkmode 'external' -extldflags '-Wl,-z,now'" -buildmode=pie -binary_flag = -ldflags="-s" -buildmode=pie -all:PREPARE BUILD COPY_FILE PACK -endif - -# Platform [X86, ARM] -ifeq (${PLATFORM}, X86) -env = CGO_CFLAGS="-fstack-protector-strong -D_FORTIFY_SOURCE=2 -O2" GOOS=linux GOARCH=amd64 -binary_env = CGO_ENABLED=0 GOOS=linux GOARCH=amd64 -else -env = CGO_CFLAGS="-fstack-protector-strong -D_FORTIFY_SOURCE=2 -O2" GOOS=linux GOARCH=arm64 -binary_env = CGO_ENABLED=0 GOOS=linux GOARCH=arm64 +arch=amd64 endif - -# Build_ON [github] -ifeq ($(BUILD_ON), github) +env = CGO_ENABLED=0 GOOS=linux GOARCH=${arch} flag = -ldflags="-s -bindnow" -buildmode=pie -env = $(binary_env) -endif + +all:PREPARE BUILD COPY_FILE PACK PREPARE: rm -rf ./${PACKAGE} @@ -58,7 +40,7 @@ BUILD: ${env} go build -o ./${PACKAGE}/bin/storage-backend-controller ${flag} ./cmd/storage-backend-controller ${env} go build -o ./${PACKAGE}/bin/storage-backend-sidecar ${flag} ./cmd/storage-backend-sidecar ${env} go build -o ./${PACKAGE}/bin/huawei-csi-extender ${flag} ./cmd/huawei-csi-extender - ${binary_env} go build -o ./${PACKAGE}/bin/oceanctl ${binary_flag} ./cli + ${env} go build -o ./${PACKAGE}/bin/oceanctl ${flag} ./cli COPY_FILE: mkdir -p ./${PACKAGE}/examples diff --git a/build.sh b/build.sh index ff7ae735..24db44f7 100644 --- a/build.sh +++ b/build.sh @@ -25,21 +25,27 @@ PLATFORM=$2 package_name="eSDK_Huawei_Storage_Kubernetes_CSI_Plugin_V${VER}_${PLATFORM}_64" echo "Start to make with Makefile" -make -f Makefile VER=$1 PLATFORM=$2 BUILD_ON=github +make -f Makefile VER=$1 PLATFORM=$2 echo "Platform confirmation" if [[ "${PLATFORM}" == "ARM" ]];then PULL_FLAG="--platform=arm64" BUILD_FLAG="--platform linux/arm64" + GO_PLATFORM="arm64" elif [[ "${PLATFORM}" == "X86" ]];then PULL_FLAG="--platform=amd64" BUILD_FLAG="--platform linux/amd64" + GO_PLATFORM="amd64" +elif [[ "${PLATFORM}" == "PPC64LE" ]];then + PULL_FLAG="--platform=ppc64le" + BUILD_FLAG="--platform linux/ppc64le" + GO_PLATFORM="ppc64le" else - echo "Wrong PLATFORM, support [X86, ARM]" + echo "Wrong PLATFORM, support [X86, ARM, PPC64LE]" exit fi -echo "Start to pull busybox image with architecture" +echo "Start to pull busybox image with architecture ${PULL_FLAG}" docker pull ${PULL_FLAG} busybox:stable-glibc docker pull ${PULL_FLAG} gcr.io/distroless/base:latest @@ -82,6 +88,9 @@ cd .. echo "Start to clear temporary files" rm -f ./huawei-csi +rm -f ./huawei-csi-extender +rm -f ./storage-backend-controller +rm -f ./storage-backend-sidecar rm -rf ./build_dir echo "Build finish" diff --git a/cli/client/client_discover.go b/cli/client/client_discover.go index ff2bb6b2..2b9b442d 100644 --- a/cli/client/client_discover.go +++ b/cli/client/client_discover.go @@ -22,7 +22,7 @@ import ( "os/exec" "strings" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // DiscoverKubernetesCLI used to discover kubernetes CLI. diff --git a/cli/client/client_helper.go b/cli/client/client_helper.go index cd4244d4..ca7ee370 100644 --- a/cli/client/client_helper.go +++ b/cli/client/client_helper.go @@ -26,9 +26,9 @@ import ( corev1 "k8s.io/api/core/v1" - "huawei-csi-driver/cli/helper" - xuanwuV1 "huawei-csi-driver/client/apis/xuanwu/v1" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/helper" + xuanwuV1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // CommonCallHandler common call handler diff --git a/cli/client/kube_client.go b/cli/client/kube_client.go index fc60074f..c1d036d7 100644 --- a/cli/client/kube_client.go +++ b/cli/client/kube_client.go @@ -23,7 +23,7 @@ import ( "k8s.io/api/core/v1" - "huawei-csi-driver/cli/helper" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/helper" ) const ( diff --git a/cli/client/kubectl_options.go b/cli/client/kubectl_options.go index 848404a8..8016bde0 100644 --- a/cli/client/kubectl_options.go +++ b/cli/client/kubectl_options.go @@ -24,7 +24,7 @@ import ( "gopkg.in/yaml.v3" - "huawei-csi-driver/cli/helper" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/helper" ) type Filter struct { diff --git a/cli/cmd/collect.go b/cli/cmd/collect.go index 6472ebdc..24b9f9df 100644 --- a/cli/cmd/collect.go +++ b/cli/cmd/collect.go @@ -19,7 +19,7 @@ package cmd import ( "github.com/spf13/cobra" - "huawei-csi-driver/cli/cmd/options" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/cmd/options" ) func registerCollectCmd() { diff --git a/cli/cmd/collect_logs.go b/cli/cmd/collect_logs.go index 3f379812..d0d13e56 100644 --- a/cli/cmd/collect_logs.go +++ b/cli/cmd/collect_logs.go @@ -19,10 +19,10 @@ package cmd import ( "github.com/spf13/cobra" - "huawei-csi-driver/cli/cmd/options" - "huawei-csi-driver/cli/config" - "huawei-csi-driver/cli/helper" - "huawei-csi-driver/cli/resources" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/cmd/options" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/helper" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/resources" ) func registerCollectLogsCmd() { diff --git a/cli/cmd/create.go b/cli/cmd/create.go index cc43586e..ff7d2fd8 100644 --- a/cli/cmd/create.go +++ b/cli/cmd/create.go @@ -19,7 +19,7 @@ package cmd import ( "github.com/spf13/cobra" - "huawei-csi-driver/cli/cmd/options" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/cmd/options" ) func registerCreateCmd() { diff --git a/cli/cmd/create_backend.go b/cli/cmd/create_backend.go index f26561f6..bc7292b0 100644 --- a/cli/cmd/create_backend.go +++ b/cli/cmd/create_backend.go @@ -19,11 +19,11 @@ package cmd import ( "github.com/spf13/cobra" - "huawei-csi-driver/cli/client" - "huawei-csi-driver/cli/cmd/options" - "huawei-csi-driver/cli/config" - "huawei-csi-driver/cli/helper" - "huawei-csi-driver/cli/resources" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/cmd/options" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/helper" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/resources" ) func registerCreateBackendCmd() { diff --git a/cli/cmd/create_cert.go b/cli/cmd/create_cert.go index 2257f88e..db767d80 100644 --- a/cli/cmd/create_cert.go +++ b/cli/cmd/create_cert.go @@ -19,11 +19,11 @@ package cmd import ( "github.com/spf13/cobra" - "huawei-csi-driver/cli/client" - "huawei-csi-driver/cli/cmd/options" - "huawei-csi-driver/cli/config" - "huawei-csi-driver/cli/helper" - "huawei-csi-driver/cli/resources" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/cmd/options" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/helper" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/resources" ) func registerCreateCertCmd() { diff --git a/cli/cmd/delete.go b/cli/cmd/delete.go index 40cef39a..d13e29f4 100644 --- a/cli/cmd/delete.go +++ b/cli/cmd/delete.go @@ -19,7 +19,7 @@ package cmd import ( "github.com/spf13/cobra" - "huawei-csi-driver/cli/cmd/options" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/cmd/options" ) func registerDeleteCmd() { diff --git a/cli/cmd/delete_backend.go b/cli/cmd/delete_backend.go index faf4d348..57f5d545 100644 --- a/cli/cmd/delete_backend.go +++ b/cli/cmd/delete_backend.go @@ -19,11 +19,11 @@ package cmd import ( "github.com/spf13/cobra" - "huawei-csi-driver/cli/client" - "huawei-csi-driver/cli/cmd/options" - "huawei-csi-driver/cli/config" - "huawei-csi-driver/cli/helper" - "huawei-csi-driver/cli/resources" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/cmd/options" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/helper" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/resources" ) func registerDeleteBackendCmd() { diff --git a/cli/cmd/delete_cert.go b/cli/cmd/delete_cert.go index c06bfe4e..e994304b 100644 --- a/cli/cmd/delete_cert.go +++ b/cli/cmd/delete_cert.go @@ -19,10 +19,10 @@ package cmd import ( "github.com/spf13/cobra" - "huawei-csi-driver/cli/cmd/options" - "huawei-csi-driver/cli/config" - "huawei-csi-driver/cli/helper" - "huawei-csi-driver/cli/resources" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/cmd/options" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/helper" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/resources" ) func registerDeleteCertCmd() { diff --git a/cli/cmd/get.go b/cli/cmd/get.go index d8346441..aa17ae21 100644 --- a/cli/cmd/get.go +++ b/cli/cmd/get.go @@ -19,7 +19,7 @@ package cmd import ( "github.com/spf13/cobra" - "huawei-csi-driver/cli/cmd/options" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/cmd/options" ) func registerGetCmd() { diff --git a/cli/cmd/get_backend.go b/cli/cmd/get_backend.go index 9e139692..42d122fb 100644 --- a/cli/cmd/get_backend.go +++ b/cli/cmd/get_backend.go @@ -19,11 +19,11 @@ package cmd import ( "github.com/spf13/cobra" - "huawei-csi-driver/cli/client" - "huawei-csi-driver/cli/cmd/options" - "huawei-csi-driver/cli/config" - "huawei-csi-driver/cli/helper" - "huawei-csi-driver/cli/resources" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/cmd/options" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/helper" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/resources" ) func registerGetBackendCmd() { diff --git a/cli/cmd/get_cert.go b/cli/cmd/get_cert.go index 128176b2..b0f3e154 100644 --- a/cli/cmd/get_cert.go +++ b/cli/cmd/get_cert.go @@ -19,10 +19,10 @@ package cmd import ( "github.com/spf13/cobra" - "huawei-csi-driver/cli/cmd/options" - "huawei-csi-driver/cli/config" - "huawei-csi-driver/cli/helper" - "huawei-csi-driver/cli/resources" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/cmd/options" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/helper" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/resources" ) func registerGetCertCmd() { diff --git a/cli/cmd/options/flag_options.go b/cli/cmd/options/flag_options.go index a20a4df7..4b2a085b 100644 --- a/cli/cmd/options/flag_options.go +++ b/cli/cmd/options/flag_options.go @@ -20,8 +20,8 @@ package options import ( "github.com/spf13/cobra" - "huawei-csi-driver/cli/config" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // FlagsOptions is used for processing flags which user input diff --git a/cli/cmd/root.go b/cli/cmd/root.go index 9b9101e5..84aa4eec 100644 --- a/cli/cmd/root.go +++ b/cli/cmd/root.go @@ -23,10 +23,10 @@ import ( "github.com/spf13/cobra" - "huawei-csi-driver/cli/client" - "huawei-csi-driver/cli/cmd/options" - "huawei-csi-driver/cli/config" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/cmd/options" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // RootCmd is a root command of oceanctl. diff --git a/cli/cmd/update.go b/cli/cmd/update.go index db717fd4..f2d250bc 100644 --- a/cli/cmd/update.go +++ b/cli/cmd/update.go @@ -19,7 +19,7 @@ package cmd import ( "github.com/spf13/cobra" - "huawei-csi-driver/cli/cmd/options" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/cmd/options" ) func registerUpdateCmd() { diff --git a/cli/cmd/update_backend.go b/cli/cmd/update_backend.go index 8d77b12c..fd1f61c4 100644 --- a/cli/cmd/update_backend.go +++ b/cli/cmd/update_backend.go @@ -19,11 +19,11 @@ package cmd import ( "github.com/spf13/cobra" - "huawei-csi-driver/cli/client" - "huawei-csi-driver/cli/cmd/options" - "huawei-csi-driver/cli/config" - "huawei-csi-driver/cli/helper" - "huawei-csi-driver/cli/resources" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/cmd/options" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/helper" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/resources" ) func registerUpdateBackendCmd() { diff --git a/cli/cmd/update_cert.go b/cli/cmd/update_cert.go index ec3badee..d5498417 100644 --- a/cli/cmd/update_cert.go +++ b/cli/cmd/update_cert.go @@ -19,10 +19,10 @@ package cmd import ( "github.com/spf13/cobra" - "huawei-csi-driver/cli/cmd/options" - "huawei-csi-driver/cli/config" - "huawei-csi-driver/cli/helper" - "huawei-csi-driver/cli/resources" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/cmd/options" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/helper" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/resources" ) func registerUpdateCertCmd() { diff --git a/cli/cmd/version.go b/cli/cmd/version.go index 80fd3c1c..fbe891f9 100644 --- a/cli/cmd/version.go +++ b/cli/cmd/version.go @@ -21,7 +21,7 @@ import ( "github.com/spf13/cobra" - "huawei-csi-driver/cli/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/config" ) func registerVersionCmd() { diff --git a/cli/config/config.go b/cli/config/config.go index 5641acc2..6bca5b95 100644 --- a/cli/config/config.go +++ b/cli/config/config.go @@ -18,12 +18,12 @@ package config import ( - "huawei-csi-driver/cli/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/client" ) const ( // CliVersion oceanctl version - CliVersion = "v4.5.0" + CliVersion = "v4.6.0" // DefaultMaxClientThreads default max client threads DefaultMaxClientThreads = "30" diff --git a/cli/helper/async_task.go b/cli/helper/async_task.go index b52e88c4..831d2509 100644 --- a/cli/helper/async_task.go +++ b/cli/helper/async_task.go @@ -19,7 +19,7 @@ package helper import ( "sync" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // Task Define interfaces required by asynchronous tasks. diff --git a/cli/helper/command_helper.go b/cli/helper/command_helper.go index d6a0a319..38f3cb33 100644 --- a/cli/helper/command_helper.go +++ b/cli/helper/command_helper.go @@ -25,7 +25,7 @@ import ( "strconv" "strings" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const exitCommand = "exit" diff --git a/cli/helper/goroutine_limit.go b/cli/helper/goroutine_limit.go index c536a72f..3e0af242 100644 --- a/cli/helper/goroutine_limit.go +++ b/cli/helper/goroutine_limit.go @@ -19,7 +19,7 @@ package helper import ( "sync" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // GlobalGoroutineLimit is used to limit concurrency of goroutine diff --git a/cli/helper/utils.go b/cli/helper/utils.go index 4191e25b..3e2078f5 100644 --- a/cli/helper/utils.go +++ b/cli/helper/utils.go @@ -30,7 +30,7 @@ import ( "k8s.io/apimachinery/pkg/util/uuid" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const BackendNameMaxLength = 63 diff --git a/cli/main.go b/cli/main.go index c98daf6b..06619bb5 100644 --- a/cli/main.go +++ b/cli/main.go @@ -19,7 +19,7 @@ package main import ( "os" - "huawei-csi-driver/cli/cmd" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/cmd" ) const ( diff --git a/cli/resources/backend.go b/cli/resources/backend.go index a58d3ef5..299642de 100644 --- a/cli/resources/backend.go +++ b/cli/resources/backend.go @@ -26,11 +26,11 @@ import ( corev1 "k8s.io/api/core/v1" - "huawei-csi-driver/cli/client" - "huawei-csi-driver/cli/config" - "huawei-csi-driver/cli/helper" - xuanwuV1 "huawei-csi-driver/client/apis/xuanwu/v1" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/helper" + xuanwuV1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // Backend is a storage backend object diff --git a/cli/resources/backend_helper.go b/cli/resources/backend_helper.go index 1c097100..97f07b5c 100644 --- a/cli/resources/backend_helper.go +++ b/cli/resources/backend_helper.go @@ -31,8 +31,8 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "huawei-csi-driver/cli/helper" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/helper" + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" ) const ( diff --git a/cli/resources/cert.go b/cli/resources/cert.go index bdeeee10..8cff3134 100644 --- a/cli/resources/cert.go +++ b/cli/resources/cert.go @@ -23,11 +23,11 @@ import ( corev1 "k8s.io/api/core/v1" - "huawei-csi-driver/cli/client" - "huawei-csi-driver/cli/config" - "huawei-csi-driver/cli/helper" - xuanwuV1 "huawei-csi-driver/client/apis/xuanwu/v1" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/helper" + xuanwuV1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // Cert is the cert resource diff --git a/cli/resources/file_logs_factory.go b/cli/resources/file_logs_factory.go index 2cbcfd7c..1ba6aba0 100644 --- a/cli/resources/file_logs_factory.go +++ b/cli/resources/file_logs_factory.go @@ -25,9 +25,9 @@ import ( "strings" "time" - "huawei-csi-driver/cli/config" - "huawei-csi-driver/cli/helper" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/helper" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // FileLogsCollect is the interface to collect file logs. diff --git a/cli/resources/file_logs_factory_test.go b/cli/resources/file_logs_factory_test.go index ce52b2db..5789e641 100644 --- a/cli/resources/file_logs_factory_test.go +++ b/cli/resources/file_logs_factory_test.go @@ -24,9 +24,9 @@ import ( "github.com/agiledragon/gomonkey/v2" - "huawei-csi-driver/cli/client" - "huawei-csi-driver/cli/config" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/cli/resources/logs.go b/cli/resources/logs.go index 3b0dedbe..aa0e2970 100644 --- a/cli/resources/logs.go +++ b/cli/resources/logs.go @@ -32,10 +32,10 @@ import ( coreV1 "k8s.io/api/core/v1" - "huawei-csi-driver/cli/client" - "huawei-csi-driver/cli/config" - "huawei-csi-driver/cli/helper" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/helper" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/cli/resources/logs_helper.go b/cli/resources/logs_helper.go index e781845a..221065b1 100644 --- a/cli/resources/logs_helper.go +++ b/cli/resources/logs_helper.go @@ -29,9 +29,9 @@ import ( coreV1 "k8s.io/api/core/v1" - "huawei-csi-driver/cli/config" - "huawei-csi-driver/cli/helper" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/helper" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/cli/resources/resource.go b/cli/resources/resource.go index b20534bf..4c65f787 100644 --- a/cli/resources/resource.go +++ b/cli/resources/resource.go @@ -19,7 +19,7 @@ package resources import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "huawei-csi-driver/cli/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/config" ) type resourceTuple struct { diff --git a/cli/resources/validate.go b/cli/resources/validate.go index d94e3d5b..0aa74f82 100644 --- a/cli/resources/validate.go +++ b/cli/resources/validate.go @@ -21,8 +21,8 @@ import ( "fmt" "strings" - "huawei-csi-driver/cli/config" - "huawei-csi-driver/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" ) // Validator is used to validate a Resource object diff --git a/cmd/huawei-csi-extender/main.go b/cmd/huawei-csi-extender/main.go index d1b91ffd..356dfe1a 100644 --- a/cmd/huawei-csi-extender/main.go +++ b/cmd/huawei-csi-extender/main.go @@ -28,15 +28,15 @@ import ( "k8s.io/client-go/kubernetes/scheme" "k8s.io/client-go/tools/record" - "huawei-csi-driver/csi/app" - "huawei-csi-driver/lib/drcsi" - "huawei-csi-driver/lib/drcsi/rpc" - clientSet "huawei-csi-driver/pkg/client/clientset/versioned" - crdScheme "huawei-csi-driver/pkg/client/clientset/versioned/scheme" - crdInformers "huawei-csi-driver/pkg/client/informers/externalversions" - "huawei-csi-driver/pkg/modify" - "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + "github.com/Huawei/eSDK_K8S_Plugin/v4/lib/drcsi" + "github.com/Huawei/eSDK_K8S_Plugin/v4/lib/drcsi/rpc" + clientSet "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/clientset/versioned" + crdScheme "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/clientset/versioned/scheme" + crdInformers "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/informers/externalversions" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/modify" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/cmd/storage-backend-controller/main.go b/cmd/storage-backend-controller/main.go index 199c2255..9061b5de 100644 --- a/cmd/storage-backend-controller/main.go +++ b/cmd/storage-backend-controller/main.go @@ -31,14 +31,14 @@ import ( coreV1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/record" - "huawei-csi-driver/csi/app" - clientSet "huawei-csi-driver/pkg/client/clientset/versioned" - backendScheme "huawei-csi-driver/pkg/client/clientset/versioned/scheme" - backendInformers "huawei-csi-driver/pkg/client/informers/externalversions" - "huawei-csi-driver/pkg/storage-backend/controller" - "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/pkg/webhook" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + clientSet "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/clientset/versioned" + backendScheme "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/clientset/versioned/scheme" + backendInformers "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/informers/externalversions" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/storage-backend/controller" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/webhook" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/cmd/storage-backend-sidecar/main.go b/cmd/storage-backend-sidecar/main.go index fe567598..79e58c2b 100644 --- a/cmd/storage-backend-sidecar/main.go +++ b/cmd/storage-backend-sidecar/main.go @@ -33,16 +33,16 @@ import ( coreV1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/record" - "huawei-csi-driver/csi/app" - "huawei-csi-driver/lib/drcsi/connection" - "huawei-csi-driver/lib/drcsi/rpc" - clientSet "huawei-csi-driver/pkg/client/clientset/versioned" - backendScheme "huawei-csi-driver/pkg/client/clientset/versioned/scheme" - backendInformers "huawei-csi-driver/pkg/client/informers/externalversions" - "huawei-csi-driver/pkg/sidecar/controller" - storageBackend "huawei-csi-driver/pkg/storage-backend/handle" - "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + "github.com/Huawei/eSDK_K8S_Plugin/v4/lib/drcsi/connection" + "github.com/Huawei/eSDK_K8S_Plugin/v4/lib/drcsi/rpc" + clientSet "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/clientset/versioned" + backendScheme "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/clientset/versioned/scheme" + backendInformers "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/informers/externalversions" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/sidecar/controller" + storageBackend "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/storage-backend/handle" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/connector/connector.go b/connector/connector.go index 61a001da..10459118 100644 --- a/connector/connector.go +++ b/connector/connector.go @@ -20,7 +20,7 @@ import ( "context" "fmt" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/connector/connector_test.go b/connector/connector_test.go index 9da3a6df..043b71a4 100644 --- a/connector/connector_test.go +++ b/connector/connector_test.go @@ -23,9 +23,9 @@ import ( "github.com/prashantv/gostub" - "huawei-csi-driver/csi/app" - cfg "huawei-csi-driver/csi/app/config" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + cfg "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/connector/connector_utils.go b/connector/connector_utils.go index b962c35f..b565b17b 100644 --- a/connector/connector_utils.go +++ b/connector/connector_utils.go @@ -32,10 +32,10 @@ import ( "strings" "time" - "huawei-csi-driver/csi/app" - "huawei-csi-driver/pkg/constants" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/connector/connector_utils_common.go b/connector/connector_utils_common.go index e5c95c7b..03cd33f0 100644 --- a/connector/connector_utils_common.go +++ b/connector/connector_utils_common.go @@ -21,9 +21,9 @@ import ( "context" "strings" - "huawei-csi-driver/connector/utils/lock" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector/utils/lock" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // GetPhysicalDevices to get physical devices diff --git a/connector/connector_utils_nvme.go b/connector/connector_utils_nvme.go index 170eae76..0cffee71 100644 --- a/connector/connector_utils_nvme.go +++ b/connector/connector_utils_nvme.go @@ -28,8 +28,8 @@ import ( "strconv" "strings" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // DoScanNVMeDevice used to scan device by command nvme ns-rescan diff --git a/connector/connector_utils_test.go b/connector/connector_utils_test.go index f716818a..f59a7373 100644 --- a/connector/connector_utils_test.go +++ b/connector/connector_utils_test.go @@ -30,7 +30,7 @@ import ( "github.com/prashantv/gostub" "github.com/stretchr/testify/assert" - "huawei-csi-driver/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" ) func TestGetDevice(t *testing.T) { diff --git a/connector/connector_utils_ultrapath.go b/connector/connector_utils_ultrapath.go index 3772ea4d..dadebc60 100644 --- a/connector/connector_utils_ultrapath.go +++ b/connector/connector_utils_ultrapath.go @@ -25,8 +25,8 @@ import ( "regexp" "strings" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/connector/connector_utils_ultrapath_test.go b/connector/connector_utils_ultrapath_test.go index c92b6a64..5580177a 100644 --- a/connector/connector_utils_ultrapath_test.go +++ b/connector/connector_utils_ultrapath_test.go @@ -24,7 +24,7 @@ import ( "github.com/prashantv/gostub" "github.com/stretchr/testify/assert" - "huawei-csi-driver/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" ) func TestRunUpCommand(t *testing.T) { diff --git a/connector/fibrechannel/fc.go b/connector/fibrechannel/fc.go index 0438a511..904ca2c3 100644 --- a/connector/fibrechannel/fc.go +++ b/connector/fibrechannel/fc.go @@ -19,9 +19,9 @@ package fibrechannel import ( "context" - "huawei-csi-driver/connector" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // Connector implements the connector.VolumeConnector for FC protocol diff --git a/connector/fibrechannel/fc_helper.go b/connector/fibrechannel/fc_helper.go index 361d9043..18725eeb 100644 --- a/connector/fibrechannel/fc_helper.go +++ b/connector/fibrechannel/fc_helper.go @@ -27,10 +27,10 @@ import ( "sync" "time" - "huawei-csi-driver/connector" - connutils "huawei-csi-driver/connector/utils" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector" + connutils "github.com/Huawei/eSDK_K8S_Plugin/v4/connector/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/connector/fibrechannel/fc_ultrapath_helper.go b/connector/fibrechannel/fc_ultrapath_helper.go index b987b568..cf2a9096 100644 --- a/connector/fibrechannel/fc_ultrapath_helper.go +++ b/connector/fibrechannel/fc_ultrapath_helper.go @@ -25,9 +25,9 @@ import ( "path/filepath" "time" - "huawei-csi-driver/connector" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) func getVirtualLunPath(ctx context.Context, lunWWN string) (string, error) { diff --git a/connector/host/host_helper.go b/connector/host/host_helper.go index a1f3d711..13d9c972 100644 --- a/connector/host/host_helper.go +++ b/connector/host/host_helper.go @@ -29,10 +29,10 @@ import ( apiErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "huawei-csi-driver/csi/app" - "huawei-csi-driver/proto" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + "github.com/Huawei/eSDK_K8S_Plugin/v4/proto" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/connector/host/host_helper_test.go b/connector/host/host_helper_test.go index 72eb5008..c7856995 100644 --- a/connector/host/host_helper_test.go +++ b/connector/host/host_helper_test.go @@ -30,12 +30,12 @@ import ( apiErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "huawei-csi-driver/csi/app" - cfg "huawei-csi-driver/csi/app/config" - "huawei-csi-driver/proto" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/k8sutils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + cfg "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/proto" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/k8sutils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/connector/iscsi/iscsi.go b/connector/iscsi/iscsi.go index 2482b9f5..49ab53e4 100644 --- a/connector/iscsi/iscsi.go +++ b/connector/iscsi/iscsi.go @@ -19,9 +19,9 @@ package iscsi import ( "context" - "huawei-csi-driver/connector" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // Connector implements the connector.VolumeConnector for Connector protocol diff --git a/connector/iscsi/iscsi_common.go b/connector/iscsi/iscsi_common.go index 42c04b86..d0bbf458 100644 --- a/connector/iscsi/iscsi_common.go +++ b/connector/iscsi/iscsi_common.go @@ -25,9 +25,9 @@ import ( "path/filepath" "strings" - "huawei-csi-driver/connector" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) func getSessionIds(ctx context.Context, devices []string, deviceType int) ([]string, error) { diff --git a/connector/iscsi/iscsi_helper.go b/connector/iscsi/iscsi_helper.go index eb0b4a63..b16a83a0 100644 --- a/connector/iscsi/iscsi_helper.go +++ b/connector/iscsi/iscsi_helper.go @@ -30,12 +30,12 @@ import ( "sync/atomic" "time" - "huawei-csi-driver/connector" - connutils "huawei-csi-driver/connector/utils" - "huawei-csi-driver/csi/app" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/concurrent" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector" + connutils "github.com/Huawei/eSDK_K8S_Plugin/v4/connector/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/concurrent" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/connector/local/local.go b/connector/local/local.go index 0dd45060..00192416 100644 --- a/connector/local/local.go +++ b/connector/local/local.go @@ -20,9 +20,9 @@ package local import ( "context" - "huawei-csi-driver/connector" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // Connector to define a local lock when connect or disconnect, in order to preventing connect and disconnect confusion diff --git a/connector/local/local_helper.go b/connector/local/local_helper.go index eb53f4a7..c07036fb 100644 --- a/connector/local/local_helper.go +++ b/connector/local/local_helper.go @@ -24,9 +24,9 @@ import ( "strings" "time" - "huawei-csi-driver/connector" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const waitDevOnlineTimeInterval = 2 * time.Second diff --git a/connector/local/local_test.go b/connector/local/local_test.go index 1c733cc6..3aed9a5a 100644 --- a/connector/local/local_test.go +++ b/connector/local/local_test.go @@ -24,12 +24,12 @@ import ( "github.com/prashantv/gostub" - "huawei-csi-driver/connector" - "huawei-csi-driver/connector/utils/lock" - "huawei-csi-driver/csi/app" - "huawei-csi-driver/csi/app/config" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector/utils/lock" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/connector/nfs/nfs.go b/connector/nfs/nfs.go index 3d052464..9b324b48 100644 --- a/connector/nfs/nfs.go +++ b/connector/nfs/nfs.go @@ -20,8 +20,8 @@ package nfs import ( "context" - "huawei-csi-driver/connector" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // Connector to define a local lock when connect or disconnect, in order to preventing mounting and unmounting confusion diff --git a/connector/nfs/nfs_helper.go b/connector/nfs/nfs_helper.go index 4e73bd70..ab9d7893 100644 --- a/connector/nfs/nfs_helper.go +++ b/connector/nfs/nfs_helper.go @@ -23,16 +23,15 @@ import ( "fmt" "os" "os/exec" - "path" - "path/filepath" "strings" "time" "github.com/container-storage-interface/spec/lib/go/csi" - "huawei-csi-driver/connector" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector" + connUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/connector/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( @@ -46,15 +45,10 @@ type connectorInfo struct { sourcePath string targetPath string fsType string - mntFlags mountParam + mntFlags connUtils.MountParam accessMode csi.VolumeCapability_AccessMode_Mode } -type mountParam struct { - dashT string - dashO string -} - func parseNFSInfo(ctx context.Context, connectionProperties map[string]interface{}) (*connectorInfo, error) { var con connectorInfo @@ -97,7 +91,7 @@ func parseNFSInfo(ctx context.Context, con.targetPath = targetPath con.fsType = fsType con.accessMode = accessMode - con.mntFlags = mountParam{dashO: strings.TrimSpace(mntDashO), dashT: mntDashT} + con.mntFlags = connUtils.MountParam{DashO: strings.TrimSpace(mntDashO), DashT: mntDashT} return &con, nil } @@ -146,133 +140,8 @@ func preMount(sourcePath, targetPath string, checkSourcePath bool) error { return nil } -func mountFS(ctx context.Context, sourcePath, targetPath string, flags mountParam) error { - return mountUnix(ctx, sourcePath, targetPath, flags, false) -} - -func compareMountPath(ctx context.Context, sourcePath, mountSourcePath string) error { - // the mount source path is like: /dev/mapper/mpath or /dev/sd - // but the source path is like: /dev/dm- or /dev/sd. - // The relationship of these two path as follows: - // lrwxrwxrwx 1 root root 7 Aug 1 17:13 /dev/mapper/mpathc -> ../dm-3 - _, err := connector.ReadDevice(ctx, mountSourcePath) - if err != nil { - return err - } - - mountRealPath, err := filepath.EvalSymlinks(mountSourcePath) - if err != nil { - return err - } - - sourceRealPath, err := filepath.EvalSymlinks(sourcePath) - if err != nil { - return err - } - - if sourceRealPath != mountRealPath { - msg := fmt.Sprintf("The source path is %s, the real path is %s", - sourcePath, mountRealPath) - log.AddContext(ctx).Errorln(msg) - return errors.New(msg) - } - - return nil -} - -func appendXFSMountFlags(ctx context.Context, sourcePath string, flags mountParam) mountParam { - // Only disk devices need to be determined whether the system is XFS. - if !strings.Contains(sourcePath, "/dev/") { - return flags - } - fsType, err := connector.GetFsTypeByDevPath(ctx, sourcePath) - if err != nil { - log.AddContext(ctx).Warningf("Get device: [%s] FsType failed. error: %v", sourcePath, err) - return flags - } - if fsType == "xfs" { - // xfs volumes are always mounted with '-o nouuid' to allow clones to be mounted to the same node as the source - if flags.dashO == "" { - flags.dashO = "nouuid" - } else { - flags.dashO = fmt.Sprintf("%s,nouuid", flags.dashO) - } - } - - log.AddContext(ctx).Infof("mount flags: [%v]", flags) - return flags -} - -func mountUnix(ctx context.Context, sourcePath, targetPath string, flags mountParam, checkSourcePath bool) error { - var output string - var err error - err = preMount(sourcePath, targetPath, checkSourcePath) - if err != nil { - return err - } - - mountMap, err := connector.ReadMountPoints(ctx) - value, exist := mountMap[targetPath] - if exist { - // checkSourcePath means the source is block type, need to check the realpath - if checkSourcePath { - err := compareMountPath(ctx, sourcePath, value) - if err != nil { - return err - } - log.AddContext(ctx).Infof("%s is already mount to %s", sourcePath, targetPath) - return nil - } - - // if the checkSourcePath is false, check the filesystem by comparing the sourcePath and mountPath - if value == sourcePath || path.Base(path.Dir(targetPath)) == path.Base(path.Dir(sourcePath)) || - ContainSourceDevice(ctx, sourcePath, value) { - log.AddContext(ctx).Infof("Mount %s to %s is already exist", sourcePath, targetPath) - return nil - } - - return utils.Errorf(ctx, "The mount %s is already exist, but the source path is not %s, instead of %s", - targetPath, sourcePath, value) - } - - flags = appendXFSMountFlags(ctx, sourcePath, flags) - - if flags.dashT != "" { - flags.dashT = fmt.Sprintf("-t %s", flags.dashT) - } - - if flags.dashO != "" { - flags.dashO = fmt.Sprintf("-o %s", flags.dashO) - } - - output, err = utils.ExecShellCmd(ctx, "mount %s %s %s %s", sourcePath, targetPath, flags.dashT, flags.dashO) - if err != nil { - log.AddContext(ctx).Errorf("Mount %s to %s failed, error res: %s, error: %s", - sourcePath, targetPath, output, err) - return err - } - - return nil -} - -// ContainSourceDevice used to check target path referenced source device is equal sourceDev -func ContainSourceDevice(ctx context.Context, targetPath, sourceDev string) bool { - for _, value := range findSourceDevice(ctx, targetPath) { - if value == sourceDev { - return true - } - } - return false -} - -// findSourceDevice use findmnt command to find mountPath referenced source device -func findSourceDevice(ctx context.Context, targetPath string) []string { - output, err := utils.ExecShellCmd(ctx, "findmnt -o source --noheadings --target %s", targetPath) - if err != nil { - return []string{} - } - - return strings.Split(output, "\n") +func mountFS(ctx context.Context, sourcePath, targetPath string, flags connUtils.MountParam) error { + return connUtils.MountToDir(ctx, sourcePath, targetPath, flags, false) } func getFSType(ctx context.Context, sourcePath string) (string, error) { @@ -397,12 +266,12 @@ func mountDisk(ctx context.Context, conn *connectorInfo) error { return err } - err = mountUnix(ctx, conn.sourcePath, conn.targetPath, conn.mntFlags, true) + err = connUtils.MountToDir(ctx, conn.sourcePath, conn.targetPath, conn.mntFlags, true) if err != nil { return err } } else { - err = mountUnix(ctx, conn.sourcePath, conn.targetPath, conn.mntFlags, true) + err = connUtils.MountToDir(ctx, conn.sourcePath, conn.targetPath, conn.mntFlags, true) if err != nil { return err } @@ -426,22 +295,6 @@ func mountDisk(ctx context.Context, conn *connectorInfo) error { return nil } -func unmountUnix(ctx context.Context, targetPath string) error { - _, err := os.Stat(targetPath) - if err != nil && os.IsNotExist(err) { - return nil - } - - output, err := utils.ExecShellCmd(ctx, "umount %s", targetPath) - if err != nil && !(strings.Contains(output, "not mounted") || - strings.Contains(output, "not found")) { - log.AddContext(ctx).Errorf("Unmount %s error: %s", targetPath, output) - return err - } - - return nil -} - func removeTargetPath(targetPath string) error { _, err := os.Stat(targetPath) if err != nil && os.IsNotExist(err) { @@ -463,7 +316,7 @@ func removeTargetPath(targetPath string) error { } func tryDisConnectVolume(ctx context.Context, targetPath string) error { - err := unmountUnix(ctx, targetPath) + err := connUtils.Unmount(ctx, targetPath) if err != nil { return err } diff --git a/connector/nfs/nfs_test.go b/connector/nfs/nfs_test.go index edcd32ea..191961c9 100644 --- a/connector/nfs/nfs_test.go +++ b/connector/nfs/nfs_test.go @@ -26,9 +26,9 @@ import ( "github.com/agiledragon/gomonkey/v2" "github.com/prashantv/gostub" - "huawei-csi-driver/connector" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( @@ -169,6 +169,8 @@ func TestDisConnectVolume(t *testing.T) { } stubs := gostub.Stub(&utils.ExecShellCmd, testExecShellCmd) + gomonkey.ApplyFuncReturn(connector.MountPathIsExist, true, nil) + defer stubs.Reset() for _, tt := range tests { diff --git a/connector/nfsplus/nfs_plus.go b/connector/nfsplus/nfs_plus.go index e92081da..3333cdda 100644 --- a/connector/nfsplus/nfs_plus.go +++ b/connector/nfsplus/nfs_plus.go @@ -20,8 +20,8 @@ package nfsplus import ( "context" - "huawei-csi-driver/connector" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // Connector to define a local lock when connect or disconnect, in order to preventing mounting and unmounting confusion diff --git a/connector/nfsplus/nfs_plus_helper.go b/connector/nfsplus/nfs_plus_helper.go index dc9857ba..420f3efd 100644 --- a/connector/nfsplus/nfs_plus_helper.go +++ b/connector/nfsplus/nfs_plus_helper.go @@ -25,12 +25,12 @@ import ( "path" "strings" - "huawei-csi-driver/connector" - "huawei-csi-driver/connector/nfs" - "huawei-csi-driver/csi/backend/plugin" - pkgUtils "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector" + connUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/connector/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend/plugin" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( @@ -128,7 +128,7 @@ func mountNFSPlus(ctx context.Context, conn *connectorInfo) error { if exist { // check the filesystem by comparing the sourcePath and mountPath if value == conn.sourcePath || path.Base(path.Dir(conn.targetPath)) == path.Base(path.Dir(conn.sourcePath)) || - nfs.ContainSourceDevice(ctx, conn.sourcePath, value) { + connUtils.ContainSourceDevice(ctx, conn.sourcePath, value) { log.AddContext(ctx).Infof("Mount %s to %s is already exist", conn.sourcePath, conn.targetPath) return nil } diff --git a/connector/nvme/nvme.go b/connector/nvme/nvme.go index 53270309..83af29b5 100644 --- a/connector/nvme/nvme.go +++ b/connector/nvme/nvme.go @@ -20,9 +20,9 @@ import ( "context" "sync" - "huawei-csi-driver/connector" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // FCNVMe implements the connector.VolumeConnector for FCNVMe protocol diff --git a/connector/nvme/nvme_helper.go b/connector/nvme/nvme_helper.go index d5bf8752..ff685072 100644 --- a/connector/nvme/nvme_helper.go +++ b/connector/nvme/nvme_helper.go @@ -23,9 +23,9 @@ import ( "strings" "time" - "huawei-csi-driver/connector" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const flushTimeInterval = 3 * time.Second diff --git a/connector/nvme/nvme_test.go b/connector/nvme/nvme_test.go index 19272a30..c5ea2c1c 100644 --- a/connector/nvme/nvme_test.go +++ b/connector/nvme/nvme_test.go @@ -26,11 +26,11 @@ import ( "github.com/prashantv/gostub" - "huawei-csi-driver/connector" - "huawei-csi-driver/connector/utils/lock" - "huawei-csi-driver/csi/app" - "huawei-csi-driver/csi/app/config" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector/utils/lock" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/connector/roce/roce.go b/connector/roce/roce.go index abd694f7..bb758ab1 100644 --- a/connector/roce/roce.go +++ b/connector/roce/roce.go @@ -19,9 +19,9 @@ package roce import ( "context" - "huawei-csi-driver/connector" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // Connector implements the connector.VolumeConnector for Connector protocol diff --git a/connector/roce/roce_common.go b/connector/roce/roce_common.go index 40af1312..a798ff5a 100644 --- a/connector/roce/roce_common.go +++ b/connector/roce/roce_common.go @@ -23,9 +23,9 @@ import ( "fmt" "strings" - "huawei-csi-driver/connector" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) func getSessionPorts(ctx context.Context, devices []string, deviceType int) ([]string, error) { diff --git a/connector/roce/roce_helper.go b/connector/roce/roce_helper.go index bfbbea91..876f668d 100644 --- a/connector/roce/roce_helper.go +++ b/connector/roce/roce_helper.go @@ -27,11 +27,11 @@ import ( "sync/atomic" "time" - "huawei-csi-driver/connector" - connutils "huawei-csi-driver/connector/utils" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/concurrent" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector" + connutils "github.com/Huawei/eSDK_K8S_Plugin/v4/connector/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/concurrent" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) type connectorInfo struct { diff --git a/connector/ultrapath.go b/connector/ultrapath.go index aec73a0f..266a2164 100644 --- a/connector/ultrapath.go +++ b/connector/ultrapath.go @@ -26,8 +26,8 @@ import ( "strings" "time" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/connector/ultrapath_test.go b/connector/ultrapath_test.go index 8ae4172f..5db9aec3 100644 --- a/connector/ultrapath_test.go +++ b/connector/ultrapath_test.go @@ -22,7 +22,7 @@ import ( "github.com/prashantv/gostub" - "huawei-csi-driver/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" ) var showArrayOutput = ` diff --git a/connector/ultrapath_vlun.go b/connector/ultrapath_vlun.go index 497ac1bb..f7c7da57 100644 --- a/connector/ultrapath_vlun.go +++ b/connector/ultrapath_vlun.go @@ -23,7 +23,7 @@ import ( "strings" "time" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/connector/utils/lock/lock.go b/connector/utils/lock/lock.go index 7b645b20..cc25aafa 100644 --- a/connector/utils/lock/lock.go +++ b/connector/utils/lock/lock.go @@ -23,9 +23,9 @@ import ( "path/filepath" "time" - "huawei-csi-driver/csi/app" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) var ( diff --git a/connector/utils/lock/lock_utils.go b/connector/utils/lock/lock_utils.go index bb97f72e..ac3ca6ff 100644 --- a/connector/utils/lock/lock_utils.go +++ b/connector/utils/lock/lock_utils.go @@ -27,9 +27,9 @@ import ( "sync" "time" - pkgUtils "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) var mutex sync.Mutex diff --git a/connector/utils/mount.go b/connector/utils/mount.go new file mode 100644 index 00000000..88518dce --- /dev/null +++ b/connector/utils/mount.go @@ -0,0 +1,280 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package utils + +import ( + "context" + "errors" + "fmt" + "os" + "path" + "path/filepath" + "strings" + + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" +) + +const ( + targetMountPathPermission = 0750 +) + +// MountParam is the parameters for mount +type MountParam struct { + DashT string + DashO string +} + +// BindMountRawBlockDevice mounts the raw block device to target path +func BindMountRawBlockDevice(ctx context.Context, sourcePath, targetPath string, mountFlags []string) error { + exists, err := connector.MountPathIsExist(ctx, targetPath) + if err != nil { + return err + } + + mountParams := MountParam{DashO: strings.Join(append(mountFlags, "bind"), ",")} + if exists { + return Mount(ctx, sourcePath, targetPath, mountParams, false) + } + + file, err := os.Lstat(targetPath) + if err != nil { + if !os.IsNotExist(err) { + return fmt.Errorf("failed to lstat target file %s: %w", targetPath, err) + } + + if err = createMountTargetFile(targetPath); err != nil { + return err + } + + return Mount(ctx, sourcePath, targetPath, mountParams, false) + } + + // Check if device file + if file.Mode()&os.ModeDevice == os.ModeDevice { + srcFile, err := os.Stat(sourcePath) + if err != nil { + return fmt.Errorf("failed to stat source file %s: %w", sourcePath, err) + } + if os.SameFile(srcFile, file) { + log.AddContext(ctx).Infof("Skipped bind mount because it is already exist on: %s", targetPath) + return nil + } + + return fmt.Errorf("bind mount already exist on: %s,and it is not the expected device: %s", + targetPath, sourcePath) + } + + // Before CSI V4.6.0, raw block volumes are used by creating symlink of devices, + // so we need to check if the target path is a symlink and recreate it. + if file.Mode()&os.ModeSymlink == os.ModeSymlink { + if err = createMountTargetFile(targetPath); err != nil { + return err + } + } + + log.AddContext(ctx).Infof("File %s is already exist but not mounted, skip creating file", targetPath) + + return Mount(ctx, sourcePath, targetPath, mountParams, false) +} + +// MountToDir mounts source to target which is a directory. +func MountToDir(ctx context.Context, sourcePath, targetPath string, flags MountParam, checkSourcePath bool) error { + err := preMount(sourcePath, targetPath, checkSourcePath) + if err != nil { + return err + } + + return Mount(ctx, sourcePath, targetPath, flags, checkSourcePath) +} + +// Mount mounts source to target with given flags. +func Mount(ctx context.Context, sourcePath, targetPath string, flags MountParam, checkSourcePath bool) error { + var output string + var err error + + mountMap, err := connector.ReadMountPoints(ctx) + value, exist := mountMap[targetPath] + if exist { + // checkSourcePath means the source is block type, need to check the realpath + if checkSourcePath { + err := compareMountPath(ctx, sourcePath, value) + if err != nil { + return err + } + log.AddContext(ctx).Infof("%s is already mount to %s", sourcePath, targetPath) + return nil + } + + // if the checkSourcePath is false, check the filesystem by comparing the sourcePath and mountPath + if value == sourcePath || path.Base(path.Dir(targetPath)) == path.Base(path.Dir(sourcePath)) || + ContainSourceDevice(ctx, sourcePath, value) { + log.AddContext(ctx).Infof("Mount %s to %s is already exist", sourcePath, targetPath) + return nil + } + + return fmt.Errorf("the mount %s is already exist, but the source path is not %s, instead of %s", + targetPath, sourcePath, value) + } + + flags = appendXFSMountFlags(ctx, sourcePath, flags) + + if flags.DashT != "" { + flags.DashT = fmt.Sprintf("-t %s", flags.DashT) + } + + if flags.DashO != "" { + flags.DashO = fmt.Sprintf("-o %s", flags.DashO) + } + + output, err = utils.ExecShellCmd(ctx, "mount %s %s %s %s", sourcePath, targetPath, flags.DashT, flags.DashO) + if err != nil { + log.AddContext(ctx).Errorf("Mount %s to %s failed, error res: %s, error: %s", + sourcePath, targetPath, output, err) + return err + } + + return nil +} + +// Unmount unmounts the target path +func Unmount(ctx context.Context, targetPath string) error { + mounted, err := connector.MountPathIsExist(ctx, targetPath) + if err != nil { + return err + } + if !mounted { + return nil + } + + output, err := utils.ExecShellCmd(ctx, "umount %s", targetPath) + if err != nil && !(strings.Contains(output, "not mounted") || + strings.Contains(output, "not found")) { + log.AddContext(ctx).Errorf("Unmount %s error: %s", targetPath, output) + return err + } + + return nil +} + +// ContainSourceDevice used to check whether target path referenced source device is equal to sourceDev +func ContainSourceDevice(ctx context.Context, targetPath, sourceDev string) bool { + for _, value := range findSourceDevice(ctx, targetPath) { + if value == sourceDev { + return true + } + } + return false +} + +func createMountTargetFile(targetPath string) error { + // Remove old mount target file then create new one + if err := os.Remove(targetPath); err != nil && !os.IsNotExist(err) { + return fmt.Errorf("failed to remove target file %s: %w", targetPath, err) + } + + newFile, err := os.OpenFile(targetPath, os.O_CREATE|os.O_RDWR, targetMountPathPermission) + if err != nil { + return fmt.Errorf("failed to open file %s: %w", targetPath, err) + } + + if err = newFile.Close(); err != nil { + return fmt.Errorf("failed to close file %s: %w", targetPath, err) + } + return nil +} + +func preMount(sourcePath, targetPath string, checkSourcePath bool) error { + if checkSourcePath { + if _, err := os.Stat(sourcePath); err != nil && os.IsNotExist(err) { + return errors.New("source path does not exist") + } + } + + if _, err := os.Stat(targetPath); err != nil && os.IsNotExist(err) { + log.Infof("target path %s does not exist, create it", targetPath) + if err := os.MkdirAll(targetPath, targetMountPathPermission); err != nil { + return errors.New("can not create a target path") + } + } + + return nil +} + +func compareMountPath(ctx context.Context, sourcePath, mountSourcePath string) error { + // the mount source path is like: /dev/mapper/mpath or /dev/sd + // but the source path is like: /dev/dm- or /dev/sd. + // The relationship of these two path as follows: + // lrwxrwxrwx 1 root root 7 Aug 1 17:13 /dev/mapper/mpathc -> ../dm-3 + _, err := connector.ReadDevice(ctx, mountSourcePath) + if err != nil { + return err + } + + mountRealPath, err := filepath.EvalSymlinks(mountSourcePath) + if err != nil { + return err + } + + sourceRealPath, err := filepath.EvalSymlinks(sourcePath) + if err != nil { + return err + } + + if sourceRealPath != mountRealPath { + msg := fmt.Sprintf("The source path is %s, the real path is %s", + sourcePath, mountRealPath) + log.AddContext(ctx).Errorln(msg) + return errors.New(msg) + } + + return nil +} + +// findSourceDevice use findmnt command to find mountPath referenced source device +func findSourceDevice(ctx context.Context, targetPath string) []string { + output, err := utils.ExecShellCmd(ctx, "findmnt -o source --noheadings --target %s", targetPath) + if err != nil { + return []string{} + } + + return strings.Split(output, "\n") +} + +func appendXFSMountFlags(ctx context.Context, sourcePath string, flags MountParam) MountParam { + // Only disk devices need to be determined whether the system is XFS. + if !strings.Contains(sourcePath, "/dev/") { + return flags + } + fsType, err := connector.GetFsTypeByDevPath(ctx, sourcePath) + if err != nil { + log.AddContext(ctx).Warningf("Get device: [%s] FsType failed. error: %v", sourcePath, err) + return flags + } + if fsType == "xfs" { + // xfs volumes are always mounted with '-o nouuid' to allow clones to be mounted to the same node as the source + if flags.DashO == "" { + flags.DashO = "nouuid" + } else { + flags.DashO = fmt.Sprintf("%s,nouuid", flags.DashO) + } + } + + log.AddContext(ctx).Infof("mount flags: [%v]", flags) + return flags +} diff --git a/connector/utils/utils.go b/connector/utils/utils.go index cf5774e6..3b5b362d 100644 --- a/connector/utils/utils.go +++ b/connector/utils/utils.go @@ -24,8 +24,8 @@ import ( "regexp" "strings" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/csi/app/app.go b/csi/app/app.go index 3095bfef..7768dbab 100644 --- a/csi/app/app.go +++ b/csi/app/app.go @@ -24,8 +24,8 @@ import ( "github.com/sirupsen/logrus" "github.com/spf13/cobra" - cfg "huawei-csi-driver/csi/app/config" - "huawei-csi-driver/csi/app/options" + cfg "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app/options" ) const ( diff --git a/csi/app/config/config.go b/csi/app/config/config.go index 2c167eeb..ed2f4180 100644 --- a/csi/app/config/config.go +++ b/csi/app/config/config.go @@ -22,8 +22,8 @@ import ( "github.com/sirupsen/logrus" - clientSet "huawei-csi-driver/pkg/client/clientset/versioned" - "huawei-csi-driver/utils/k8sutils" + clientSet "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/clientset/versioned" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/k8sutils" ) type loggingConfig struct { @@ -53,6 +53,9 @@ type serviceConfig struct { WorkerThreads int BackendUpdateInterval int + ExportCsiServerAddress string + ExportCsiServerPort int + LeaderLeaseDuration time.Duration LeaderRenewDeadline time.Duration LeaderRetryPeriod time.Duration diff --git a/csi/app/config/config_mock.go b/csi/app/config/config_mock.go index d21a8ced..3a358336 100644 --- a/csi/app/config/config_mock.go +++ b/csi/app/config/config_mock.go @@ -20,8 +20,8 @@ package config import ( "time" - clientSet "huawei-csi-driver/pkg/client/clientset/versioned" - "huawei-csi-driver/utils/k8sutils" + clientSet "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/clientset/versioned" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/k8sutils" ) const ( diff --git a/csi/app/options/connector.go b/csi/app/options/connector.go index d38ac079..3f9c0dd4 100644 --- a/csi/app/options/connector.go +++ b/csi/app/options/connector.go @@ -21,7 +21,7 @@ import ( "flag" "fmt" - "huawei-csi-driver/csi/app/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app/config" ) const ( diff --git a/csi/app/options/extender.go b/csi/app/options/extender.go index a8b06f9e..ce2337c9 100644 --- a/csi/app/options/extender.go +++ b/csi/app/options/extender.go @@ -21,7 +21,7 @@ import ( "flag" "time" - "huawei-csi-driver/csi/app/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app/config" ) const ( diff --git a/csi/app/options/k8s.go b/csi/app/options/k8s.go index 24ef5994..c3914d3a 100644 --- a/csi/app/options/k8s.go +++ b/csi/app/options/k8s.go @@ -23,8 +23,8 @@ import ( "github.com/sirupsen/logrus" - "huawei-csi-driver/csi/app/config" - "huawei-csi-driver/pkg/constants" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" ) type k8sOptions struct { diff --git a/csi/app/options/logging.go b/csi/app/options/logging.go index 5ddbfad1..9dd058f3 100644 --- a/csi/app/options/logging.go +++ b/csi/app/options/logging.go @@ -22,7 +22,7 @@ import ( "fmt" "strconv" - "huawei-csi-driver/csi/app/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app/config" ) const ( diff --git a/csi/app/options/options.go b/csi/app/options/options.go index 9991c51c..862c1592 100644 --- a/csi/app/options/options.go +++ b/csi/app/options/options.go @@ -22,7 +22,7 @@ import ( "fmt" "strings" - "huawei-csi-driver/csi/app/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app/config" ) type optionsManager struct { diff --git a/csi/app/options/options_test.go b/csi/app/options/options_test.go index 1fc2ce86..10c88ed9 100644 --- a/csi/app/options/options_test.go +++ b/csi/app/options/options_test.go @@ -27,7 +27,7 @@ import ( "github.com/agiledragon/gomonkey/v2" "github.com/sirupsen/logrus" - "huawei-csi-driver/csi/app/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app/config" ) func initOptions() *optionsManager { diff --git a/csi/app/options/service.go b/csi/app/options/service.go index 4cede849..f66f55ff 100644 --- a/csi/app/options/service.go +++ b/csi/app/options/service.go @@ -22,8 +22,8 @@ import ( "os" "time" - "huawei-csi-driver/csi/app/config" - "huawei-csi-driver/pkg/constants" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" ) const ( @@ -34,6 +34,7 @@ const ( defaultLeaderRenewDeadline = 6 * time.Second defaultLeaderLeaseDuration = 8 * time.Second defaultBackendUpdateIntervalSeconds = 60 + defaultExportCsiServerPort = 9090 ) // serviceOptions include service's configuration @@ -55,6 +56,9 @@ type serviceOptions struct { backendUpdateInterval int workerThreads int + exportCsiServerAddress string + exportCsiServerPort int + leaderLeaseDuration time.Duration leaderRenewDeadline time.Duration leaderRetryPeriod time.Duration @@ -111,6 +115,10 @@ func (opt *serviceOptions) AddFlags(ff *flag.FlagSet) { ff.DurationVar(&opt.timeout, "timeout", defaultRpcTimeout, "timeout for any RPCs") ff.StringVar(&opt.kubeletVolumeDevicesDirName, "kubelet-volume-devices-dir-name", constants.DefaultKubeletVolumeDevicesDirName, "The dir name of volume devices") + ff.IntVar(&opt.exportCsiServerPort, "export-csi-service-port", defaultExportCsiServerPort, + "The port of exported csi server") + ff.StringVar(&opt.exportCsiServerAddress, "export-csi-service-address", "", + "The address of exported csi server") } // ApplyFlags assign the service flags @@ -135,6 +143,8 @@ func (opt *serviceOptions) ApplyFlags(cfg *config.AppConfig) { cfg.WorkerThreads = opt.workerThreads cfg.Timeout = opt.timeout cfg.KubeletVolumeDevicesDirName = opt.kubeletVolumeDevicesDirName + cfg.ExportCsiServerAddress = opt.exportCsiServerAddress + cfg.ExportCsiServerPort = opt.exportCsiServerPort } // ValidateFlags validate the service flags diff --git a/csi/backend/backend.go b/csi/backend/backend.go index 412aba2a..3bd9fe94 100644 --- a/csi/backend/backend.go +++ b/csi/backend/backend.go @@ -26,16 +26,17 @@ import ( "strconv" "strings" - v1 "huawei-csi-driver/client/apis/xuanwu/v1" - "huawei-csi-driver/csi/app" - "huawei-csi-driver/csi/backend/cache" - "huawei-csi-driver/csi/backend/model" - "huawei-csi-driver/csi/backend/plugin" - pkgUtils "huawei-csi-driver/pkg/utils" - fsUtils "huawei-csi-driver/storage/fusionstorage/utils" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/k8sutils" - "huawei-csi-driver/utils/log" + v1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend/cache" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend/model" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend/plugin" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + fsUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/fusionstorage/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/k8sutils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( @@ -486,15 +487,17 @@ func filterByVolumeType(ctx context.Context, volumeType string, candidatePools [ for _, pool := range candidatePools { if volumeType == "lun" || volumeType == "" { - if pool.Storage == "oceanstor-san" || pool.Storage == "fusionstorage-san" { + if pool.Storage == constants.OceanStorSan || pool.Storage == constants.FusionSan || + pool.Storage == constants.OceandiskSan { filterPools = append(filterPools, pool) } } else if volumeType == "fs" { - if pool.Storage == "oceanstor-nas" || pool.Storage == "oceanstor-9000" || pool.Storage == "fusionstorage-nas" { + if pool.Storage == constants.OceanStorNas || pool.Storage == constants.OceanStor9000 || + pool.Storage == constants.FusionNas { filterPools = append(filterPools, pool) } } else if volumeType == "dtree" { - if pool.Storage == "oceanstor-dtree" { + if pool.Storage == constants.OceanStorDtree { filterPools = append(filterPools, pool) } } @@ -510,7 +513,7 @@ func filterByAllocType(ctx context.Context, allocType string, candidatePools []* for _, pool := range candidatePools { valid := false - if pool.Storage == "oceanstor-9000" { + if pool.Storage == constants.OceanStor9000 { valid = true } else if allocType == "thin" || allocType == "" { supportThin, exist := pool.Capabilities["SupportThin"] @@ -829,6 +832,8 @@ func filterByNFSProtocol(ctx context.Context, nfsProtocol string, candidatePools filterPools = append(filterPools, pool) } else if nfsProtocol == "nfs41" && pool.Capabilities["SupportNFS41"] { filterPools = append(filterPools, pool) + } else if nfsProtocol == "nfs42" && pool.Capabilities["SupportNFS42"] { + filterPools = append(filterPools, pool) } } diff --git a/csi/backend/backend_get.go b/csi/backend/backend_get.go index 628c1ce2..df8d67c3 100644 --- a/csi/backend/backend_get.go +++ b/csi/backend/backend_get.go @@ -25,10 +25,10 @@ import ( coreV1 "k8s.io/api/core/v1" - xuanwuV1 "huawei-csi-driver/client/apis/xuanwu/v1" - "huawei-csi-driver/csi/app" - pkgUtils "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/utils/log" + xuanwuV1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // GetBackendConfigmap used to get Configmap diff --git a/csi/backend/backend_test.go b/csi/backend/backend_test.go index 7c4dd5d9..dd2f78b9 100644 --- a/csi/backend/backend_test.go +++ b/csi/backend/backend_test.go @@ -26,11 +26,11 @@ import ( "github.com/prashantv/gostub" "github.com/stretchr/testify/require" - "huawei-csi-driver/csi/app" - cfg "huawei-csi-driver/csi/app/config" - "huawei-csi-driver/csi/backend/cache" - "huawei-csi-driver/csi/backend/model" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + cfg "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend/cache" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend/model" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/csi/backend/cache/backend_cache.go b/csi/backend/cache/backend_cache.go index 79fde0bd..2ea20494 100644 --- a/csi/backend/cache/backend_cache.go +++ b/csi/backend/cache/backend_cache.go @@ -21,8 +21,8 @@ import ( "context" "sync" - "huawei-csi-driver/csi/backend/model" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend/model" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // BackendCacheProvider provider for backend cache diff --git a/csi/backend/cache/backend_cache_test.go b/csi/backend/cache/backend_cache_test.go index db1f154e..d245529a 100644 --- a/csi/backend/cache/backend_cache_test.go +++ b/csi/backend/cache/backend_cache_test.go @@ -25,10 +25,10 @@ import ( "github.com/prashantv/gostub" - "huawei-csi-driver/csi/app" - "huawei-csi-driver/csi/app/config" - "huawei-csi-driver/csi/backend/model" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend/model" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/csi/backend/handler/backend_cache_wrapper.go b/csi/backend/handler/backend_cache_wrapper.go index ae49d42a..bbbeaebd 100644 --- a/csi/backend/handler/backend_cache_wrapper.go +++ b/csi/backend/handler/backend_cache_wrapper.go @@ -20,11 +20,11 @@ package handler import ( "context" - "huawei-csi-driver/client/apis/xuanwu/v1" - "huawei-csi-driver/csi/backend" - "huawei-csi-driver/csi/backend/cache" - "huawei-csi-driver/csi/backend/model" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend/cache" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend/model" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // BackendCacheWrapperInterface wrapping interface of the backend cache, diff --git a/csi/backend/handler/backend_cache_wrapper_test.go b/csi/backend/handler/backend_cache_wrapper_test.go index b2fb3033..456d4e75 100644 --- a/csi/backend/handler/backend_cache_wrapper_test.go +++ b/csi/backend/handler/backend_cache_wrapper_test.go @@ -25,13 +25,13 @@ import ( "github.com/prashantv/gostub" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "huawei-csi-driver/client/apis/xuanwu/v1" - "huawei-csi-driver/csi/app" - "huawei-csi-driver/csi/app/config" - "huawei-csi-driver/csi/backend" - "huawei-csi-driver/csi/backend/model" - "huawei-csi-driver/csi/backend/plugin" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend/model" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend/plugin" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/csi/backend/handler/backend_fetcher.go b/csi/backend/handler/backend_fetcher.go index 52cb73d3..cac4d086 100644 --- a/csi/backend/handler/backend_fetcher.go +++ b/csi/backend/handler/backend_fetcher.go @@ -21,10 +21,10 @@ import ( "errors" "fmt" - "huawei-csi-driver/client/apis/xuanwu/v1" - "huawei-csi-driver/csi/app" - pkgUtils "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // BackendFetchInterface fetch backend operation set diff --git a/csi/backend/handler/backend_fetcher_test.go b/csi/backend/handler/backend_fetcher_test.go index 573e8ead..4f0e0e99 100644 --- a/csi/backend/handler/backend_fetcher_test.go +++ b/csi/backend/handler/backend_fetcher_test.go @@ -23,9 +23,9 @@ import ( "github.com/agiledragon/gomonkey/v2" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - v1 "huawei-csi-driver/client/apis/xuanwu/v1" - clientSet "huawei-csi-driver/pkg/client/clientset/versioned" - pkgUtils "huawei-csi-driver/pkg/utils" + v1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + clientSet "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/clientset/versioned" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" ) func TestBackendFetcher_FetchAllBackends(t *testing.T) { diff --git a/csi/backend/handler/backend_register.go b/csi/backend/handler/backend_register.go index 8776fc73..7442d04c 100644 --- a/csi/backend/handler/backend_register.go +++ b/csi/backend/handler/backend_register.go @@ -20,10 +20,10 @@ import ( "context" "fmt" - "huawei-csi-driver/client/apis/xuanwu/v1" - "huawei-csi-driver/csi/backend/model" - pkgUtils "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend/model" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // BackendRegisterInterface register backend operation set diff --git a/csi/backend/handler/backend_register_test.go b/csi/backend/handler/backend_register_test.go index c220fe9a..fe882b47 100644 --- a/csi/backend/handler/backend_register_test.go +++ b/csi/backend/handler/backend_register_test.go @@ -24,8 +24,8 @@ import ( "github.com/agiledragon/gomonkey/v2" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - v1 "huawei-csi-driver/client/apis/xuanwu/v1" - "huawei-csi-driver/csi/backend/model" + v1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend/model" ) func TestBackendRegister_FetchAndRegisterAllBackend(t *testing.T) { diff --git a/csi/backend/handler/backend_selector.go b/csi/backend/handler/backend_selector.go index c421a5f3..1f9cf12a 100644 --- a/csi/backend/handler/backend_selector.go +++ b/csi/backend/handler/backend_selector.go @@ -20,10 +20,10 @@ import ( "context" "fmt" - "huawei-csi-driver/csi/backend" - "huawei-csi-driver/csi/backend/model" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend/model" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // BackendSelectInterface all backend select operation set diff --git a/csi/backend/handler/backend_selector_test.go b/csi/backend/handler/backend_selector_test.go index 516a8683..d242e71f 100644 --- a/csi/backend/handler/backend_selector_test.go +++ b/csi/backend/handler/backend_selector_test.go @@ -24,8 +24,8 @@ import ( "github.com/agiledragon/gomonkey/v2" - "huawei-csi-driver/csi/backend" - "huawei-csi-driver/csi/backend/model" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend/model" ) func TestBackendSelector_SelectBackend(t *testing.T) { diff --git a/csi/backend/handler/backend_service.go b/csi/backend/handler/backend_service.go index a8a70d1e..af94c68e 100644 --- a/csi/backend/handler/backend_service.go +++ b/csi/backend/handler/backend_service.go @@ -20,10 +20,10 @@ import ( "context" "strconv" - "huawei-csi-driver/lib/drcsi" - "huawei-csi-driver/pkg/constants" - pkgUtils "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/lib/drcsi" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // StorageBackendDetails backend details diff --git a/csi/backend/job/backend_sync_job.go b/csi/backend/job/backend_sync_job.go index 1e6eb976..dee5b8f1 100644 --- a/csi/backend/job/backend_sync_job.go +++ b/csi/backend/job/backend_sync_job.go @@ -18,10 +18,10 @@ package job import ( - "huawei-csi-driver/csi/backend/handler" - pkgUtils "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend/handler" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) var backendSyncInterface = handler.BackendRegisterInterface(nil) diff --git a/csi/backend/model/model.go b/csi/backend/model/model.go index f94e1a46..9b442f6e 100644 --- a/csi/backend/model/model.go +++ b/csi/backend/model/model.go @@ -20,9 +20,9 @@ package model import ( "context" - xuanwuV1 "huawei-csi-driver/client/apis/xuanwu/v1" - "huawei-csi-driver/csi/backend/plugin" - "huawei-csi-driver/utils/log" + xuanwuV1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend/plugin" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // StorageBackendTuple contains sbc and sbct diff --git a/csi/backend/model/storage_pool.go b/csi/backend/model/storage_pool.go index 4bd89142..c16d25ba 100644 --- a/csi/backend/model/storage_pool.go +++ b/csi/backend/model/storage_pool.go @@ -20,9 +20,9 @@ package model import ( "context" - xuanwuV1 "huawei-csi-driver/client/apis/xuanwu/v1" - "huawei-csi-driver/csi/backend/plugin" - "huawei-csi-driver/utils/log" + xuanwuV1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend/plugin" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // StoragePool field and method of storage pool diff --git a/csi/backend/model/storage_pool_test.go b/csi/backend/model/storage_pool_test.go index d7a1c37a..7a413d52 100644 --- a/csi/backend/model/storage_pool_test.go +++ b/csi/backend/model/storage_pool_test.go @@ -25,10 +25,10 @@ import ( "github.com/prashantv/gostub" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" - "huawei-csi-driver/csi/app" - "huawei-csi-driver/csi/app/config" - "huawei-csi-driver/utils/log" + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/csi/backend/plugin/fusionstorage-nas.go b/csi/backend/plugin/fusionstorage-nas.go index 891cdede..93d3a291 100644 --- a/csi/backend/plugin/fusionstorage-nas.go +++ b/csi/backend/plugin/fusionstorage-nas.go @@ -21,10 +21,11 @@ import ( "errors" "fmt" - "huawei-csi-driver/storage/fusionstorage/client" - "huawei-csi-driver/storage/fusionstorage/volume" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/fusionstorage/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/fusionstorage/volume" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // FusionStorageNasPlugin implements storage StoragePlugin interface @@ -76,21 +77,13 @@ func (p *FusionStorageNasPlugin) updateNasCapacity(ctx context.Context, if !exist { return utils.Errorf(ctx, "the size does not exist in parameters %v", parameters) } - params["capacity"] = utils.RoundUpSize(size, fileCapacityUnit) + params["capacity"] = utils.RoundUpSize(size, constants.FusionFileCapacityUnit) return nil } // CreateVolume used to create volume func (p *FusionStorageNasPlugin) CreateVolume(ctx context.Context, name string, parameters map[string]interface{}) ( utils.Volume, error) { - - size, ok := parameters["size"].(int64) - // for fusionStorage filesystem, the unit is KiB - if !ok || !utils.IsCapacityAvailable(size, fileCapacityUnit) { - return nil, utils.Errorf(ctx, "Create Volume: the capacity %d is not an integer or not multiple of %d.", - size, fileCapacityUnit) - } - params, err := p.getParams(name, parameters) if err != nil { return nil, err @@ -161,7 +154,7 @@ func (p *FusionStorageNasPlugin) ExpandVolume(ctx context.Context, name string, size int64) (bool, error) { nas := volume.NewNAS(p.cli) - return false, nas.Expand(ctx, name, size) + return false, nas.Expand(ctx, name, utils.TransK8SCapacity(size, p.GetSectorSize())) } // UpdatePoolCapabilities used to update pool capabilities @@ -170,6 +163,11 @@ func (p *FusionStorageNasPlugin) UpdatePoolCapabilities(ctx context.Context, return p.updatePoolCapabilities(ctx, poolNames, FusionStorageNas) } +// GetSectorSize get sector size of plugin +func (p *FusionStorageNasPlugin) GetSectorSize() int64 { + return constants.FusionFileCapacityUnit +} + func (p *FusionStorageNasPlugin) updateNFS4Capability(ctx context.Context, capabilities map[string]interface{}) error { if capabilities == nil { capabilities = make(map[string]interface{}) @@ -247,13 +245,3 @@ func (p *FusionStorageNasPlugin) verifyFusionStorageNasParam(ctx context.Context return nil } - -// DeleteDTreeVolume used to delete DTree volume -func (p *FusionStorageNasPlugin) DeleteDTreeVolume(ctx context.Context, m map[string]interface{}) error { - return errors.New("not implement") -} - -// ExpandDTreeVolume used to expand DTree volume -func (p *FusionStorageNasPlugin) ExpandDTreeVolume(ctx context.Context, m map[string]interface{}) (bool, error) { - return false, errors.New("not implement") -} diff --git a/csi/backend/plugin/fusionstorage-san.go b/csi/backend/plugin/fusionstorage-san.go index bddc5e3b..e9cfa654 100644 --- a/csi/backend/plugin/fusionstorage-san.go +++ b/csi/backend/plugin/fusionstorage-san.go @@ -25,12 +25,13 @@ import ( "strings" "sync" - "huawei-csi-driver/proto" - "huawei-csi-driver/storage/fusionstorage/attacher" - "huawei-csi-driver/storage/fusionstorage/client" - "huawei-csi-driver/storage/fusionstorage/volume" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" + "github.com/Huawei/eSDK_K8S_Plugin/v4/proto" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/fusionstorage/attacher" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/fusionstorage/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/fusionstorage/volume" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // FusionStorageSanPlugin implements storage StoragePlugin interface @@ -143,16 +144,6 @@ func (p *FusionStorageSanPlugin) getParams(name string, // CreateVolume used to create volume func (p *FusionStorageSanPlugin) CreateVolume(ctx context.Context, name string, parameters map[string]interface{}) ( utils.Volume, error) { - - size, ok := parameters["size"].(int64) - // for fusionStorage block, the unit is MiB - if !ok || !utils.IsCapacityAvailable(size, CapacityUnit) { - msg := fmt.Sprintf("Create Volume: the capacity %d is not an integer or not multiple of %d.", - size, CapacityUnit) - log.AddContext(ctx).Errorln(msg) - return nil, errors.New(msg) - } - params, err := p.getParams(name, parameters) if err != nil { return nil, err @@ -182,15 +173,8 @@ func (p *FusionStorageSanPlugin) DeleteVolume(ctx context.Context, name string) // ExpandVolume used to expand volume func (p *FusionStorageSanPlugin) ExpandVolume(ctx context.Context, name string, size int64) (bool, error) { - // for fusionStorage block, the unit is MiB - if !utils.IsCapacityAvailable(size, CapacityUnit) { - return false, utils.Errorf(ctx, "Expand Volume: the capacity %d is not an integer multiple of %d.", - size, CapacityUnit) - } san := volume.NewSAN(p.cli) - newSize := utils.TransVolumeCapacity(size, CapacityUnit) - isAttach, err := san.Expand(ctx, name, newSize) - return isAttach, err + return san.Expand(ctx, name, size) } // AttachVolume attach volume to node and return storage mapping info. @@ -234,6 +218,11 @@ func (p *FusionStorageSanPlugin) DetachVolume(ctx context.Context, return nil } +// GetSectorSize get sector size of plugin +func (p *FusionStorageSanPlugin) GetSectorSize() int64 { + return constants.FusionAllocUnitBytes +} + func (p *FusionStorageSanPlugin) mutexReleaseClient(ctx context.Context, plugin *FusionStorageSanPlugin, cli *client.RestClient) { @@ -349,13 +338,3 @@ func (p *FusionStorageSanPlugin) verifyFusionStorageSanParam(ctx context.Context return nil } - -// DeleteDTreeVolume used to delete DTree volume -func (p *FusionStorageSanPlugin) DeleteDTreeVolume(ctx context.Context, m map[string]interface{}) error { - return errors.New("not implement") -} - -// ExpandDTreeVolume used to expand DTree volume -func (p *FusionStorageSanPlugin) ExpandDTreeVolume(ctx context.Context, m map[string]interface{}) (bool, error) { - return false, errors.New("not implement") -} diff --git a/csi/backend/plugin/fusionstorage.go b/csi/backend/plugin/fusionstorage.go index 93f5cc20..391c741b 100644 --- a/csi/backend/plugin/fusionstorage.go +++ b/csi/backend/plugin/fusionstorage.go @@ -22,18 +22,17 @@ import ( "fmt" "strings" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" - pkgUtils "huawei-csi-driver/pkg/utils" - pkgVolume "huawei-csi-driver/pkg/volume" - "huawei-csi-driver/storage/fusionstorage/client" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + pkgVolume "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/volume" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/fusionstorage/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( // CapacityUnit unit of capacity - CapacityUnit int64 = 1024 * 1024 - fileCapacityUnit int64 = 1024 + CapacityUnit int64 = 1024 * 1024 // ProtocolDpc protocol DPC string ProtocolDpc = "dpc" @@ -226,9 +225,19 @@ func (p *FusionStoragePlugin) getNewClientConfig(ctx context.Context, return newClientConfig, nil } +// DeleteDTreeVolume used to delete DTree volume +func (p *FusionStoragePlugin) DeleteDTreeVolume(ctx context.Context, m map[string]interface{}) error { + return errors.New("fusion storage does not support DTree feature") +} + +// ExpandDTreeVolume used to expand DTree volume +func (p *FusionStoragePlugin) ExpandDTreeVolume(context.Context, string, string, int64) (bool, error) { + return false, errors.New("fusion storage does not support DTree feature") +} + // ModifyVolume used to modify volume hyperMetro status func (p *FusionStoragePlugin) ModifyVolume(ctx context.Context, volumeName string, modifyType pkgVolume.ModifyVolumeType, param map[string]string) error { - return errors.New("not implement") + return errors.New("fusion storage does not support modify volume feature") } diff --git a/csi/backend/plugin/oceandisk-san.go b/csi/backend/plugin/oceandisk-san.go new file mode 100644 index 00000000..67d75b5f --- /dev/null +++ b/csi/backend/plugin/oceandisk-san.go @@ -0,0 +1,302 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package plugin + +import ( + "context" + "errors" + "fmt" + + pkgVolume "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/volume" + "github.com/Huawei/eSDK_K8S_Plugin/v4/proto" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceandisk/attacher" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceandisk/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceandisk/volume" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" +) + +const oceandiskSanBackend = "oceandisk-san" + +// OceandiskSanPlugin implements storage StoragePlugin interface +type OceandiskSanPlugin struct { + OceandiskPlugin + attacher *attacher.OceandiskAttacher +} + +func init() { + RegPlugin(oceandiskSanBackend, &OceandiskSanPlugin{}) +} + +// NewPlugin used to create new plugin +func (p *OceandiskSanPlugin) NewPlugin() StoragePlugin { + return &OceandiskSanPlugin{} +} + +// Init used to init the plugin +func (p *OceandiskSanPlugin) Init(ctx context.Context, config map[string]interface{}, + parameters map[string]interface{}, keepLogin bool) error { + protocol, exist := parameters["protocol"].(string) + if !exist || (protocol != "iscsi" && protocol != "fc" && protocol != "roce") { + return errors.New("protocol must be provided as 'iscsi', 'fc' or " + + "'roce' for oceandisk-san backend") + } + + alua, _ := parameters["ALUA"].(map[string]interface{}) + + var ips []string + var err error + if protocol == "iscsi" || protocol == "roce" { + portals, exist := parameters["portals"].([]interface{}) + if !exist { + return fmt.Errorf("portals are required to configure for %s backend", protocol) + } + + ips, err = proto.VerifyIscsiPortals(ctx, portals) + if err != nil { + return err + } + } + + err = p.init(ctx, config, keepLogin) + if err != nil { + return err + } + + p.attacher = attacher.NewOceanDiskAttacher(attacher.OceanDiskAttacherConfig{ + Cli: p.cli, + Protocol: protocol, + Invoker: csiInvoker, + Portals: ips, + Alua: alua, + }) + + return nil +} + +func (p *OceandiskSanPlugin) getSanObj() *volume.SAN { + return volume.NewSAN(p.cli) +} + +// CreateVolume used to create volume +func (p *OceandiskSanPlugin) CreateVolume(ctx context.Context, name string, + parameters map[string]interface{}) (utils.Volume, error) { + + params := getParams(ctx, name, parameters) + san := p.getSanObj() + + return san.Create(ctx, params) +} + +// QueryVolume used to query volume +func (p *OceandiskSanPlugin) QueryVolume(ctx context.Context, name string, params map[string]interface{}) ( + utils.Volume, error) { + san := p.getSanObj() + return san.Query(ctx, name) +} + +// DeleteVolume used to delete volume +func (p *OceandiskSanPlugin) DeleteVolume(ctx context.Context, name string) error { + san := p.getSanObj() + return san.Delete(ctx, name) +} + +// ExpandVolume used to expand volume +func (p *OceandiskSanPlugin) ExpandVolume(ctx context.Context, name string, size int64) (bool, error) { + san := p.getSanObj() + return san.Expand(ctx, name, size) +} + +// AttachVolume attach volume to node,return storage mapping info. +func (p *OceandiskSanPlugin) AttachVolume(ctx context.Context, name string, + parameters map[string]interface{}) (map[string]interface{}, error) { + namespace, err := p.cli.GetNamespaceByName(ctx, name) + if err != nil { + return nil, fmt.Errorf("get namespace %s error: %v", name, err) + } + + if namespace == nil { + return nil, fmt.Errorf("get empty namespace info, namespaceName: %v", name) + } + + return p.attacher.ControllerAttach(ctx, name, parameters) +} + +// DetachVolume used to detach volume from node +func (p *OceandiskSanPlugin) DetachVolume(ctx context.Context, name string, parameters map[string]interface{}) error { + namespace, err := p.cli.GetNamespaceByName(ctx, name) + if err != nil { + return fmt.Errorf("get namespace %s error: %v", name, err) + } + + if namespace == nil { + log.AddContext(ctx).Warningf("namespace %s to detach doesn't exist", name) + return nil + } + + _, err = p.attacher.ControllerDetach(ctx, name, parameters) + return err +} + +// UpdatePoolCapabilities used to update pool capabilities +func (p *OceandiskSanPlugin) UpdatePoolCapabilities(ctx context.Context, + poolNames []string) (map[string]interface{}, error) { + return p.updatePoolCapacities(ctx, poolNames) +} + +// CreateSnapshot used to create snapshot +func (p *OceandiskSanPlugin) CreateSnapshot(ctx context.Context, + namespaceName, snapshotName string) (map[string]interface{}, error) { + return nil, errors.New("oceandisk does not support snapshot feature") +} + +// DeleteSnapshot used to delete snapshot +func (p *OceandiskSanPlugin) DeleteSnapshot(ctx context.Context, + snapshotParentID, snapshotName string) error { + return errors.New("oceandisk does not support snapshot feature") +} + +// UpdateBackendCapabilities to update the block storage capabilities +func (p *OceandiskSanPlugin) UpdateBackendCapabilities(ctx context.Context) (map[string]interface{}, + map[string]interface{}, error) { + return p.OceandiskPlugin.UpdateBackendCapabilities() +} + +// Validate used to validate OceandiskSanPlugin parameters +func (p *OceandiskSanPlugin) Validate(ctx context.Context, param map[string]interface{}) error { + log.AddContext(ctx).Infoln("Start to validate OceandiskSanPlugin parameters.") + + err := p.verifyOceandiskSanParam(ctx, param) + if err != nil { + return err + } + + clientConfig, err := p.getNewClientConfig(ctx, param) + if err != nil { + return err + } + + // Login verification + cli, err := client.NewClient(ctx, clientConfig) + if err != nil { + return err + } + + err = cli.ValidateLogin(ctx) + if err != nil { + return err + } + cli.Logout(ctx) + + return nil +} + +func (p *OceandiskSanPlugin) getNewClientConfig(ctx context.Context, + param map[string]interface{}) (*client.NewClientConfig, error) { + data := &client.NewClientConfig{} + configUrls, ok := utils.GetValue[[]interface{}](param, "urls") + if !ok || len(configUrls) <= 0 { + return data, fmt.Errorf("verify urls: [%v] failed. urls must be provided", param["urls"]) + } + + for _, configUrl := range configUrls { + url, ok := configUrl.(string) + if !ok { + return data, fmt.Errorf("verify url: [%v] failed. url convert to string failed", configUrl) + } + data.Urls = append(data.Urls, url) + } + + data.User, ok = utils.GetValue[string](param, "user") + if !ok { + return data, fmt.Errorf("can not convert user type %T to string", param["user"]) + } + + data.SecretName, ok = utils.GetValue[string](param, "secretName") + if !ok { + return data, fmt.Errorf("can not convert secretName type %T to string", param["secretName"]) + } + + data.SecretNamespace, ok = utils.GetValue[string](param, "secretNamespace") + if !ok { + return data, fmt.Errorf("can not convert secretNamespace type %T to string", param["secretNamespace"]) + } + + data.BackendID, ok = utils.GetValue[string](param, "backendID") + if !ok { + return data, fmt.Errorf("can not convert backendID type %T to string", param["backendID"]) + } + + data.ParallelNum, _ = utils.GetValue[string](param, "maxClientThreads") + data.UseCert, _ = utils.GetValue[bool](param, "useCert") + data.CertSecretMeta, _ = utils.GetValue[string](param, "certSecret") + + return data, nil +} + +func (p *OceandiskSanPlugin) verifyOceandiskSanParam(ctx context.Context, config map[string]interface{}) error { + parameters, exist := config["parameters"].(map[string]interface{}) + if !exist { + return fmt.Errorf("verify parameters: [%v] failed. parameters must be provided", config["parameters"]) + } + + return verifyOceandiskProtocolParams(ctx, parameters) +} + +func verifyOceandiskProtocolParams(ctx context.Context, parameters map[string]interface{}) error { + protocol, exist := parameters["protocol"].(string) + if !exist || (protocol != "iscsi" && protocol != "fc" && protocol != "roce") { + return fmt.Errorf("verify protocol: [%v] failed. protocol must be provided and be one of "+ + "[iscsi, fc, roce] for oceandisk-san backend", parameters["protocol"]) + } + + if protocol == "iscsi" || protocol == "roce" { + portals, exist := parameters["portals"].([]interface{}) + if !exist { + return fmt.Errorf("verify portals: [%v] failed. portals are required to configure for "+ + "iscsi or roce for oceandisk-san backend", parameters["portals"]) + } + + _, err := proto.VerifyIscsiPortals(ctx, portals) + if err != nil { + return err + } + } + + return nil +} + +// DeleteDTreeVolume used to delete DTree volume +func (p *OceandiskSanPlugin) DeleteDTreeVolume(ctx context.Context, m map[string]interface{}) error { + return errors.New("oceandisk does not support dtree feature") +} + +// ExpandDTreeVolume used to expand DTree volume +func (p *OceandiskSanPlugin) ExpandDTreeVolume(context.Context, string, string, int64) (bool, error) { + return false, errors.New("oceandisk does not support dtree feature") +} + +// ModifyVolume used to modify volume hyperMetro status +func (p *OceandiskSanPlugin) ModifyVolume(ctx context.Context, volumeName string, + modifyType pkgVolume.ModifyVolumeType, param map[string]string) error { + return errors.New("oceandisk does not support volume modify feature") +} + +// GetSectorSize gets the sector size of plugin +func (p *OceandiskSanPlugin) GetSectorSize() int64 { + return SectorSize +} diff --git a/csi/backend/plugin/oceandisk.go b/csi/backend/plugin/oceandisk.go new file mode 100644 index 00000000..4e2b27a0 --- /dev/null +++ b/csi/backend/plugin/oceandisk.go @@ -0,0 +1,125 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package plugin + +import ( + "context" + + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceandisk/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceandisk/smartx" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" +) + +// OceandiskPlugin provides oceandisk plugin base operations +type OceandiskPlugin struct { + basePlugin + + cli client.OceandiskClientInterface +} + +func (p *OceandiskPlugin) init(ctx context.Context, config map[string]interface{}, keepLogin bool) error { + backendClientConfig, err := formatOceandiskInitParam(config) + if err != nil { + return err + } + + cli, err := client.NewClient(ctx, backendClientConfig) + if err != nil { + return err + } + + if err = cli.Login(ctx); err != nil { + log.AddContext(ctx).Errorf("plugin init login failed, err: %v", err) + return err + } + + if err = cli.SetSystemInfo(ctx); err != nil { + cli.Logout(ctx) + log.AddContext(ctx).Errorf("set client info failed, err: %v", err) + return err + } + + p.name = backendClientConfig.Name + p.cli = cli + + if !keepLogin { + cli.Logout(ctx) + } + return nil +} + +func (p *OceandiskPlugin) getBackendCapabilities() map[string]interface{} { + capabilities := map[string]interface{}{ + "SupportThin": true, + "SupportThick": false, + "SupportQoS": true, + "SupportMetro": false, + "SupportReplication": false, + "SupportApplicationType": true, + "SupportClone": false, + "SupportMetroNAS": false, + } + + return capabilities +} + +func (p *OceandiskPlugin) getBackendSpecifications() map[string]interface{} { + specifications := map[string]interface{}{ + "LocalDeviceSN": p.cli.GetDeviceSN(), + } + return specifications +} + +// UpdateBackendCapabilities used to update backend capabilities +func (p *OceandiskPlugin) UpdateBackendCapabilities() (map[string]interface{}, map[string]interface{}, error) { + return p.getBackendCapabilities(), p.getBackendSpecifications(), nil +} + +func (p *OceandiskPlugin) updatePoolCapacities(ctx context.Context, + poolNames []string) (map[string]interface{}, error) { + pools, err := p.cli.GetAllPools(ctx) + if err != nil { + log.AddContext(ctx).Errorf("get all pools error: %v", err) + return nil, err + } + + log.AddContext(ctx).Debugf("get pools: %v", pools) + + var validPools []map[string]interface{} + for _, name := range poolNames { + if pool, exist := pools[name].(map[string]interface{}); exist { + validPools = append(validPools, pool) + } else { + log.AddContext(ctx).Warningf("pool %s does not exist", name) + } + } + + capacities := analyzePoolsCapacity(ctx, validPools, nil) + return capacities, nil +} + +// SupportQoSParameters checks requested QoS parameters support by Oceandisk plugin +func (p *OceandiskPlugin) SupportQoSParameters(ctx context.Context, qosConfig string) error { + return smartx.CheckQoSParameterSupport(ctx, qosConfig) +} + +// Logout is to logout the storage session +func (p *OceandiskPlugin) Logout(ctx context.Context) { + if p.cli != nil { + p.cli.Logout(ctx) + } +} diff --git a/csi/backend/plugin/oceanstor-dtree.go b/csi/backend/plugin/oceanstor-dtree.go index 77c8e5f1..62bf5626 100644 --- a/csi/backend/plugin/oceanstor-dtree.go +++ b/csi/backend/plugin/oceanstor-dtree.go @@ -21,14 +21,14 @@ import ( "errors" "fmt" - v1 "huawei-csi-driver/client/apis/xuanwu/v1" - "huawei-csi-driver/pkg/constants" - pkgUtils "huawei-csi-driver/pkg/utils" - pkgVolume "huawei-csi-driver/pkg/volume" - "huawei-csi-driver/storage/oceanstor/client" - "huawei-csi-driver/storage/oceanstor/volume" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + v1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + pkgVolume "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/volume" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceanstor/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceanstor/volume" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( @@ -93,16 +93,9 @@ func (p *OceanstorDTreePlugin) CreateVolume(ctx context.Context, name string, pa return nil, errors.New("empty parameters") } - size, ok := parameters["size"].(int64) - if !ok || !utils.IsCapacityAvailable(size, SectorSize) { - msg := fmt.Sprintf("Create Volume: the capacity %d is not an integer multiple of 512.", size) - log.AddContext(ctx).Errorln(msg) - return nil, errors.New(msg) - } - parameters["vstoreId"] = p.vStoreId parameters["parentname"] = p.parentName - params := p.getParams(ctx, name, parameters) + params := getParams(ctx, name, parameters) volObj, err := p.getDTreeObj().Create(ctx, params) if err != nil { @@ -136,30 +129,19 @@ func (p *OceanstorDTreePlugin) DeleteDTreeVolume(ctx context.Context, params map } // ExpandDTreeVolume used to expand DTree volume -func (p *OceanstorDTreePlugin) ExpandDTreeVolume(ctx context.Context, params map[string]interface{}) (bool, error) { +func (p *OceanstorDTreePlugin) ExpandDTreeVolume(ctx context.Context, + dTreeName, parentName string, spaceHardQuota int64) (bool, error) { dTree := p.getDTreeObj() - dTreeName, _ := utils.ToStringWithFlag(params["name"]) - spaceHardQuota, ok := params["spacehardquota"].(int64) - if !ok { - log.AddContext(ctx).Errorln("expand dTree volume failed, spacehardquota is not found") - return false, errors.New("spacehardquota not found") - } - - if !utils.IsCapacityAvailable(spaceHardQuota, SectorSize) { - msg := fmt.Sprintf("Create Volume: the capacity %d is not an integer multiple of 512.", spaceHardQuota) - log.AddContext(ctx).Errorln(msg) - return false, errors.New(msg) - } - - parentName, _ := utils.ToStringWithFlag(params["parentname"]) + // The unit of DTree's quota is bytes, but not sector size. + spaceHardQuota = utils.TransK8SCapacity(spaceHardQuota, p.GetSectorSize()) err := dTree.Expand(ctx, parentName, dTreeName, p.vStoreId, spaceHardQuota) if err != nil { log.AddContext(ctx).Errorf("expand dTree volume failed, ") return false, err } log.AddContext(ctx).Infof("expand dTree volume success, parentName: %v, dTreeName: %v,"+ - " vStoreId: %v, spaceHardQuota: %v", params, dTreeName, p.vStoreId, spaceHardQuota) + " vStoreId: %v, spaceHardQuota: %v", parentName, dTreeName, p.vStoreId, spaceHardQuota) return false, nil } @@ -178,7 +160,7 @@ func (p *OceanstorDTreePlugin) ExpandVolume(ctx context.Context, name string, si func (p *OceanstorDTreePlugin) Validate(ctx context.Context, param map[string]interface{}) error { log.AddContext(ctx).Infoln("Start to validate OceanstorDTreePlugin parameters.") - clientConfig, err := p.getNewClientConfig(ctx, param) + clientConfig, err := getNewClientConfig(ctx, param) if err != nil { return err } @@ -307,6 +289,7 @@ func (p *OceanstorDTreePlugin) updateNFS4Capability(ctx context.Context, capabil capabilities["SupportNFS3"] = true capabilities["SupportNFS4"] = false capabilities["SupportNFS41"] = false + capabilities["SupportNFS42"] = false if !nfsServiceSetting["SupportNFS3"] { capabilities["SupportNFS3"] = false @@ -317,6 +300,9 @@ func (p *OceanstorDTreePlugin) updateNFS4Capability(ctx context.Context, capabil if nfsServiceSetting["SupportNFS41"] { capabilities["SupportNFS41"] = true } + if nfsServiceSetting["SupportNFS42"] { + capabilities["SupportNFS42"] = true + } return nil } @@ -326,7 +312,7 @@ func (p *OceanstorDTreePlugin) updateSmartThin(capabilities map[string]interface if capabilities == nil { return nil } - if p.product == "Dorado" || p.product == "DoradoV6" { + if p.product.IsDorado() || p.product.IsDoradoV6OrV7() { capabilities["SupportThin"] = true } return nil diff --git a/csi/backend/plugin/oceanstor-nas.go b/csi/backend/plugin/oceanstor-nas.go index 3708c03c..a46d3d48 100644 --- a/csi/backend/plugin/oceanstor-nas.go +++ b/csi/backend/plugin/oceanstor-nas.go @@ -24,16 +24,16 @@ import ( "net" "strconv" - "huawei-csi-driver/cli/helper" - xuanwuV1 "huawei-csi-driver/client/apis/xuanwu/v1" - "huawei-csi-driver/pkg/constants" - pkgUtils "huawei-csi-driver/pkg/utils" - pkgVolume "huawei-csi-driver/pkg/volume" - "huawei-csi-driver/storage/oceanstor/client" - "huawei-csi-driver/storage/oceanstor/volume" - "huawei-csi-driver/storage/oceanstor/volume/creator" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/helper" + xuanwuV1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + pkgVolume "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/volume" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceanstor/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceanstor/volume" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceanstor/volume/creator" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( @@ -97,10 +97,12 @@ func (p *OceanstorNasPlugin) Init(ctx context.Context, config map[string]interfa return err } - if protocol == ProtocolNfsPlus && p.cli.GetStorageVersion() < constants.MinVersionSupportNfsPlus { + // only Dorado V6 6.1.7 and later versions support this nfs+ feature. + if protocol == ProtocolNfsPlus && (!p.product.IsDoradoV6OrV7() || + (p.product.IsDoradoV6() && p.cli.GetStorageVersion() < constants.MinVersionSupportNfsPlus)) { p.Logout(ctx) - return errors.New("only oceanstor nas version gte 6.1.7 support nfs plus") + return errors.New("current storage version doesn't support nfs+") } return nil @@ -129,7 +131,7 @@ func (p *OceanstorNasPlugin) checkNfsPlusPortalsFormat(portals []string) bool { } func (p *OceanstorNasPlugin) getNasObj() *volume.NAS { - var metroRemoteCli client.BaseClientInterface + var metroRemoteCli client.OceanstorClientInterface if p.metroRemotePlugin != nil { metroRemoteCli = p.metroRemotePlugin.cli @@ -146,14 +148,8 @@ func (p *OceanstorNasPlugin) CreateVolume(ctx context.Context, name string, para return nil, err } } - size, ok := parameters["size"].(int64) - if !ok || !utils.IsCapacityAvailable(size, SectorSize) { - msg := fmt.Sprintf("Create Volume: the capacity %d is not an integer multiple of 512.", size) - log.AddContext(ctx).Errorln(msg) - return nil, errors.New(msg) - } - params := p.getParams(ctx, name, parameters) + params := getParams(ctx, name, parameters) params["metroDomainID"] = p.metroDomainID nas := p.getNasObj() volObj, err := nas.Create(ctx, params) @@ -164,8 +160,8 @@ func (p *OceanstorNasPlugin) CreateVolume(ctx context.Context, name string, para return volObj, nil } -func (p *OceanstorNasPlugin) getClient() (client.BaseClientInterface, client.BaseClientInterface) { - var replicaRemoteCli client.BaseClientInterface +func (p *OceanstorNasPlugin) getClient() (client.OceanstorClientInterface, client.OceanstorClientInterface) { + var replicaRemoteCli client.OceanstorClientInterface if p.replicaRemotePlugin != nil { replicaRemoteCli = p.replicaRemotePlugin.cli } @@ -175,7 +171,7 @@ func (p *OceanstorNasPlugin) getClient() (client.BaseClientInterface, client.Bas // QueryVolume used to query volume func (p *OceanstorNasPlugin) QueryVolume(ctx context.Context, name string, parameters map[string]interface{}) ( utils.Volume, error) { - params := p.getParams(ctx, name, parameters) + params := getParams(ctx, name, parameters) nas := p.getNasObj() return nas.Query(ctx, name, params) } @@ -198,14 +194,8 @@ func (p *OceanstorNasPlugin) ExpandVolume(ctx context.Context, name string, size return false, err } } - if !utils.IsCapacityAvailable(size, SectorSize) { - msg := fmt.Sprintf("Expand Volume: the capacity %d is not an integer multiple of 512.", size) - log.AddContext(ctx).Errorln(msg) - return false, errors.New(msg) - } - newSize := utils.TransVolumeCapacity(size, SectorSize) nas := p.getNasObj() - return false, nas.Expand(ctx, name, newSize) + return false, nas.Expand(ctx, name, size) } // UpdatePoolCapabilities used to update pool capabilities @@ -221,8 +211,10 @@ func (p *OceanstorNasPlugin) UpdatePoolCapabilities(ctx context.Context, } func (p *OceanstorNasPlugin) getVstoreCapacity(ctx context.Context) (map[string]interface{}, error) { - if p.product != constants.OceanStorDoradoV6 || p.cli.GetvStoreName() == "" || - p.cli.GetStorageVersion() < constants.DoradoV615 { + // only Dorado V6 6.1.5 and later versions need to get vStore's capacity. + if !p.product.IsDoradoV6OrV7() || + (p.product.IsDoradoV6() && p.cli.GetStorageVersion() < constants.DoradoV615) || + p.cli.GetvStoreName() == "" { return map[string]interface{}{}, nil } vStore, err := p.cli.GetvStoreByName(ctx, p.cli.GetvStoreName()) @@ -347,7 +339,7 @@ func (p *OceanstorNasPlugin) UpdateBackendCapabilities(ctx context.Context) (map func (p *OceanstorNasPlugin) updateHyperMetroCapability(ctx context.Context, capabilities map[string]interface{}) error { - if p.product == "DoradoV6" { + if p.product.IsDoradoV6OrV7() { capabilities["SupportMetro"] = capabilities["SupportMetroNAS"] } delete(capabilities, "SupportMetroNAS") @@ -367,7 +359,7 @@ func (p *OceanstorNasPlugin) updateHyperMetroCapability(ctx context.Context, } var ok bool - if p.product == "DoradoV6" && vStorePair != nil { + if p.product.IsDoradoV6OrV7() && vStorePair != nil { fsHyperMetroDomain, err := p.cli.GetFSHyperMetroDomain(ctx, vStorePair["DOMAINNAME"].(string)) if err != nil { @@ -402,13 +394,15 @@ func (p *OceanstorNasPlugin) updateHyperMetroCapability(ctx context.Context, } func (p *OceanstorNasPlugin) updateConsistentSnapshotCapability(capabilities, specifications map[string]any) error { - if p.cli.GetStorageVersion() < supportConsistentSnapshotsMinVersion { - capabilities["SupportConsistentSnapshot"] = false + // only storage version gte DoradoV6 6.1.7 or DoradoV7 support consistent snapshot feature. + if p.product.IsDoradoV6() && p.cli.GetStorageVersion() >= supportConsistentSnapshotsMinVersion || + p.product.IsDoradoV7() { + capabilities["SupportConsistentSnapshot"] = true + specifications["ConsistentSnapshotLimits"] = ConsistentSnapshotsSpecification return nil } - capabilities["SupportConsistentSnapshot"] = true - specifications["ConsistentSnapshotLimits"] = ConsistentSnapshotsSpecification + capabilities["SupportConsistentSnapshot"] = false return nil } @@ -433,6 +427,7 @@ func (p *OceanstorNasPlugin) updateNFS4Capability(ctx context.Context, capabilit capabilities["SupportNFS3"] = true capabilities["SupportNFS4"] = false capabilities["SupportNFS41"] = false + capabilities["SupportNFS42"] = false if !nfsServiceSetting["SupportNFS3"] { capabilities["SupportNFS3"] = false @@ -446,6 +441,10 @@ func (p *OceanstorNasPlugin) updateNFS4Capability(ctx context.Context, capabilit capabilities["SupportNFS41"] = true } + if nfsServiceSetting["SupportNFS42"] { + capabilities["SupportNFS42"] = true + } + return nil } @@ -495,7 +494,7 @@ func (p *OceanstorNasPlugin) Validate(ctx context.Context, param map[string]inte return err } - clientConfig, err := p.getNewClientConfig(ctx, param) + clientConfig, err := getNewClientConfig(ctx, param) if err != nil { return err } @@ -521,7 +520,7 @@ func (p *OceanstorNasPlugin) DeleteDTreeVolume(ctx context.Context, m map[string } // ExpandDTreeVolume used to expand DTree volume -func (p *OceanstorNasPlugin) ExpandDTreeVolume(ctx context.Context, m map[string]interface{}) (bool, error) { +func (p *OceanstorNasPlugin) ExpandDTreeVolume(context.Context, string, string, int64) (bool, error) { return false, errors.New("not implement") } diff --git a/csi/backend/plugin/oceanstor-nas_test.go b/csi/backend/plugin/oceanstor-nas_test.go index 431eb6f2..69fe87c2 100644 --- a/csi/backend/plugin/oceanstor-nas_test.go +++ b/csi/backend/plugin/oceanstor-nas_test.go @@ -25,7 +25,8 @@ import ( "github.com/agiledragon/gomonkey/v2" "github.com/stretchr/testify/require" - "huawei-csi-driver/storage/oceanstor/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceanstor/client" ) func TestInit(t *testing.T) { @@ -59,15 +60,15 @@ func TestInit(t *testing.T) { }, } - var cli *client.BaseClient + var cli *client.RestClient p := gomonkey.ApplyMethod(reflect.TypeOf(cli), "Logout", - func(*client.BaseClient, context.Context) {}). + func(*client.RestClient, context.Context) {}). ApplyMethod(reflect.TypeOf(cli), "Login", - func(*client.BaseClient, context.Context) error { + func(*client.RestClient, context.Context) error { return nil }). ApplyMethod(reflect.TypeOf(cli), "SetSystemInfo", - func(*client.BaseClient, context.Context) error { + func(*client.RestClient, context.Context) error { return nil }) defer p.Reset() @@ -104,9 +105,9 @@ func TestValidate(t *testing.T) { "backendID": "mock-backendID", } - m := gomonkey.ApplyMethod(reflect.TypeOf(&client.BaseClient{}), + m := gomonkey.ApplyMethod(reflect.TypeOf(&client.OceanstorClient{}), "ValidateLogin", - func(_ *client.BaseClient, _ context.Context) error { return nil }, + func(_ *client.OceanstorClient, _ context.Context) error { return nil }, ) defer m.Reset() @@ -134,12 +135,12 @@ func TestDeleteRemoteFilesystem_GetFsFailed(t *testing.T) { // arrange p := &OceanstorNasPlugin{} p.metroRemotePlugin = &OceanstorNasPlugin{} - p.metroRemotePlugin.cli = &client.BaseClient{} + p.metroRemotePlugin.cli = &client.OceanstorClient{} want := errors.New("mock GetFileSystemByName failed") // mock m := gomonkey.ApplyMethod(reflect.TypeOf(p.metroRemotePlugin.cli), "GetFileSystemByName", - func(c *client.BaseClient, ctx context.Context, name string) (map[string]interface{}, error) { + func(c *client.OceanstorClient, ctx context.Context, name string) (map[string]interface{}, error) { return nil, errors.New("mock GetFileSystemByName failed") }) defer m.Reset() @@ -155,18 +156,18 @@ func TestDeleteRemoteFilesystem_GetNfsShareByPathFailed(t *testing.T) { // arrange p := &OceanstorNasPlugin{} p.metroRemotePlugin = &OceanstorNasPlugin{} - p.metroRemotePlugin.cli = &client.BaseClient{} + p.metroRemotePlugin.cli = &client.OceanstorClient{} want := errors.New("mock GetNfsShareByPath failed") // mock m := gomonkey.ApplyMethod(reflect.TypeOf(p.metroRemotePlugin.cli), "GetFileSystemByName", - func(c *client.BaseClient, ctx context.Context, name string) (map[string]interface{}, error) { + func(c *client.OceanstorClient, ctx context.Context, name string) (map[string]interface{}, error) { return map[string]interface{}{ "vstoreId": "1", }, nil }) m.ApplyMethod(reflect.TypeOf(p.metroRemotePlugin.cli), "GetNfsShareByPath", - func(c *client.BaseClient, ctx context.Context, path, vStoreID string) (map[string]interface{}, error) { + func(c *client.OceanstorClient, ctx context.Context, path, vStoreID string) (map[string]interface{}, error) { return nil, errors.New("mock GetNfsShareByPath failed") }) defer m.Reset() @@ -182,24 +183,24 @@ func TestDeleteRemoteFilesystem_DeleteNfsShareFailed(t *testing.T) { // arrange p := &OceanstorNasPlugin{} p.metroRemotePlugin = &OceanstorNasPlugin{} - p.metroRemotePlugin.cli = &client.BaseClient{} + p.metroRemotePlugin.cli = &client.OceanstorClient{} want := errors.New("mock DeleteNfsShare failed") // mock m := gomonkey.ApplyMethod(reflect.TypeOf(p.metroRemotePlugin.cli), "GetFileSystemByName", - func(c *client.BaseClient, ctx context.Context, name string) (map[string]interface{}, error) { + func(c *client.OceanstorClient, ctx context.Context, name string) (map[string]interface{}, error) { return map[string]interface{}{ "vstoreId": "1", }, nil }) m.ApplyMethod(reflect.TypeOf(p.metroRemotePlugin.cli), "GetNfsShareByPath", - func(c *client.BaseClient, ctx context.Context, path, vStoreID string) (map[string]interface{}, error) { + func(c *client.OceanstorClient, ctx context.Context, path, vStoreID string) (map[string]interface{}, error) { return map[string]interface{}{ "ID": "test-id", }, nil }) m.ApplyMethod(reflect.TypeOf(p.metroRemotePlugin.cli), "SafeDeleteNfsShare", - func(c *client.BaseClient, ctx context.Context, id, vStoreID string) error { + func(c *client.OceanstorClient, ctx context.Context, id, vStoreID string) error { return errors.New("mock DeleteNfsShare failed") }) defer m.Reset() @@ -215,28 +216,28 @@ func TestDeleteRemoteFilesystem_DeleteFsFailed(t *testing.T) { // arrange p := &OceanstorNasPlugin{} p.metroRemotePlugin = &OceanstorNasPlugin{} - p.metroRemotePlugin.cli = &client.BaseClient{} + p.metroRemotePlugin.cli = &client.OceanstorClient{} want := errors.New("use backend [] deleteFileSystem failed, error: mock DeleteFileSystem failed") // mock m := gomonkey.ApplyMethod(reflect.TypeOf(p.metroRemotePlugin.cli), "GetFileSystemByName", - func(c *client.BaseClient, ctx context.Context, name string) (map[string]interface{}, error) { + func(c *client.OceanstorClient, ctx context.Context, name string) (map[string]interface{}, error) { return map[string]interface{}{ "vstoreId": "1", }, nil }) m.ApplyMethod(reflect.TypeOf(p.metroRemotePlugin.cli), "GetNfsShareByPath", - func(c *client.BaseClient, ctx context.Context, path, vStoreID string) (map[string]interface{}, error) { + func(c *client.OceanstorClient, ctx context.Context, path, vStoreID string) (map[string]interface{}, error) { return map[string]interface{}{ "ID": "test-id", }, nil }) m.ApplyMethod(reflect.TypeOf(p.metroRemotePlugin.cli), "SafeDeleteNfsShare", - func(c *client.BaseClient, ctx context.Context, id, vStoreID string) error { + func(c *client.OceanstorClient, ctx context.Context, id, vStoreID string) error { return nil }) m.ApplyMethod(reflect.TypeOf(p.metroRemotePlugin.cli), "SafeDeleteFileSystem", - func(c *client.BaseClient, ctx context.Context, params map[string]interface{}) error { + func(c *client.OceanstorClient, ctx context.Context, params map[string]interface{}) error { return errors.New("mock DeleteFileSystem failed") }) defer m.Reset() @@ -251,12 +252,12 @@ func TestDeleteRemoteFilesystem_DeleteFsFailed(t *testing.T) { func TestGetLocal2HyperMetroParameters_EmptyParam(t *testing.T) { // arrange p := &OceanstorNasPlugin{} - p.cli = &client.BaseClient{} + p.cli = &client.OceanstorClient{} // mock - var cli *client.BaseClient + var cli *client.OceanstorClient m := gomonkey.ApplyMethod(reflect.TypeOf(cli), "GetFileSystemByName", - func(c *client.BaseClient, ctx context.Context, name string) (map[string]interface{}, error) { + func(c *client.OceanstorClient, ctx context.Context, name string) (map[string]interface{}, error) { return map[string]interface{}{ "CAPACITY": "12345", "DESCRIPTION": "test-description", @@ -288,7 +289,14 @@ func TestOceanstorNasPluginUpdateConsistentSnapshotCapability(t *testing.T) { {version: "6.1.8", supported: true}, } var genNas = func(version string) *OceanstorNasPlugin { - return &OceanstorNasPlugin{OceanstorPlugin: OceanstorPlugin{cli: &client.BaseClient{StorageVersion: version}}} + return &OceanstorNasPlugin{ + OceanstorPlugin: OceanstorPlugin{ + cli: &client.OceanstorClient{ + RestClient: &client.RestClient{StorageVersion: version}, + }, + product: constants.OceanStorDoradoV6, + }, + } } for _, c := range cases { diff --git a/csi/backend/plugin/oceanstor-san.go b/csi/backend/plugin/oceanstor-san.go index 490f208a..b9afe634 100644 --- a/csi/backend/plugin/oceanstor-san.go +++ b/csi/backend/plugin/oceanstor-san.go @@ -25,21 +25,22 @@ import ( "strconv" "sync" - xuanwuV1 "huawei-csi-driver/client/apis/xuanwu/v1" - "huawei-csi-driver/pkg/constants" - pkgVolume "huawei-csi-driver/pkg/volume" - "huawei-csi-driver/proto" - "huawei-csi-driver/storage/oceanstor/attacher" - "huawei-csi-driver/storage/oceanstor/client" - "huawei-csi-driver/storage/oceanstor/volume" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + xuanwuV1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" + pkgVolume "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/volume" + "github.com/Huawei/eSDK_K8S_Plugin/v4/proto" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceanstor/attacher" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceanstor/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceanstor/volume" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( hyperMetroPairRunningStatusNormal = "1" hyperMetroPairRunningStatusPause = "41" reflectResultLength = 2 + csiInvoker = "csi" ) // OceanstorSanPlugin implements storage StoragePlugin interface @@ -57,8 +58,8 @@ type OceanstorSanPlugin struct { } type handlerRequest struct { - localCli client.BaseClientInterface - metroCli client.BaseClientInterface + localCli client.OceanstorClientInterface + metroCli client.OceanstorClientInterface lun map[string]interface{} parameters map[string]interface{} method string @@ -103,7 +104,7 @@ func (p *OceanstorSanPlugin) Init(ctx context.Context, config map[string]interfa return err } - if (protocol == "roce" || protocol == "fc-nvme") && p.product != "DoradoV6" { + if (protocol == "roce" || protocol == "fc-nvme") && !p.product.IsDoradoV6OrV7() { p.Logout(ctx) msg := fmt.Sprintf("The storage backend %s does not support NVME protocol", p.product) @@ -118,8 +119,8 @@ func (p *OceanstorSanPlugin) Init(ctx context.Context, config map[string]interfa } func (p *OceanstorSanPlugin) getSanObj() *volume.SAN { - var metroRemoteCli client.BaseClientInterface - var replicaRemoteCli client.BaseClientInterface + var metroRemoteCli client.OceanstorClientInterface + var replicaRemoteCli client.OceanstorClientInterface if p.metroRemotePlugin != nil { metroRemoteCli = p.metroRemotePlugin.cli @@ -133,23 +134,16 @@ func (p *OceanstorSanPlugin) getSanObj() *volume.SAN { // CreateVolume used to create volume func (p *OceanstorSanPlugin) CreateVolume(ctx context.Context, - name string, - parameters map[string]interface{}) (utils.Volume, error) { - size, ok := parameters["size"].(int64) - if !ok || !utils.IsCapacityAvailable(size, SectorSize) { - msg := fmt.Sprintf("Create Volume: the capacity %d is not an integer multiple of 512.", size) - log.AddContext(ctx).Errorln(msg) - return nil, errors.New(msg) - } + name string, parameters map[string]interface{}) (utils.Volume, error) { - params := p.getParams(ctx, name, parameters) + params := getParams(ctx, name, parameters) san := p.getSanObj() - volObl, err := san.Create(ctx, params) + volObj, err := san.Create(ctx, params) if err != nil { return nil, err } - return volObl, nil + return volObj, nil } // QueryVolume used to query volume @@ -167,15 +161,8 @@ func (p *OceanstorSanPlugin) DeleteVolume(ctx context.Context, name string) erro // ExpandVolume used to expand volume func (p *OceanstorSanPlugin) ExpandVolume(ctx context.Context, name string, size int64) (bool, error) { - if !utils.IsCapacityAvailable(size, SectorSize) { - msg := fmt.Sprintf("Expand Volume: the capacity %d is not an integer multiple of 512.", size) - log.AddContext(ctx).Errorln(msg) - return false, errors.New(msg) - } san := p.getSanObj() - newSize := utils.TransVolumeCapacity(size, constants.AllocationUnitBytes) - isAttach, err := san.Expand(ctx, name, newSize) - return isAttach, err + return san.Expand(ctx, name, size) } func (p *OceanstorSanPlugin) isHyperMetro(ctx context.Context, lun map[string]interface{}) bool { @@ -294,7 +281,7 @@ func (p *OceanstorSanPlugin) handler(ctx context.Context, req handlerRequest) ([ // AttachVolume attach volume to node,return storage mapping info. func (p *OceanstorSanPlugin) AttachVolume(ctx context.Context, name string, parameters map[string]interface{}) (map[string]interface{}, error) { - var localCli, metroCli client.BaseClientInterface + var localCli, metroCli client.OceanstorClientInterface if p.storageOnline { localCli = p.cli } @@ -338,7 +325,7 @@ func (p *OceanstorSanPlugin) AttachVolume(ctx context.Context, name string, // DetachVolume used to detach volume from node func (p *OceanstorSanPlugin) DetachVolume(ctx context.Context, name string, parameters map[string]interface{}) error { - var localCli, metroCli client.BaseClientInterface + var localCli, metroCli client.OceanstorClientInterface if p.storageOnline { localCli = p.cli } @@ -378,7 +365,7 @@ func (p *OceanstorSanPlugin) DetachVolume(ctx context.Context, name string, para func (p *OceanstorSanPlugin) mutexReleaseClient(ctx context.Context, plugin *OceanstorSanPlugin, - cli client.BaseClientInterface) { + cli client.OceanstorClientInterface) { plugin.clientMutex.Lock() defer plugin.clientMutex.Unlock() plugin.clientCount-- @@ -388,7 +375,7 @@ func (p *OceanstorSanPlugin) mutexReleaseClient(ctx context.Context, } } -func (p *OceanstorSanPlugin) releaseClient(ctx context.Context, cli, metroCli client.BaseClientInterface) { +func (p *OceanstorSanPlugin) releaseClient(ctx context.Context, cli, metroCli client.OceanstorClientInterface) { if p.storageOnline { p.mutexReleaseClient(ctx, p, cli) } @@ -411,8 +398,10 @@ func (p *OceanstorSanPlugin) UpdatePoolCapabilities(ctx context.Context, poolNam } func (p *OceanstorSanPlugin) getVstoreCapacity(ctx context.Context) (map[string]interface{}, error) { - if p.product != constants.OceanStorDoradoV6 || p.cli.GetvStoreName() == "" || - p.cli.GetStorageVersion() < constants.DoradoV615 { + // only Dorado V6 6.1.5 and later versions need to get vStore's capacity. + if !p.product.IsDoradoV6OrV7() || + (p.product.IsDoradoV6() && p.cli.GetStorageVersion() < constants.DoradoV615) || + p.cli.GetvStoreName() == "" { return map[string]interface{}{}, nil } vStore, err := p.cli.GetvStoreByName(ctx, p.cli.GetvStoreName()) @@ -485,7 +474,7 @@ func (p *OceanstorSanPlugin) DeleteSnapshot(ctx context.Context, return nil } -func (p *OceanstorSanPlugin) mutexGetClient(ctx context.Context) (client.BaseClientInterface, error) { +func (p *OceanstorSanPlugin) mutexGetClient(ctx context.Context) (client.OceanstorClientInterface, error) { p.clientMutex.Lock() defer p.clientMutex.Unlock() var err error @@ -502,10 +491,10 @@ func (p *OceanstorSanPlugin) mutexGetClient(ctx context.Context) (client.BaseCli return p.cli, err } -func (p *OceanstorSanPlugin) getClient(ctx context.Context) (client.BaseClientInterface, - client.BaseClientInterface, error) { +func (p *OceanstorSanPlugin) getClient(ctx context.Context) (client.OceanstorClientInterface, + client.OceanstorClientInterface, error) { cli, locErr := p.mutexGetClient(ctx) - var metroCli client.BaseClientInterface + var metroCli client.OceanstorClientInterface var rmtErr error if p.metroRemotePlugin != nil { metroCli, rmtErr = p.metroRemotePlugin.mutexGetClient(ctx) @@ -520,7 +509,7 @@ func (p *OceanstorSanPlugin) getClient(ctx context.Context) (client.BaseClientIn return cli, metroCli, nil } -func (p *OceanstorSanPlugin) getLunInfo(ctx context.Context, localCli, remoteCli client.BaseClientInterface, +func (p *OceanstorSanPlugin) getLunInfo(ctx context.Context, localCli, remoteCli client.OceanstorClientInterface, lunName string) (map[string]interface{}, error) { var lun map[string]interface{} var err error @@ -576,7 +565,7 @@ func (p *OceanstorSanPlugin) Validate(ctx context.Context, param map[string]inte return err } - clientConfig, err := p.getNewClientConfig(ctx, param) + clientConfig, err := getNewClientConfig(ctx, param) if err != nil { return err } @@ -636,7 +625,7 @@ func (p *OceanstorSanPlugin) DeleteDTreeVolume(ctx context.Context, m map[string } // ExpandDTreeVolume used to expand DTree volume -func (p *OceanstorSanPlugin) ExpandDTreeVolume(ctx context.Context, m map[string]interface{}) (bool, error) { +func (p *OceanstorSanPlugin) ExpandDTreeVolume(context.Context, string, string, int64) (bool, error) { return false, errors.New("not implement") } diff --git a/csi/backend/plugin/oceanstor.go b/csi/backend/plugin/oceanstor.go index cf3549a0..feded6ff 100644 --- a/csi/backend/plugin/oceanstor.go +++ b/csi/backend/plugin/oceanstor.go @@ -20,17 +20,15 @@ import ( "context" "errors" "fmt" - "strconv" "strings" - xuanwuV1 "huawei-csi-driver/client/apis/xuanwu/v1" - "huawei-csi-driver/pkg/constants" - pkgUtils "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/storage/oceanstor/client" - "huawei-csi-driver/storage/oceanstor/clientv6" - "huawei-csi-driver/storage/oceanstor/smartx" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceanstor/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceanstor/clientv6" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceanstor/smartx" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( @@ -52,13 +50,13 @@ type OceanstorPlugin struct { vStoreId string - cli client.BaseClientInterface - product string + cli client.OceanstorClientInterface + product constants.OceanstorVersion capabilities map[string]interface{} } func (p *OceanstorPlugin) init(ctx context.Context, config map[string]interface{}, keepLogin bool) error { - backendClientConfig, err := p.formatInitParam(config) + backendClientConfig, err := formatOceanstorInitParam(config) if err != nil { return err } @@ -82,7 +80,7 @@ func (p *OceanstorPlugin) init(ctx context.Context, config map[string]interface{ p.name = backendClientConfig.Name p.product = cli.Product - if p.product == constants.OceanStorDoradoV6 { + if p.product.IsDoradoV6OrV7() { clientV6, err := clientv6.NewClientV6(ctx, backendClientConfig) if err != nil { cli.Logout(ctx) @@ -165,13 +163,13 @@ func (p *OceanstorPlugin) updateBackendCapabilities(ctx context.Context) (map[st log.AddContext(ctx).Debugf("Get license feature: %v", features) supportThin := utils.IsSupportFeature(features, "SmartThin") - supportThick := p.product != "Dorado" && p.product != "DoradoV6" + supportThick := !p.product.IsDorado() && !p.product.IsDoradoV6OrV7() supportQoS := utils.IsSupportFeature(features, "SmartQoS") supportMetro := utils.IsSupportFeature(features, "HyperMetro") supportMetroNAS := utils.IsSupportFeature(features, "HyperMetroNAS") supportReplication := utils.IsSupportFeature(features, "HyperReplication") supportClone := utils.IsSupportFeature(features, "HyperClone") || utils.IsSupportFeature(features, "HyperCopy") - supportApplicationType := p.product == "DoradoV6" + supportApplicationType := p.product.IsDoradoV6OrV7() log.AddContext(ctx).Debugf("storageVersion: %v", p.cli.GetStorageVersion()) @@ -228,8 +226,10 @@ func (p *OceanstorPlugin) updateVStorePair(ctx context.Context, specifications m specifications = map[string]interface{}{} } - if p.product != constants.OceanStorDoradoV6 || p.cli.GetvStoreID() == "" || - p.cli.GetStorageVersion() < constants.DoradoV615 { + // only Dorado V6 6.1.5 and later versions need to update vStorePair. + if !p.product.IsDoradoV6OrV7() || + (p.product.IsDoradoV6() && p.cli.GetStorageVersion() < constants.DoradoV615) || + p.cli.GetvStoreID() == "" { log.AddContext(ctx).Debugf("storage product is %s,version is %s, vStore id is %s, "+ "do not update VStorePairId", p.product, p.cli.GetStorageVersion(), p.cli.GetvStoreID()) return @@ -264,7 +264,7 @@ func (p *OceanstorNasPlugin) updateSmartThin(capabilities map[string]interface{} if capabilities == nil { return nil } - if p.product == "Dorado" || p.product == "DoradoV6" { + if p.product.IsDorado() || p.product.IsDoradoV6OrV7() { capabilities["SupportThin"] = true } return nil @@ -288,7 +288,7 @@ func (p *OceanstorPlugin) UpdateBackendCapabilities(ctx context.Context) (map[st return capabilities, specifications, nil } -func (p *OceanstorPlugin) getParams(ctx context.Context, name string, +func getParams(ctx context.Context, name string, parameters map[string]interface{}) map[string]interface{} { params := map[string]interface{}{ @@ -394,52 +394,10 @@ func (p *OceanstorPlugin) updatePoolCapabilities(ctx context.Context, poolNames } } - capabilities := p.analyzePoolsCapacity(ctx, validPools, vStoreQuotaMap) + capabilities := analyzePoolsCapacity(ctx, validPools, vStoreQuotaMap) return capabilities, nil } -func (p *OceanstorPlugin) analyzePoolsCapacity(ctx context.Context, pools []map[string]interface{}, - vStoreQuotaMap map[string]interface{}) map[string]interface{} { - capabilities := make(map[string]interface{}) - - for _, pool := range pools { - name, ok := pool["NAME"].(string) - if !ok { - continue - } - var err error - var freeCapacity, totalCapacity int64 - if freeStr, ok := pool["USERFREECAPACITY"].(string); ok { - freeCapacity, err = strconv.ParseInt(freeStr, constants.DefaultIntBase, constants.DefaultIntBitSize) - } - if totalStr, ok := pool["USERTOTALCAPACITY"].(string); ok { - totalCapacity, err = strconv.ParseInt(totalStr, constants.DefaultIntBase, constants.DefaultIntBitSize) - } - if err != nil { - log.AddContext(ctx).Warningf("parse capacity failed, error: %v", err) - } - poolCapacityMap := map[string]interface{}{ - string(xuanwuV1.FreeCapacity): freeCapacity * constants.AllocationUnitBytes, - string(xuanwuV1.TotalCapacity): totalCapacity * constants.AllocationUnitBytes, - string(xuanwuV1.UsedCapacity): totalCapacity - freeCapacity, - } - if len(vStoreQuotaMap) == 0 { - capabilities[name] = poolCapacityMap - continue - } - log.AddContext(ctx).Debugf("analyzePoolsCapacity poolName: %s, poolCapacity: %+v, vstoreQuota: %+v", - name, poolCapacityMap, vStoreQuotaMap) - free, ok := vStoreQuotaMap[string(xuanwuV1.FreeCapacity)].(int64) - if ok && free < freeCapacity*constants.AllocationUnitBytes { - capabilities[name] = vStoreQuotaMap - } else { - capabilities[name] = poolCapacityMap - } - } - - return capabilities -} - // SupportQoSParameters checks requested QoS parameters support by Oceanstor plugin func (p *OceanstorPlugin) SupportQoSParameters(ctx context.Context, qosConfig string) error { return smartx.CheckQoSParameterSupport(ctx, p.product, qosConfig) @@ -451,8 +409,14 @@ func (p *OceanstorPlugin) Logout(ctx context.Context) { p.cli.Logout(ctx) } } -func (p *OceanstorPlugin) switchClient(ctx context.Context, newClient client.BaseClientInterface) error { - log.AddContext(ctx).Infoln("Using OceanStor V6 or Dorado V6 BaseClient.") + +// GetSectorSize get sector size of plugin +func (p *OceanstorPlugin) GetSectorSize() int64 { + return SectorSize +} + +func (p *OceanstorPlugin) switchClient(ctx context.Context, newClient client.OceanstorClientInterface) error { + log.AddContext(ctx).Infoln("Using OceanStor V6 or Dorado V6 client.") p.cli = newClient err := p.cli.Login(ctx) if err != nil { @@ -471,8 +435,7 @@ func (p *OceanstorPlugin) switchClient(ctx context.Context, newClient client.Bas return nil } -func (p *OceanstorPlugin) getNewClientConfig(ctx context.Context, - param map[string]interface{}) (*client.NewClientConfig, error) { +func getNewClientConfig(ctx context.Context, param map[string]interface{}) (*client.NewClientConfig, error) { data := &client.NewClientConfig{} configUrls, exist := param["urls"].([]interface{}) if !exist || len(configUrls) <= 0 { diff --git a/csi/backend/plugin/plugin.go b/csi/backend/plugin/plugin.go index 31a9d47f..76f5b6da 100644 --- a/csi/backend/plugin/plugin.go +++ b/csi/backend/plugin/plugin.go @@ -20,9 +20,9 @@ import ( "context" // init the nfs connector - _ "huawei-csi-driver/connector/nfs" - pkgVolume "huawei-csi-driver/pkg/volume" - "huawei-csi-driver/utils" + _ "github.com/Huawei/eSDK_K8S_Plugin/v4/connector/nfs" + pkgVolume "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/volume" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" ) // StoragePlugin defines storage plugin interfaces @@ -48,12 +48,14 @@ type StoragePlugin interface { Validate(context.Context, map[string]interface{}) error DeleteDTreeVolume(context.Context, map[string]interface{}) error - ExpandDTreeVolume(context.Context, map[string]interface{}) (bool, error) + ExpandDTreeVolume(context.Context, string, string, int64) (bool, error) // SetOnline sets the online status of plugin SetOnline(bool) // GetOnline gets the online status of plugin GetOnline() bool + // GetSectorSize gets the sector size of plugin + GetSectorSize() int64 } // SmartXQoSQuery provides Quality of Service(QoS) Query operations diff --git a/csi/backend/plugin/plugin_helper.go b/csi/backend/plugin/plugin_helper.go index b8f1b356..a1dd6c58 100644 --- a/csi/backend/plugin/plugin_helper.go +++ b/csi/backend/plugin/plugin_helper.go @@ -17,11 +17,19 @@ package plugin import ( + "context" "errors" "fmt" "net" + "strconv" - pkgUtils "huawei-csi-driver/pkg/utils" + xuanwuV1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + oceandisk "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceandisk/client" + oceanstor "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceanstor/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // verifyProtocolAndPortals verifyProtocolAndPortals @@ -67,3 +75,142 @@ func checkNfsPlusPortalsFormat(portals []string) bool { return true } + +func formatOceanstorInitParam(config map[string]interface{}) (res *oceanstor.NewClientConfig, err error) { + res = &oceanstor.NewClientConfig{} + + configUrls, exist := config["urls"].([]interface{}) + if !exist || len(configUrls) <= 0 { + err = errors.New("urls must be provided") + return + } + for _, i := range configUrls { + res.Urls = append(res.Urls, i.(string)) + } + res.User, exist = config["user"].(string) + if !exist { + err = errors.New("user must be provided") + return + } + res.SecretName, exist = config["secretName"].(string) + if !exist { + err = errors.New("SecretName must be provided") + return + } + res.SecretNamespace, exist = config["secretNamespace"].(string) + if !exist { + err = errors.New("SecretNamespace must be provided") + return + } + res.BackendID, exist = config["backendID"].(string) + if !exist { + err = errors.New("backendID must be provided") + return + } + res.VstoreName, _ = config["vstoreName"].(string) + res.ParallelNum, _ = config["maxClientThreads"].(string) + + res.UseCert, _ = config["useCert"].(bool) + res.CertSecretMeta, _ = config["certSecret"].(string) + + res.Storage, exist = config["storage"].(string) + if !exist { + return nil, errors.New("storage type must be configured for backend") + } + + res.Name, exist = config["name"].(string) + if !exist { + return nil, errors.New("storage name must be configured for backend") + } + return +} + +func formatOceandiskInitParam(config map[string]interface{}) (*oceandisk.NewClientConfig, error) { + res := &oceandisk.NewClientConfig{} + + configUrls, ok := utils.GetValue[[]interface{}](config, "urls") + if !ok || len(configUrls) <= 0 { + return nil, fmt.Errorf("urls is not provided in config, or it is invalid, config: %v", config) + } + for _, i := range configUrls { + res.Urls = append(res.Urls, i.(string)) + } + + res.User, ok = utils.GetValue[string](config, "user") + if !ok { + return nil, fmt.Errorf("user is not provided in config, or it is invalid, config: %v", config) + } + + res.SecretName, ok = utils.GetValue[string](config, "secretName") + if !ok { + return nil, fmt.Errorf("secretName is not provided in config, or it is invalid, config: %v", config) + } + + res.SecretNamespace, ok = utils.GetValue[string](config, "secretNamespace") + if !ok { + return nil, fmt.Errorf("secretNamespace is not provided in config, or it is invalid, config: %v", config) + } + + res.BackendID, ok = utils.GetValue[string](config, "backendID") + if !ok { + return nil, fmt.Errorf("backendID is not provided in config, or it is invalid, config: %v", config) + } + + res.ParallelNum, _ = utils.GetValue[string](config, "maxClientThreads") + res.UseCert, _ = utils.GetValue[bool](config, "useCert") + res.CertSecretMeta, _ = utils.GetValue[string](config, "certSecret") + + res.Storage, ok = utils.GetValue[string](config, "storage") + if !ok { + return nil, fmt.Errorf("storage is not provided in config, or it is invalid, config: %v", config) + } + + res.Name, ok = utils.GetValue[string](config, "name") + if !ok { + return nil, fmt.Errorf("name is not provided in config, or it is invalid, config: %v", config) + } + + return res, nil +} + +func analyzePoolsCapacity(ctx context.Context, pools []map[string]interface{}, + vStoreQuotaMap map[string]interface{}) map[string]interface{} { + capacities := make(map[string]interface{}) + + for _, pool := range pools { + name, ok := pool["NAME"].(string) + if !ok { + continue + } + var err error + var freeCapacity, totalCapacity int64 + if freeStr, ok := pool["USERFREECAPACITY"].(string); ok { + freeCapacity, err = strconv.ParseInt(freeStr, constants.DefaultIntBase, constants.DefaultIntBitSize) + } + if totalStr, ok := pool["USERTOTALCAPACITY"].(string); ok { + totalCapacity, err = strconv.ParseInt(totalStr, constants.DefaultIntBase, constants.DefaultIntBitSize) + } + if err != nil { + log.AddContext(ctx).Warningf("parse capacity failed, error: %v", err) + } + poolCapacityMap := map[string]interface{}{ + string(xuanwuV1.FreeCapacity): freeCapacity * constants.AllocationUnitBytes, + string(xuanwuV1.TotalCapacity): totalCapacity * constants.AllocationUnitBytes, + string(xuanwuV1.UsedCapacity): totalCapacity - freeCapacity, + } + if len(vStoreQuotaMap) == 0 { + capacities[name] = poolCapacityMap + continue + } + log.AddContext(ctx).Debugf("analyzePoolsCapacity poolName: %s, poolCapacity: %+v, vstoreQuota: %+v", + name, poolCapacityMap, vStoreQuotaMap) + free, ok := vStoreQuotaMap[string(xuanwuV1.FreeCapacity)].(int64) + if ok && free < freeCapacity*constants.AllocationUnitBytes { + capacities[name] = vStoreQuotaMap + } else { + capacities[name] = poolCapacityMap + } + } + + return capacities +} diff --git a/csi/backend/plugin/plugin_test.go b/csi/backend/plugin/plugin_test.go index 108369a9..7cc42515 100644 --- a/csi/backend/plugin/plugin_test.go +++ b/csi/backend/plugin/plugin_test.go @@ -22,9 +22,9 @@ import ( "github.com/prashantv/gostub" - "huawei-csi-driver/csi/app" - cfg "huawei-csi-driver/csi/app/config" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + cfg "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/csi/driver/controller.go b/csi/driver/controller.go index 2f095d27..5e8b0868 100644 --- a/csi/driver/controller.go +++ b/csi/driver/controller.go @@ -27,10 +27,10 @@ import ( "google.golang.org/grpc/status" "google.golang.org/protobuf/types/known/timestamppb" - "huawei-csi-driver/csi/app" - "huawei-csi-driver/csi/backend/plugin" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend/plugin" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // CreateVolume used to create volume @@ -112,17 +112,7 @@ func (d *CsiDriver) ControllerExpandVolume(ctx context.Context, req *csi.Control return nil, status.Error(codes.InvalidArgument, "no volume ID provided") } - log.AddContext(ctx).Infof("Start to controller expand volume %s", volumeId) - if req.GetCapacityRange() == nil { - return nil, status.Error(codes.InvalidArgument, "no capacity range provided") - } - - minSize := req.GetCapacityRange().GetRequiredBytes() - maxSize := req.GetCapacityRange().GetLimitBytes() - if 0 < maxSize && maxSize < minSize { - return nil, status.Error(codes.InvalidArgument, "limitBytes is smaller than requiredBytes") - } - + log.AddContext(ctx).Infof("Start to controller expand volume %s, req: %v", volumeId, req) backendName, volName := utils.SplitVolumeId(volumeId) backend, err := d.backendSelector.SelectBackend(ctx, backendName) if backend == nil || err != nil { @@ -131,19 +121,32 @@ func (d *CsiDriver) ControllerExpandVolume(ctx context.Context, req *csi.Control return nil, status.Error(codes.Internal, msg) } - if support, err := isSupportExpandVolume(ctx, req, backend); !support { - return nil, status.Error(codes.InvalidArgument, err.Error()) + err = verifyExpandArguments(ctx, req, backend) + if err != nil { + msg := fmt.Sprintf("Verify expand arguments error: %v", err) + log.AddContext(ctx).Errorln(msg) + return nil, status.Error(codes.InvalidArgument, msg) } + minSize := req.GetCapacityRange().GetRequiredBytes() + sectorSize := backend.Plugin.GetSectorSize() + size := utils.TransVolumeCapacity(minSize, sectorSize) + if size < minSize { + log.AddContext(ctx).Infof("Required capacity is %d,"+ + " actual capacity %d is an integer multiple of the required sector size %d", minSize, size, sectorSize) + } var nodeExpansionRequired bool if backend.Storage == plugin.DTreeStorage { - nodeExpansionRequired, err = backend.Plugin.ExpandDTreeVolume(ctx, map[string]interface{}{ - "name": volName, - "parentname": backend.Parameters["parentname"], - "spacehardquota": minSize, - }) + parentName, ok := backend.Parameters["parentname"].(string) + if !ok || parentName == "" { + msg := "DTree volume must provide parent name" + log.AddContext(ctx).Errorln(msg) + return nil, status.Error(codes.InvalidArgument, msg) + } + + nodeExpansionRequired, err = backend.Plugin.ExpandDTreeVolume(ctx, volName, parentName, size) } else { - nodeExpansionRequired, err = backend.Plugin.ExpandVolume(ctx, volName, minSize) + nodeExpansionRequired, err = backend.Plugin.ExpandVolume(ctx, volName, size) } if err != nil { log.AddContext(ctx).Errorf("Expand volume %s error: %v", volumeId, err) @@ -151,11 +154,10 @@ func (d *CsiDriver) ControllerExpandVolume(ctx context.Context, req *csi.Control } log.AddContext(ctx).Infof("Volume %s is expanded to %d, nodeExpansionRequired %t", - volName, minSize, nodeExpansionRequired) + volName, size, nodeExpansionRequired) return &csi.ControllerExpandVolumeResponse{ - CapacityBytes: minSize, - NodeExpansionRequired: nodeExpansionRequired, - }, nil + CapacityBytes: utils.TransK8SCapacity(size, sectorSize), + NodeExpansionRequired: nodeExpansionRequired}, nil } // ControllerPublishVolume used to controller publish volume diff --git a/csi/driver/controller_helper.go b/csi/driver/controller_helper.go index db5d8f60..1e72a798 100644 --- a/csi/driver/controller_helper.go +++ b/csi/driver/controller_helper.go @@ -28,15 +28,15 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "huawei-csi-driver/cli/helper" - "huawei-csi-driver/csi/app" - "huawei-csi-driver/csi/backend" - "huawei-csi-driver/csi/backend/handler" - "huawei-csi-driver/csi/backend/model" - "huawei-csi-driver/csi/backend/plugin" - "huawei-csi-driver/pkg/constants" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/helper" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend/handler" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend/model" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend/plugin" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( @@ -62,6 +62,7 @@ var ( "nfsvers=4": "nfs4", "nfsvers=4.0": "nfs4", "nfsvers=4.1": "nfs41", + "nfsvers=4.2": "nfs42", } annManageVolumeName = "/manageVolumeName" @@ -109,6 +110,38 @@ func processNFSProtocol(ctx context.Context, req *csi.CreateVolumeRequest, return nil } +func verifyExpandArguments(ctx context.Context, req *csi.ControllerExpandVolumeRequest, backend *model.Backend) error { + if req.GetCapacityRange() == nil { + return errors.New("no capacity range provided") + } + + minSize := req.GetCapacityRange().GetRequiredBytes() + maxSize := req.GetCapacityRange().GetLimitBytes() + if 0 < maxSize && maxSize < minSize { + return errors.New("limitBytes is smaller than requiredBytes") + } + + if support, err := isSupportExpandVolume(ctx, req, backend); !support { + return err + } + + err := verifySectorSize(ctx, req.GetVolumeId(), backend, minSize) + if err != nil { + return err + } + + return nil +} + +func verifySectorSize(ctx context.Context, volumeId string, backend *model.Backend, minSize int64) error { + volumeAttrs, err := app.GetGlobalConfig().K8sUtils.GetVolumeAttrByVolumeId(volumeId) + if err != nil { + return status.Error(codes.InvalidArgument, fmt.Sprintf("failed to verify expand arguments: %v", err)) + } + + return utils.IsCapacityAvailable(minSize, backend.Plugin.GetSectorSize(), utils.CopyMap(volumeAttrs)) +} + func isSupportExpandVolume(ctx context.Context, req *csi.ControllerExpandVolumeRequest, b *model.Backend) ( bool, error) { if b.Storage == "fusionstorage-nas" || b.Storage == "oceanstor-nas" || b.Storage == "oceanstor-dtree" { @@ -258,10 +291,11 @@ func getAccessibleTopologies(ctx context.Context, req *csi.CreateVolumeRequest, func getAttributes(req *csi.CreateVolumeRequest, vol utils.Volume, backendName string) map[string]string { attributes := map[string]string{ - "backend": backendName, - "name": vol.GetVolumeName(), - "fsPermission": req.Parameters["fsPermission"], - "dTreeParentName": vol.GetDTreeParentName(), + "backend": backendName, + "name": vol.GetVolumeName(), + "fsPermission": req.Parameters["fsPermission"], + "dTreeParentName": vol.GetDTreeParentName(), + "disableVerifyCapacity": req.Parameters["disableVerifyCapacity"], } if lunWWN, err := vol.GetLunWWN(); err == nil { @@ -284,11 +318,10 @@ func getVolumeResponse(accessibleTopologies []*csi.Topology, func makeCreateVolumeResponse(ctx context.Context, req *csi.CreateVolumeRequest, vol utils.Volume, pool *model.StoragePool) *csi.Volume { contentSource := req.GetVolumeContentSource() - size := req.GetCapacityRange().GetRequiredBytes() accessibleTopologies := getAccessibleTopologies(ctx, req, pool) attributes := getAttributes(req, vol, pool.Parent) - csiVolume := getVolumeResponse(accessibleTopologies, attributes, pool.Parent+"."+vol.GetVolumeName(), size) + csiVolume := getVolumeResponse(accessibleTopologies, attributes, pool.Parent+"."+vol.GetVolumeName(), vol.GetSize()) if contentSource != nil { csiVolume.ContentSource = contentSource } @@ -432,8 +465,8 @@ func processCreateVolumeParameters(ctx context.Context, req *csi.CreateVolumeReq return parameters, nil } -func processCreateVolumeParametersAfterSelect(parameters map[string]interface{}, localPool *model.StoragePool, - remotePool *model.StoragePool) { +func processCreateVolumeParametersAfterSelect(parameters map[string]interface{}, + localPool *model.StoragePool, remotePool *model.StoragePool) error { parameters["storagepool"] = localPool.Name if remotePool != nil { @@ -443,6 +476,9 @@ func processCreateVolumeParametersAfterSelect(parameters map[string]interface{}, } parameters["accountName"] = backend.GetAccountName(localPool.Parent) + + size := utils.GetValueOrFallback(parameters, "size", int64(0)) + return utils.IsCapacityAvailable(size, localPool.Plugin.GetSectorSize(), parameters) } // createVolume used to create a lun/filesystem in huawei storage @@ -457,7 +493,11 @@ func (d *CsiDriver) createVolume(ctx context.Context, req *csi.CreateVolumeReque return nil, status.Error(codes.Internal, err.Error()) } - processCreateVolumeParametersAfterSelect(parameters, storagePoolPair.Local, storagePoolPair.Remote) + err = processCreateVolumeParametersAfterSelect(parameters, storagePoolPair.Local, storagePoolPair.Remote) + if err != nil { + log.AddContext(ctx).Errorln(err) + return nil, status.Error(codes.InvalidArgument, err.Error()) + } vol, err := storagePoolPair.Local.Plugin.CreateVolume(ctx, req.GetName(), parameters) if err != nil { @@ -465,6 +505,9 @@ func (d *CsiDriver) createVolume(ctx context.Context, req *csi.CreateVolumeReque return nil, status.Error(codes.Internal, err.Error()) } + recordCapacityChanged(ctx, req.GetCapacityRange().GetRequiredBytes(), + vol.GetSize(), storagePoolPair.Local.Plugin.GetSectorSize()) + log.AddContext(ctx).Infof("Volume %s is created", req.GetName()) res := &csi.CreateVolumeResponse{ Volume: makeCreateVolumeResponse(ctx, req, vol, storagePoolPair.Local), @@ -473,6 +516,15 @@ func (d *CsiDriver) createVolume(ctx context.Context, req *csi.CreateVolumeReque return res, nil } +func recordCapacityChanged(ctx context.Context, required, actual, sectorSize int64) { + if required < actual { + log.AddContext(ctx).Infof("Required capacity is %d, actual capacity is %d, "+ + "when this parameter is set to true,"+ + " if required capacity is not an integer multiple of the sector size %d,"+ + " it is automatically filled up.", required, actual, sectorSize) + } +} + // In the volume import scenario, only the fields in the annotation are obtained. // Other information are ignored (e.g. the capacity, backend, and QoS ...). func (d *CsiDriver) manageVolume(ctx context.Context, req *csi.CreateVolumeRequest, volumeName, backendName string) ( @@ -528,9 +580,9 @@ func (d *CsiDriver) manageVolume(ctx context.Context, req *csi.CreateVolumeReque } func validateCapacity(ctx context.Context, req *csi.CreateVolumeRequest, vol utils.Volume) error { - actualCapacity, err := vol.GetSize() - if err != nil { - return err + actualCapacity := vol.GetSize() + if actualCapacity == 0 { + return errors.New("empty Size") } if actualCapacity != req.GetCapacityRange().RequiredBytes { diff --git a/csi/driver/controller_helper_test.go b/csi/driver/controller_helper_test.go index ee1d9d1b..6b2e8f0a 100644 --- a/csi/driver/controller_helper_test.go +++ b/csi/driver/controller_helper_test.go @@ -22,20 +22,21 @@ import ( "reflect" "testing" - "github.com/stretchr/testify/require" - - "huawei-csi-driver/csi/backend/model" - "github.com/agiledragon/gomonkey/v2" "github.com/container-storage-interface/spec/lib/go/csi" "github.com/prashantv/gostub" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" - "huawei-csi-driver/csi/app" - cfg "huawei-csi-driver/csi/app/config" - "huawei-csi-driver/csi/backend/handler" - "huawei-csi-driver/csi/backend/plugin" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + cfg "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend/handler" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend/model" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend/plugin" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/k8sutils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( @@ -237,3 +238,67 @@ func Test_processAnnotations(t *testing.T) { "but got = %v", annotations, volume) } } + +func Test_VerifyExpandArguments_Success(t *testing.T) { + // arrange + req := &csi.ControllerExpandVolumeRequest{CapacityRange: &csi.CapacityRange{RequiredBytes: 1073741824}} + backend := &model.Backend{} + + // mock + patches := gomonkey.ApplyFuncReturn(isSupportExpandVolume, true, nil). + ApplyFuncReturn(verifySectorSize, nil) + defer patches.Reset() + + // action + err := verifyExpandArguments(context.Background(), req, backend) + + // assert + assert.NoError(t, err) +} + +func Test_VerifySectorSize_Success(t *testing.T) { + // arrange + req := &csi.ControllerExpandVolumeRequest{VolumeId: "backend.pvc_test_volume_id"} + backend := &model.Backend{ + Plugin: &plugin.OceanstorNasPlugin{}, + } + minSize := int64(1024 * 1024 * 1024) + app.GetGlobalConfig().K8sUtils = &k8sutils.KubeClient{} + + // mock + patches := gomonkey.ApplyMethodReturn(backend.Plugin, + "GetSectorSize", int64(constants.AllocationUnitBytes)). + ApplyMethodReturn(app.GetGlobalConfig().K8sUtils, "GetVolumeAttrByVolumeId", nil, nil). + ApplyFuncReturn(utils.GetValueOrFallback[string], "true") + defer patches.Reset() + + // action + err := verifySectorSize(context.Background(), req.GetVolumeId(), backend, minSize) + + // assert + assert.NoError(t, err) +} + +func Test_VerifySectorSize_CapacityNotMultiple(t *testing.T) { + // arrange + req := &csi.ControllerExpandVolumeRequest{VolumeId: "backend.pvc_test_volume_id"} + backend := &model.Backend{ + Plugin: &plugin.OceanstorNasPlugin{}, + } + minSize := int64(1024*1024*1024) + 511 + app.GetGlobalConfig().K8sUtils = &k8sutils.KubeClient{} + + // mock + patches := gomonkey.ApplyMethodReturn(backend.Plugin, + "GetSectorSize", int64(constants.AllocationUnitBytes)). + ApplyMethodReturn(app.GetGlobalConfig().K8sUtils, "GetVolumeAttrByVolumeId", nil, nil). + ApplyFuncReturn(utils.GetValueOrFallback[string], "true") + defer patches.Reset() + + // action + err := verifySectorSize(context.Background(), req.GetVolumeId(), backend, minSize) + + // assert + assert.Error(t, err) + assert.ErrorContains(t, err, "is not an integer or not multiple of") +} diff --git a/csi/driver/driver.go b/csi/driver/driver.go index 95d61143..ca9a0979 100644 --- a/csi/driver/driver.go +++ b/csi/driver/driver.go @@ -19,8 +19,8 @@ package driver import ( "strings" - "huawei-csi-driver/csi/backend/handler" - "huawei-csi-driver/utils/k8sutils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend/handler" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/k8sutils" ) // CsiDriver defines csi driver diff --git a/csi/driver/identity.go b/csi/driver/identity.go index 822d172b..b1e2e76a 100644 --- a/csi/driver/identity.go +++ b/csi/driver/identity.go @@ -21,7 +21,7 @@ import ( "github.com/container-storage-interface/spec/lib/go/csi" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // GetPluginInfo used to get plugin info diff --git a/csi/driver/node.go b/csi/driver/node.go index 7f3cfdcf..0b7742f0 100644 --- a/csi/driver/node.go +++ b/csi/driver/node.go @@ -20,6 +20,7 @@ import ( "context" "encoding/json" "fmt" + "os" "strings" "time" @@ -27,13 +28,14 @@ import ( "google.golang.org/grpc/codes" "google.golang.org/grpc/status" - "huawei-csi-driver/connector" - _ "huawei-csi-driver/connector/nfs" // init the nfs connector - "huawei-csi-driver/csi/app" - "huawei-csi-driver/csi/manage" - "huawei-csi-driver/pkg/constants" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector" + _ "github.com/Huawei/eSDK_K8S_Plugin/v4/connector/nfs" // init the nfs connector + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/manage" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/retry" ) const checkSymlinkPathTimeout = 10 * time.Second @@ -123,35 +125,32 @@ func (d *CsiDriver) NodeUnpublishVolume(ctx context.Context, log.AddContext(ctx).Infof("Start to node unpublish volume %s from %s", volumeId, targetPath) - if !strings.Contains(targetPath, app.GetGlobalConfig().KubeletVolumeDevicesDirName) { - log.AddContext(ctx).Infof("Unmounting the targetPath [%s]", targetPath) - mounted, err := connector.MountPathIsExist(ctx, targetPath) - if err != nil { - log.AddContext(ctx).Errorf("Failed to get mount point [%s], error: %v", targetPath, err) - return nil, status.Error(codes.Internal, err.Error()) - } - if mounted { - umountRes, err := utils.ExecShellCmd(ctx, "umount %s", targetPath) - if err != nil && !strings.Contains(umountRes, constants.NotMountStr) { - log.AddContext(ctx).Errorf("umount %s for volume %s msg:%s error: %s", targetPath, volumeId, - umountRes, err) - return nil, err - } - } - } else { - symLink, err := utils.IsPathSymlinkWithTimeout(targetPath, checkSymlinkPathTimeout) - if err != nil { - log.AddContext(ctx).Errorf("Failed to Access path %s, error: %v", targetPath, err) + mounted, err := connector.MountPathIsExist(ctx, targetPath) + if err != nil { + log.AddContext(ctx).Errorf("Failed to get mount point [%s], error: %v", targetPath, err) + return nil, status.Error(codes.Internal, err.Error()) + } + if mounted { + umountRes, err := utils.ExecShellCmd(ctx, "umount %s", targetPath) + if err != nil && !strings.Contains(umountRes, constants.NotMountStr) { + log.AddContext(ctx).Errorf("umount %s for volume %s msg:%s error: %s", targetPath, volumeId, + umountRes, err) return nil, status.Error(codes.Internal, err.Error()) } - if symLink { - log.AddContext(ctx).Infof("Removing the symlink [%s]", targetPath) - err := utils.RemoveSymlink(ctx, targetPath) - if err != nil { - log.AddContext(ctx).Errorf("Failed to remove symlink for target path [%v]", targetPath) - return nil, err - } + } + + log.AddContext(ctx).Infof("remove target path %s", targetPath) + const attempts = 3 + if err := retry.Attempts(attempts).Period(time.Second).Do(func() error { + err := os.Remove(targetPath) + if err != nil && !os.IsNotExist(err) { + return err } + + return nil + }); err != nil { + log.AddContext(ctx).Errorf("Failed to delete the target [%v]", targetPath) + return nil, status.Error(codes.Internal, err.Error()) } log.AddContext(ctx).Infof("Volume %s is node unpublished from %s", volumeId, targetPath) diff --git a/csi/garbage_collector.go b/csi/garbage_collector.go index 7af85658..bd3edbb0 100644 --- a/csi/garbage_collector.go +++ b/csi/garbage_collector.go @@ -24,12 +24,12 @@ import ( "path/filepath" "strings" - "huawei-csi-driver/connector/utils/lock" - "huawei-csi-driver/csi/app" - "huawei-csi-driver/csi/manage" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/k8sutils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector/utils/lock" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/manage" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/k8sutils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/csi/main.go b/csi/main.go index b9963dcf..35697fdf 100644 --- a/csi/main.go +++ b/csi/main.go @@ -18,6 +18,7 @@ package main import ( "context" + "fmt" "net" "os" "os/signal" @@ -29,19 +30,20 @@ import ( "github.com/sirupsen/logrus" "google.golang.org/grpc" - "huawei-csi-driver/connector/host" - connUtils "huawei-csi-driver/connector/utils" - "huawei-csi-driver/connector/utils/lock" - "huawei-csi-driver/csi/app" - "huawei-csi-driver/csi/backend/handler" - "huawei-csi-driver/csi/backend/job" - "huawei-csi-driver/csi/driver" - "huawei-csi-driver/csi/provider" - "huawei-csi-driver/lib/drcsi" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" - "huawei-csi-driver/utils/notify" - "huawei-csi-driver/utils/version" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector/host" + connUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/connector/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector/utils/lock" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend/handler" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend/job" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/driver" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/provider" + "github.com/Huawei/eSDK_K8S_Plugin/v4/lib/drcsi" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/cert" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/notify" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/version" ) const ( @@ -49,7 +51,7 @@ const ( controllerLogFile = "huawei-csi-controller" nodeLogFile = "huawei-csi-node" - csiVersion = "4.5.0" + csiVersion = "4.6.0" endpointDirPerm = 0755 ) @@ -87,7 +89,7 @@ func releaseStorageClient(ctx context.Context) { handler.NewCacheWrapper().Clear(ctx) } -func runCSIController(ctx context.Context) { +func runCSIController(ctx context.Context, csiDriver *driver.CsiDriver) { log.AddContext(ctx).Infoln("Run as huawei-csi-controller.") app.GetGlobalConfig().K8sUtils.Activate() @@ -101,11 +103,14 @@ func runCSIController(ctx context.Context) { // register the kahu community DRCSI service go registerDRCSIServer() + // expose csi controller server on k8s service + go runCsiControllerOnService(ctx, csiDriver) + // register the K8S community CSI service - registerCSIServer() + registerCSIServer(csiDriver) } -func runCSINode(ctx context.Context) { +func runCSINode(ctx context.Context, csiDriver *driver.CsiDriver) { go exitClean(false) // Init file lock @@ -133,7 +138,7 @@ func runCSINode(ctx context.Context) { }() // register the K8S community CSI service - registerCSIServer() + registerCSIServer(csiDriver) } func main() { @@ -155,11 +160,16 @@ func main() { logrus.Fatalf("Init log error: %v", err) } + csiDriver := driver.NewServer(app.GetGlobalConfig().DriverName, + csiVersion, + app.GetGlobalConfig().K8sUtils, + app.GetGlobalConfig().NodeName) + // Start CSI service if app.GetGlobalConfig().Controller { - runCSIController(context.Background()) + runCSIController(context.Background(), csiDriver) } else { - runCSINode(context.Background()) + runCSINode(context.Background(), csiDriver) } } @@ -179,13 +189,45 @@ func registerDRCSIServer() { } } -func registerCSIServer() { - d := driver.NewServer(app.GetGlobalConfig().DriverName, - csiVersion, - app.GetGlobalConfig().K8sUtils, - app.GetGlobalConfig().NodeName) +func runCsiControllerOnService(ctx context.Context, csiDriver *driver.CsiDriver) { + if app.GetGlobalConfig().ExportCsiServerAddress == "" { + return + } + + address := fmt.Sprintf("%s:%d", + app.GetGlobalConfig().ExportCsiServerAddress, + app.GetGlobalConfig().ExportCsiServerPort) + listen, err := net.Listen("tcp", address) + if err != nil { + notify.Stop("listen on %s error: %v", address, err) + } + registerServerOnService(ctx, listen, csiDriver) +} + +func registerServerOnService(ctx context.Context, listener net.Listener, d *driver.CsiDriver) { + cred, err := cert.GetGrpcCredential(ctx) + if err != nil { + notify.Stop("start Huawei CSI driver on service error: %v", err) + } + opts := []grpc.ServerOption{ + grpc.UnaryInterceptor(log.EnsureGRPCContext), + grpc.Creds(cred), + } + server := grpc.NewServer(opts...) + + csi.RegisterIdentityServer(server, d) + csi.RegisterControllerServer(server, d) + csi.RegisterNodeServer(server, d) + + log.Infof("starting Huawei CSI driver on service, listening on %s", listener.Addr().String()) + if err := server.Serve(listener); err != nil { + notify.Stop("start Huawei CSI driver on service error: %v", err) + } +} + +func registerCSIServer(csiDriver *driver.CsiDriver) { listener := listenEndpoint(app.GetGlobalConfig().Endpoint) - registerServer(listener, d) + registerServer(listener, csiDriver) } func listenEndpoint(endpoint string) net.Listener { diff --git a/csi/manage/manager_helper.go b/csi/manage/manager_helper.go index 7017db81..b6e4065b 100644 --- a/csi/manage/manager_helper.go +++ b/csi/manage/manager_helper.go @@ -27,16 +27,17 @@ import ( "github.com/container-storage-interface/spec/lib/go/csi" - "huawei-csi-driver/connector" - _ "huawei-csi-driver/connector/nfsplus" - "huawei-csi-driver/csi/app" - "huawei-csi-driver/csi/backend" - "huawei-csi-driver/csi/backend/plugin" - "huawei-csi-driver/pkg/constants" - pkgUtils "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/storage/oceanstor/client" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector" + _ "github.com/Huawei/eSDK_K8S_Plugin/v4/connector/nfsplus" + connUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/connector/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend/plugin" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceanstor/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // BuildParameterOption define build function @@ -334,22 +335,16 @@ func PublishBlock(ctx context.Context, req *csi.NodePublishVolumeRequest) error targetPath := req.GetTargetPath() // If the request is to publish raw block device then create symlink of the device // from the staging are to publish. Do not create fs and mount - log.AddContext(ctx).Infoln("Creating symlink for the staged device on the node to publish") + log.AddContext(ctx).Infoln("Bind mount for the staged device on the node to publish") sourcePath = sourcePath + "/" + volumeId - err := utils.CreateSymlink(ctx, sourcePath, targetPath) + err := connUtils.BindMountRawBlockDevice(ctx, sourcePath, targetPath, + req.GetVolumeCapability().GetMount().GetMountFlags()) if err != nil { - log.AddContext(ctx).Errorf("Failed to create symlink for the staging path [%v] to target path [%v]", + log.AddContext(ctx).Errorf("Failed to bind mount for the staging path [%v] to target path [%v]", sourcePath, targetPath) return err } - accessMode := utils.GetAccessModeType(req.GetVolumeCapability().GetAccessMode().GetMode()) - if accessMode == "ReadOnly" { - _, err = utils.ExecShellCmd(ctx, "chmod 440 %s", targetPath) - if err != nil { - log.AddContext(ctx).Errorln("Unable to set ReadOnlyMany permission") - return err - } - } + log.AddContext(ctx).Infof("Raw Block Volume %s is node published to %s", volumeId, targetPath) return nil } diff --git a/csi/manage/manager_helper_test.go b/csi/manage/manager_helper_test.go index 4cceb56d..47941510 100644 --- a/csi/manage/manager_helper_test.go +++ b/csi/manage/manager_helper_test.go @@ -28,11 +28,11 @@ import ( "github.com/container-storage-interface/spec/lib/go/csi" "github.com/prashantv/gostub" - "huawei-csi-driver/connector" - "huawei-csi-driver/connector/nvme" - "huawei-csi-driver/csi/app" - cfg "huawei-csi-driver/csi/app/config" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector/nvme" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + cfg "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/csi/manage/nas_manager.go b/csi/manage/nas_manager.go index 49426008..22c5317e 100644 --- a/csi/manage/nas_manager.go +++ b/csi/manage/nas_manager.go @@ -21,11 +21,11 @@ import ( "github.com/container-storage-interface/spec/lib/go/csi" - "huawei-csi-driver/connector" - "huawei-csi-driver/csi/backend/plugin" - pkgUtils "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend/plugin" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // NasManager implements VolumeManager interface diff --git a/csi/manage/nas_manager_test.go b/csi/manage/nas_manager_test.go index 955d4967..5794c5a5 100644 --- a/csi/manage/nas_manager_test.go +++ b/csi/manage/nas_manager_test.go @@ -25,7 +25,7 @@ import ( "github.com/agiledragon/gomonkey/v2" "github.com/container-storage-interface/spec/lib/go/csi" - "huawei-csi-driver/connector" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector" ) func mockNasStageVolumeRequest() *csi.NodeStageVolumeRequest { diff --git a/csi/manage/san_manager.go b/csi/manage/san_manager.go index 45cdfb7b..d20c76a6 100644 --- a/csi/manage/san_manager.go +++ b/csi/manage/san_manager.go @@ -19,13 +19,15 @@ package manage import ( "context" "errors" + "path" "github.com/container-storage-interface/spec/lib/go/csi" - "huawei-csi-driver/connector" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/flow" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector" + connUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/connector/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/flow" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // SanManager implements VolumeManager interface @@ -101,7 +103,15 @@ func (m *SanManager) UnStageVolume(ctx context.Context, req *csi.NodeUnstageVolu return nil } - if err = Unmount(ctx, targetPath); err != nil { + rawBlockStagePath := path.Join(targetPath, volumeId) + rawBlock, err := connector.MountPathIsExist(ctx, rawBlockStagePath) + + if rawBlock { + err = connUtils.Unmount(ctx, rawBlockStagePath) + } else { + err = Unmount(ctx, targetPath) + } + if err != nil { log.AddContext(ctx).Errorf("umount target path failed while unstage volume, error: %v", err) return err } @@ -281,7 +291,7 @@ func stageForBlock(ctx context.Context, parameters map[string]interface{}) error return errors.New("device path doesn't exist while stage for block") } - err := utils.CreateSymlink(ctx, devPath, mountPoint) + err := connUtils.BindMountRawBlockDevice(ctx, devPath, mountPoint, nil) if err != nil { log.AddContext(ctx).Errorln("create system link failed, error: %v", err) return err diff --git a/csi/manage/san_manager_test.go b/csi/manage/san_manager_test.go index a2aa7232..d8e82859 100644 --- a/csi/manage/san_manager_test.go +++ b/csi/manage/san_manager_test.go @@ -27,14 +27,15 @@ import ( "github.com/agiledragon/gomonkey/v2" "github.com/container-storage-interface/spec/lib/go/csi" - "huawei-csi-driver/connector" - "huawei-csi-driver/connector/fibrechannel" - "huawei-csi-driver/connector/iscsi" - "huawei-csi-driver/connector/nvme" - "huawei-csi-driver/connector/roce" - "huawei-csi-driver/csi/app" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector/fibrechannel" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector/iscsi" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector/nvme" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector/roce" + connUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/connector/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) func TestSanManagerStageFileSystemVolume(t *testing.T) { @@ -107,7 +108,7 @@ func TestSanManagerStageBlockVolume(t *testing.T) { patches := gomonkey.NewPatches() mockClearResidualPath(patches, manager.protocol) mockConnectIscsiVolume(patches, manager.Conn) - mockCreateSymlink(patches) + mockBindMount(patches) request := mockSanStageVolumeRequest(t, "Block") err := manager.StageVolume(context.Background(), request) @@ -249,8 +250,8 @@ func mockChmodFsPermission(patch *gomonkey.Patches, t *testing.T) { }) } -func mockCreateSymlink(patch *gomonkey.Patches) { - patch.ApplyFunc(utils.CreateSymlink, func(ctx context.Context, source string, target string) error { +func mockBindMount(patch *gomonkey.Patches) { + patch.ApplyFunc(connUtils.BindMountRawBlockDevice, func(ctx context.Context, source string, target string) error { if source == "test_dev_path" && target == "/test_staging_target_path/test_backend.pvc-san-xxx" { return nil } diff --git a/csi/manage/types.go b/csi/manage/types.go index 866c018a..6ecd77cb 100644 --- a/csi/manage/types.go +++ b/csi/manage/types.go @@ -21,7 +21,7 @@ import ( "github.com/container-storage-interface/spec/lib/go/csi" - "huawei-csi-driver/connector/nvme" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector/nvme" ) // VolumeManager defines the operations which storage manager should implement diff --git a/csi/provider/backend.go b/csi/provider/backend.go index ef7fcd85..2868643c 100644 --- a/csi/provider/backend.go +++ b/csi/provider/backend.go @@ -22,12 +22,12 @@ import ( "errors" "fmt" - "huawei-csi-driver/client/apis/xuanwu/v1" - "huawei-csi-driver/csi/app" - "huawei-csi-driver/lib/drcsi" - "huawei-csi-driver/pkg/constants" - pkgUtils "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + "github.com/Huawei/eSDK_K8S_Plugin/v4/lib/drcsi" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // AddStorageBackend used to add storage backend, and return the backend ID diff --git a/csi/provider/identity.go b/csi/provider/identity.go index 8bda1d86..b3aedb3a 100644 --- a/csi/provider/identity.go +++ b/csi/provider/identity.go @@ -20,8 +20,8 @@ package provider import ( "context" - "huawei-csi-driver/lib/drcsi" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/lib/drcsi" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // GetProviderInfo is used to get provider info diff --git a/csi/provider/provider.go b/csi/provider/provider.go index c0c1a1e9..6f677ded 100644 --- a/csi/provider/provider.go +++ b/csi/provider/provider.go @@ -17,7 +17,7 @@ // Package provider is related with storage provider package provider -import "huawei-csi-driver/csi/backend/handler" +import "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend/handler" // StorageProvider is for storage provider type StorageProvider struct { diff --git a/csi/provider/volume.go b/csi/provider/volume.go index a7175e6c..0b91e215 100644 --- a/csi/provider/volume.go +++ b/csi/provider/volume.go @@ -22,11 +22,11 @@ import ( "errors" "fmt" - "huawei-csi-driver/lib/drcsi" - pkgUtils "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/pkg/volume" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/lib/drcsi" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/volume" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/csi/provider/volume_test.go b/csi/provider/volume_test.go index a78496e7..480e64d0 100644 --- a/csi/provider/volume_test.go +++ b/csi/provider/volume_test.go @@ -26,13 +26,13 @@ import ( "github.com/agiledragon/gomonkey/v2" "github.com/stretchr/testify/require" - "huawei-csi-driver/csi/backend/handler" - "huawei-csi-driver/csi/backend/model" - "huawei-csi-driver/csi/backend/plugin" - "huawei-csi-driver/lib/drcsi" - "huawei-csi-driver/storage/oceanstor/volume" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend/handler" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend/model" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend/plugin" + "github.com/Huawei/eSDK_K8S_Plugin/v4/lib/drcsi" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceanstor/volume" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/go.mod b/go.mod index 4394b63e..7f96805d 100644 --- a/go.mod +++ b/go.mod @@ -1,4 +1,4 @@ -module huawei-csi-driver +module github.com/Huawei/eSDK_K8S_Plugin/v4 go 1.22 diff --git a/helm/esdk/Chart.yaml b/helm/esdk/Chart.yaml index 1139424c..89fdcf96 100644 --- a/helm/esdk/Chart.yaml +++ b/helm/esdk/Chart.yaml @@ -15,14 +15,14 @@ type: application # This is the chart version. This version number should be incremented each time you make changes # to the chart and its templates, including the app version. # Versions are expected to follow Semantic Versioning (https://semver.org/) -version: 4.5.0 +version: 4.6.0 # This is the version number of the application being deployed. This version number should be # incremented each time you make changes to the application. Versions are not expected to # follow Semantic Versioning. They should reflect the version the application is using. # It is recommended to use it with quotes. # It is strongly recommended not to modify this parameter -appVersion: "4.5.0" +appVersion: "4.6.0" home: https://github.com/Huawei/eSDK_K8S_Plugin sources: diff --git a/helm/esdk/templates/huawei-csi-controller.yaml b/helm/esdk/templates/huawei-csi-controller.yaml index 678dbe04..051d8a15 100644 --- a/helm/esdk/templates/huawei-csi-controller.yaml +++ b/helm/esdk/templates/huawei-csi-controller.yaml @@ -765,6 +765,10 @@ spec: - "--log-file-size={{ .Values.csiDriver.controllerLogging.fileSize }}" - "--max-backups={{ .Values.csiDriver.controllerLogging.maxBackups }}" {{ end }} + {{ if .Values.controller.exportCsiService.enabled }} + - "--export-csi-service-address=$(POD_IP)" + - "--export-csi-service-port={{ .Values.controller.exportCsiService.port }}" + {{ end }} env: - name: CSI_ENDPOINT value: {{ .Values.csiDriver.endpoint }} @@ -775,6 +779,10 @@ spec: fieldRef: apiVersion: v1 fieldPath: metadata.namespace + - name: POD_IP + valueFrom: + fieldRef: + fieldPath: status.podIP livenessProbe: failureThreshold: 5 httpGet: @@ -829,4 +837,10 @@ spec: - name: storage-backend-controller protocol: TCP port: {{ int .Values.controller.webhookPort | default 4433 }} - targetPort: {{ int .Values.controller.webhookPort | default 4433 }} \ No newline at end of file + targetPort: {{ int .Values.controller.webhookPort | default 4433 }} + {{ if .Values.controller.exportCsiService.enabled }} + - name: huawei-csi-driver-csi-grpc + protocol: TCP + port: {{ int .Values.controller.exportCsiService.port | default 9090 }} + targetPort: {{ int .Values.controller.exportCsiService.port | default 9090 }} + {{ end }} \ No newline at end of file diff --git a/helm/esdk/values.yaml b/helm/esdk/values.yaml index a9419130..b3d7a58b 100644 --- a/helm/esdk/values.yaml +++ b/helm/esdk/values.yaml @@ -1,20 +1,20 @@ images: # Images provided by Huawei - huaweiCSIService: huawei-csi:4.5.0 - storageBackendSidecar: storage-backend-sidecar:4.5.0 - storageBackendController: storage-backend-controller:4.5.0 - huaweiCSIExtender: huawei-csi-extender:4.5.0 + huaweiCSIService: huawei-csi:4.6.0 + storageBackendSidecar: storage-backend-sidecar:4.6.0 + storageBackendController: storage-backend-controller:4.6.0 + huaweiCSIExtender: huawei-csi-extender:4.6.0 # CSI-related sidecar images provided by the Kubernetes community. # These must match the appropriate Kubernetes version. sidecar: - attacher: k8s.gcr.io/sig-storage/csi-attacher:v3.4.0 - provisioner: k8s.gcr.io/sig-storage/csi-provisioner:v3.0.0 - resizer: k8s.gcr.io/sig-storage/csi-resizer:v1.4.0 - registrar: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.3.0 - livenessProbe: k8s.gcr.io/sig-storage/livenessprobe:v2.5.0 - snapshotter: k8s.gcr.io/sig-storage/csi-snapshotter:v4.2.1 - snapshotController: k8s.gcr.io/sig-storage/snapshot-controller:v4.2.1 + attacher: registry.k8s.io/sig-storage/csi-attacher:v4.4.0 + provisioner: registry.k8s.io/sig-storage/csi-provisioner:v3.6.0 + resizer: registry.k8s.io/sig-storage/csi-resizer:v1.9.0 + registrar: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.9.0 + livenessProbe: registry.k8s.io/sig-storage/livenessprobe:v2.12.0 + snapshotter: registry.k8s.io/sig-storage/csi-snapshotter:v6.3.0 + snapshotController: registry.k8s.io/sig-storage/snapshot-controller:v6.3.0 # Default image pull policy for sidecar container images, support [IfNotPresent, Always, Never] sidecarImagePullPolicy: "IfNotPresent" @@ -115,6 +115,17 @@ controller: # Duration, volume modify resource reconcile delay time reconcileDelay: 1s + exportCsiService: + # enabled: Enable/Disable running the CSI exported server on service, so that other pod can call CSI + # Allowed values: + # true: enable running the exported CSI server + # false: disable running the exported CSI server on service + # Default value: false + enabled: false + # Port used by the CSI server when it is exported on service. The default port is 9090. + # You can change the port to another port that is not occupied. + port: 9090 + # nodeSelector: Define node selection constraints for controller pods. # For the pod to be eligible to run on a node, the node must have each # of the indicated key-value pairs as labels. diff --git a/lib/drcsi/rpc/common.go b/lib/drcsi/rpc/common.go index bc3d1bc9..8354209a 100644 --- a/lib/drcsi/rpc/common.go +++ b/lib/drcsi/rpc/common.go @@ -25,10 +25,10 @@ import ( "github.com/kubernetes-csi/csi-lib-utils/metrics" "google.golang.org/grpc" - "huawei-csi-driver/csi/app" - "huawei-csi-driver/lib/drcsi" - "huawei-csi-driver/lib/drcsi/connection" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + "github.com/Huawei/eSDK_K8S_Plugin/v4/lib/drcsi" + "github.com/Huawei/eSDK_K8S_Plugin/v4/lib/drcsi/connection" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // ConnectProvider connect provider diff --git a/manual/esdk/deploy/huawei-csi-controller-extender.yaml b/manual/esdk/deploy/huawei-csi-controller-extender.yaml index 2a9c48f0..b07a3e62 100644 --- a/manual/esdk/deploy/huawei-csi-controller-extender.yaml +++ b/manual/esdk/deploy/huawei-csi-controller-extender.yaml @@ -461,7 +461,7 @@ spec: args: - "--csi-address=/csi/csi.sock" - "--health-port=9808" - image: k8s.gcr.io/sig-storage/livenessprobe:v2.5.0 + image: registry.k8s.io/sig-storage/livenessprobe:v2.12.0 imagePullPolicy: "IfNotPresent" volumeMounts: - mountPath: /csi @@ -482,7 +482,7 @@ spec: env: - name: ADDRESS value: /csi/csi.sock - image: k8s.gcr.io/sig-storage/csi-provisioner:v3.0.0 + image: registry.k8s.io/sig-storage/csi-provisioner:v3.6.0 imagePullPolicy: "IfNotPresent" volumeMounts: - mountPath: /csi @@ -501,7 +501,7 @@ spec: env: - name: ADDRESS value: /csi/csi.sock - image: k8s.gcr.io/sig-storage/csi-attacher:v3.4.0 + image: registry.k8s.io/sig-storage/csi-attacher:v4.4.0 imagePullPolicy: "IfNotPresent" volumeMounts: - mountPath: /csi @@ -522,7 +522,7 @@ spec: env: - name: ADDRESS value: /csi/csi.sock - image: k8s.gcr.io/sig-storage/csi-resizer:v1.4.0 + image: registry.k8s.io/sig-storage/csi-resizer:v1.9.0 imagePullPolicy: IfNotPresent volumeMounts: - mountPath: /csi @@ -542,7 +542,7 @@ spec: env: - name: ADDRESS value: /csi/csi.sock - image: k8s.gcr.io/sig-storage/csi-snapshotter:v4.2.1 + image: registry.k8s.io/sig-storage/csi-snapshotter:v6.3.0 imagePullPolicy: "IfNotPresent" volumeMounts: - mountPath: /csi @@ -558,7 +558,7 @@ spec: args: - "--v=5" - "--leader-election" - image: k8s.gcr.io/sig-storage/snapshot-controller:v4.2.1 + image: registry.k8s.io/sig-storage/snapshot-controller:v6.3.0 imagePullPolicy: "IfNotPresent" resources: limits: @@ -568,7 +568,7 @@ spec: cpu: 50m memory: 128Mi - name: storage-backend-controller - image: storage-backend-controller:4.5.0 + image: storage-backend-controller:4.6.0 imagePullPolicy: "IfNotPresent" env: - name: CSI_NAMESPACE @@ -607,7 +607,7 @@ spec: cpu: 50m memory: 128Mi - name: storage-backend-sidecar - image: storage-backend-sidecar:4.5.0 + image: storage-backend-sidecar:4.6.0 imagePullPolicy: "IfNotPresent" env: - name: DRCSI_ENDPOINT @@ -644,7 +644,7 @@ spec: cpu: 50m memory: 128Mi - name: huawei-csi-extender - image: huawei-csi-extender:4.5.0 + image: huawei-csi-extender:4.6.0 imagePullPolicy: "IfNotPresent" env: - name: DRCSI_ENDPOINT @@ -683,7 +683,7 @@ spec: cpu: 50m memory: 128Mi - name: huawei-csi-driver - image: huawei-csi:4.5.0 + image: huawei-csi:4.6.0 imagePullPolicy: "IfNotPresent" args: - "--endpoint=$(CSI_ENDPOINT)" diff --git a/manual/esdk/deploy/huawei-csi-controller.yaml b/manual/esdk/deploy/huawei-csi-controller.yaml index cd27f4b0..3d7b6e39 100644 --- a/manual/esdk/deploy/huawei-csi-controller.yaml +++ b/manual/esdk/deploy/huawei-csi-controller.yaml @@ -429,7 +429,7 @@ spec: args: - "--csi-address=/csi/csi.sock" - "--health-port=9808" - image: k8s.gcr.io/sig-storage/livenessprobe:v2.5.0 + image: registry.k8s.io/sig-storage/livenessprobe:v2.12.0 imagePullPolicy: "IfNotPresent" volumeMounts: - mountPath: /csi @@ -450,7 +450,7 @@ spec: env: - name: ADDRESS value: /csi/csi.sock - image: k8s.gcr.io/sig-storage/csi-provisioner:v3.0.0 + image: registry.k8s.io/sig-storage/csi-provisioner:v3.6.0 imagePullPolicy: "IfNotPresent" volumeMounts: - mountPath: /csi @@ -469,7 +469,7 @@ spec: env: - name: ADDRESS value: /csi/csi.sock - image: k8s.gcr.io/sig-storage/csi-attacher:v3.4.0 + image: registry.k8s.io/sig-storage/csi-attacher:v4.4.0 imagePullPolicy: "IfNotPresent" volumeMounts: - mountPath: /csi @@ -490,7 +490,7 @@ spec: env: - name: ADDRESS value: /csi/csi.sock - image: k8s.gcr.io/sig-storage/csi-resizer:v1.4.0 + image: registry.k8s.io/sig-storage/csi-resizer:v1.9.0 imagePullPolicy: IfNotPresent volumeMounts: - mountPath: /csi @@ -510,7 +510,7 @@ spec: env: - name: ADDRESS value: /csi/csi.sock - image: k8s.gcr.io/sig-storage/csi-snapshotter:v4.2.1 + image: registry.k8s.io/sig-storage/csi-snapshotter:v6.3.0 imagePullPolicy: "IfNotPresent" volumeMounts: - mountPath: /csi @@ -526,7 +526,7 @@ spec: args: - "--v=5" - "--leader-election" - image: k8s.gcr.io/sig-storage/snapshot-controller:v4.2.1 + image: registry.k8s.io/sig-storage/snapshot-controller:v6.3.0 imagePullPolicy: "IfNotPresent" resources: limits: @@ -536,7 +536,7 @@ spec: cpu: 50m memory: 128Mi - name: storage-backend-controller - image: storage-backend-controller:4.5.0 + image: storage-backend-controller:4.6.0 imagePullPolicy: "IfNotPresent" env: - name: CSI_NAMESPACE @@ -575,7 +575,7 @@ spec: cpu: 50m memory: 128Mi - name: storage-backend-sidecar - image: storage-backend-sidecar:4.5.0 + image: storage-backend-sidecar:4.6.0 imagePullPolicy: "IfNotPresent" env: - name: DRCSI_ENDPOINT @@ -612,7 +612,7 @@ spec: cpu: 50m memory: 128Mi - name: huawei-csi-driver - image: huawei-csi:4.5.0 + image: huawei-csi:4.6.0 imagePullPolicy: "IfNotPresent" args: - "--endpoint=$(CSI_ENDPOINT)" diff --git a/manual/esdk/deploy/huawei-csi-node.yaml b/manual/esdk/deploy/huawei-csi-node.yaml index aaecb565..a99486d7 100644 --- a/manual/esdk/deploy/huawei-csi-node.yaml +++ b/manual/esdk/deploy/huawei-csi-node.yaml @@ -135,7 +135,7 @@ spec: args: - "--csi-address=/csi/csi.sock" - "--health-port=9800" - image: k8s.gcr.io/sig-storage/livenessprobe:v2.5.0 + image: registry.k8s.io/sig-storage/livenessprobe:v2.12.0 imagePullPolicy: "IfNotPresent" volumeMounts: - mountPath: /csi @@ -151,7 +151,7 @@ spec: args: - "--csi-address=/csi/csi.sock" - "--kubelet-registration-path=/var/lib/kubelet/plugins/csi.huawei.com/csi.sock" - image: k8s.gcr.io/sig-storage/csi-node-driver-registrar:v2.3.0 + image: registry.k8s.io/sig-storage/csi-node-driver-registrar:v2.9.0 imagePullPolicy: "IfNotPresent" volumeMounts: - mountPath: /csi @@ -166,7 +166,7 @@ spec: cpu: 50m memory: 128Mi - name: huawei-csi-driver - image: huawei-csi:4.5.0 + image: huawei-csi:4.6.0 imagePullPolicy: "IfNotPresent" args: - "--endpoint=/csi/csi.sock" diff --git a/pkg/client/clientset/versioned/clientset.go b/pkg/client/clientset/versioned/clientset.go index 19fb0aac..b536f19d 100644 --- a/pkg/client/clientset/versioned/clientset.go +++ b/pkg/client/clientset/versioned/clientset.go @@ -24,7 +24,7 @@ import ( "k8s.io/client-go/tools/clientcmd" flowcontrol "k8s.io/client-go/util/flowcontrol" - xuanwuv1 "huawei-csi-driver/pkg/client/clientset/versioned/typed/xuanwu/v1" + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/clientset/versioned/typed/xuanwu/v1" ) type Interface interface { diff --git a/pkg/client/clientset/versioned/fake/clientset_generated.go b/pkg/client/clientset/versioned/fake/clientset_generated.go index 97b869ba..ede0b098 100644 --- a/pkg/client/clientset/versioned/fake/clientset_generated.go +++ b/pkg/client/clientset/versioned/fake/clientset_generated.go @@ -16,15 +16,15 @@ package fake import ( - clientset "huawei-csi-driver/pkg/client/clientset/versioned" - xuanwuv1 "huawei-csi-driver/pkg/client/clientset/versioned/typed/xuanwu/v1" - fakexuanwuv1 "huawei-csi-driver/pkg/client/clientset/versioned/typed/xuanwu/v1/fake" - "k8s.io/apimachinery/pkg/runtime" "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/discovery" fakediscovery "k8s.io/client-go/discovery/fake" "k8s.io/client-go/testing" + + clientset "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/clientset/versioned" + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/clientset/versioned/typed/xuanwu/v1" + fakexuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/clientset/versioned/typed/xuanwu/v1/fake" ) // NewSimpleClientset returns a clientset that will respond with the provided objects. diff --git a/pkg/client/clientset/versioned/fake/register.go b/pkg/client/clientset/versioned/fake/register.go index 8ec64565..f23b6900 100644 --- a/pkg/client/clientset/versioned/fake/register.go +++ b/pkg/client/clientset/versioned/fake/register.go @@ -16,13 +16,13 @@ package fake import ( - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" serializer "k8s.io/apimachinery/pkg/runtime/serializer" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" ) var scheme = runtime.NewScheme() @@ -35,14 +35,14 @@ var localSchemeBuilder = runtime.SchemeBuilder{ // AddToScheme adds all types of this clientset into the given scheme. This allows composition // of clientsets, like in: // -// import ( -// "k8s.io/client-go/kubernetes" -// clientsetscheme "k8s.io/client-go/kubernetes/scheme" -// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" -// ) +// import ( +// "k8s.io/client-go/kubernetes" +// clientsetscheme "k8s.io/client-go/kubernetes/scheme" +// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" +// ) // -// kclientset, _ := kubernetes.NewForConfig(c) -// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) +// kclientset, _ := kubernetes.NewForConfig(c) +// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) // // After this, RawExtensions in Kubernetes types will serialize kube-aggregator types // correctly. diff --git a/pkg/client/clientset/versioned/scheme/register.go b/pkg/client/clientset/versioned/scheme/register.go index 97cab162..191571cd 100644 --- a/pkg/client/clientset/versioned/scheme/register.go +++ b/pkg/client/clientset/versioned/scheme/register.go @@ -16,13 +16,13 @@ package scheme import ( - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" - v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" serializer "k8s.io/apimachinery/pkg/runtime/serializer" utilruntime "k8s.io/apimachinery/pkg/util/runtime" + + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" ) var Scheme = runtime.NewScheme() @@ -35,14 +35,14 @@ var localSchemeBuilder = runtime.SchemeBuilder{ // AddToScheme adds all types of this clientset into the given scheme. This allows composition // of clientsets, like in: // -// import ( -// "k8s.io/client-go/kubernetes" -// clientsetscheme "k8s.io/client-go/kubernetes/scheme" -// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" -// ) +// import ( +// "k8s.io/client-go/kubernetes" +// clientsetscheme "k8s.io/client-go/kubernetes/scheme" +// aggregatorclientsetscheme "k8s.io/kube-aggregator/pkg/client/clientset_generated/clientset/scheme" +// ) // -// kclientset, _ := kubernetes.NewForConfig(c) -// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) +// kclientset, _ := kubernetes.NewForConfig(c) +// _ = aggregatorclientsetscheme.AddToScheme(clientsetscheme.Scheme) // // After this, RawExtensions in Kubernetes types will serialize kube-aggregator types // correctly. diff --git a/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_storagebackendclaim.go b/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_storagebackendclaim.go index c48b0b8b..d39c04be 100644 --- a/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_storagebackendclaim.go +++ b/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_storagebackendclaim.go @@ -17,7 +17,6 @@ package fake import ( "context" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" labels "k8s.io/apimachinery/pkg/labels" @@ -25,6 +24,8 @@ import ( types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" testing "k8s.io/client-go/testing" + + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" ) // FakeStorageBackendClaims implements StorageBackendClaimInterface diff --git a/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_storagebackendcontent.go b/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_storagebackendcontent.go index e8f253c4..993dfa2a 100644 --- a/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_storagebackendcontent.go +++ b/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_storagebackendcontent.go @@ -17,7 +17,6 @@ package fake import ( "context" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" labels "k8s.io/apimachinery/pkg/labels" @@ -25,6 +24,8 @@ import ( types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" testing "k8s.io/client-go/testing" + + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" ) // FakeStorageBackendContents implements StorageBackendContentInterface diff --git a/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_volumemodifyclaim.go b/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_volumemodifyclaim.go index 6e0cf1e1..d720c042 100644 --- a/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_volumemodifyclaim.go +++ b/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_volumemodifyclaim.go @@ -17,7 +17,6 @@ package fake import ( "context" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" labels "k8s.io/apimachinery/pkg/labels" @@ -25,6 +24,8 @@ import ( types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" testing "k8s.io/client-go/testing" + + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" ) // FakeVolumeModifyClaims implements VolumeModifyClaimInterface diff --git a/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_volumemodifycontent.go b/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_volumemodifycontent.go index dd2a1fa9..9102871e 100644 --- a/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_volumemodifycontent.go +++ b/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_volumemodifycontent.go @@ -17,7 +17,6 @@ package fake import ( "context" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" labels "k8s.io/apimachinery/pkg/labels" @@ -25,6 +24,8 @@ import ( types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" testing "k8s.io/client-go/testing" + + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" ) // FakeVolumeModifyContents implements VolumeModifyContentInterface diff --git a/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_xuanwu_client.go b/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_xuanwu_client.go index 4084aafc..a2a4fc72 100644 --- a/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_xuanwu_client.go +++ b/pkg/client/clientset/versioned/typed/xuanwu/v1/fake/fake_xuanwu_client.go @@ -16,10 +16,10 @@ package fake import ( - v1 "huawei-csi-driver/pkg/client/clientset/versioned/typed/xuanwu/v1" - rest "k8s.io/client-go/rest" testing "k8s.io/client-go/testing" + + v1 "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/clientset/versioned/typed/xuanwu/v1" ) type FakeXuanwuV1 struct { diff --git a/pkg/client/clientset/versioned/typed/xuanwu/v1/storagebackendclaim.go b/pkg/client/clientset/versioned/typed/xuanwu/v1/storagebackendclaim.go index c87eacf8..a9acd667 100644 --- a/pkg/client/clientset/versioned/typed/xuanwu/v1/storagebackendclaim.go +++ b/pkg/client/clientset/versioned/typed/xuanwu/v1/storagebackendclaim.go @@ -17,14 +17,15 @@ package v1 import ( "context" - v1 "huawei-csi-driver/client/apis/xuanwu/v1" - scheme "huawei-csi-driver/pkg/client/clientset/versioned/scheme" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" rest "k8s.io/client-go/rest" + + v1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + scheme "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/clientset/versioned/scheme" ) // StorageBackendClaimsGetter has a method to return a StorageBackendClaimInterface. diff --git a/pkg/client/clientset/versioned/typed/xuanwu/v1/storagebackendcontent.go b/pkg/client/clientset/versioned/typed/xuanwu/v1/storagebackendcontent.go index a1fcf13c..1b19eb34 100644 --- a/pkg/client/clientset/versioned/typed/xuanwu/v1/storagebackendcontent.go +++ b/pkg/client/clientset/versioned/typed/xuanwu/v1/storagebackendcontent.go @@ -17,14 +17,15 @@ package v1 import ( "context" - v1 "huawei-csi-driver/client/apis/xuanwu/v1" - scheme "huawei-csi-driver/pkg/client/clientset/versioned/scheme" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" rest "k8s.io/client-go/rest" + + v1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + scheme "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/clientset/versioned/scheme" ) // StorageBackendContentsGetter has a method to return a StorageBackendContentInterface. diff --git a/pkg/client/clientset/versioned/typed/xuanwu/v1/volumemodifyclaim.go b/pkg/client/clientset/versioned/typed/xuanwu/v1/volumemodifyclaim.go index 55b3cacd..3246a4c0 100644 --- a/pkg/client/clientset/versioned/typed/xuanwu/v1/volumemodifyclaim.go +++ b/pkg/client/clientset/versioned/typed/xuanwu/v1/volumemodifyclaim.go @@ -17,14 +17,15 @@ package v1 import ( "context" - v1 "huawei-csi-driver/client/apis/xuanwu/v1" - scheme "huawei-csi-driver/pkg/client/clientset/versioned/scheme" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" rest "k8s.io/client-go/rest" + + v1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + scheme "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/clientset/versioned/scheme" ) // VolumeModifyClaimsGetter has a method to return a VolumeModifyClaimInterface. diff --git a/pkg/client/clientset/versioned/typed/xuanwu/v1/volumemodifycontent.go b/pkg/client/clientset/versioned/typed/xuanwu/v1/volumemodifycontent.go index fa3d29bf..6d04a695 100644 --- a/pkg/client/clientset/versioned/typed/xuanwu/v1/volumemodifycontent.go +++ b/pkg/client/clientset/versioned/typed/xuanwu/v1/volumemodifycontent.go @@ -17,14 +17,15 @@ package v1 import ( "context" - v1 "huawei-csi-driver/client/apis/xuanwu/v1" - scheme "huawei-csi-driver/pkg/client/clientset/versioned/scheme" "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" types "k8s.io/apimachinery/pkg/types" watch "k8s.io/apimachinery/pkg/watch" rest "k8s.io/client-go/rest" + + v1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + scheme "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/clientset/versioned/scheme" ) // VolumeModifyContentsGetter has a method to return a VolumeModifyContentInterface. diff --git a/pkg/client/clientset/versioned/typed/xuanwu/v1/xuanwu_client.go b/pkg/client/clientset/versioned/typed/xuanwu/v1/xuanwu_client.go index d7548580..416cc142 100644 --- a/pkg/client/clientset/versioned/typed/xuanwu/v1/xuanwu_client.go +++ b/pkg/client/clientset/versioned/typed/xuanwu/v1/xuanwu_client.go @@ -16,11 +16,12 @@ package v1 import ( - v1 "huawei-csi-driver/client/apis/xuanwu/v1" - "huawei-csi-driver/pkg/client/clientset/versioned/scheme" "net/http" rest "k8s.io/client-go/rest" + + v1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/clientset/versioned/scheme" ) type XuanwuV1Interface interface { diff --git a/pkg/client/informers/externalversions/factory.go b/pkg/client/informers/externalversions/factory.go index e54931fd..ca33b2ad 100644 --- a/pkg/client/informers/externalversions/factory.go +++ b/pkg/client/informers/externalversions/factory.go @@ -16,17 +16,18 @@ package externalversions import ( - versioned "huawei-csi-driver/pkg/client/clientset/versioned" - internalinterfaces "huawei-csi-driver/pkg/client/informers/externalversions/internalinterfaces" - xuanwu "huawei-csi-driver/pkg/client/informers/externalversions/xuanwu" - reflect "reflect" - sync "sync" - time "time" + "reflect" + "sync" + "time" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" schema "k8s.io/apimachinery/pkg/runtime/schema" cache "k8s.io/client-go/tools/cache" + + versioned "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/clientset/versioned" + internalinterfaces "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/informers/externalversions/internalinterfaces" + xuanwu "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/informers/externalversions/xuanwu" ) // SharedInformerOption defines the functional option type for SharedInformerFactory. @@ -191,25 +192,25 @@ func (f *sharedInformerFactory) InformerFor(obj runtime.Object, newFunc internal // // It is typically used like this: // -// ctx, cancel := context.Background() -// defer cancel() -// factory := NewSharedInformerFactory(client, resyncPeriod) -// defer factory.WaitForStop() // Returns immediately if nothing was started. -// genericInformer := factory.ForResource(resource) -// typedInformer := factory.SomeAPIGroup().V1().SomeType() -// factory.Start(ctx.Done()) // Start processing these informers. -// synced := factory.WaitForCacheSync(ctx.Done()) -// for v, ok := range synced { -// if !ok { -// fmt.Fprintf(os.Stderr, "caches failed to sync: %v", v) -// return -// } -// } +// ctx, cancel := context.Background() +// defer cancel() +// factory := NewSharedInformerFactory(client, resyncPeriod) +// defer factory.WaitForStop() // Returns immediately if nothing was started. +// genericInformer := factory.ForResource(resource) +// typedInformer := factory.SomeAPIGroup().V1().SomeType() +// factory.Start(ctx.Done()) // Start processing these informers. +// synced := factory.WaitForCacheSync(ctx.Done()) +// for v, ok := range synced { +// if !ok { +// fmt.Fprintf(os.Stderr, "caches failed to sync: %v", v) +// return +// } +// } // -// // Creating informers can also be created after Start, but then -// // Start must be called again: -// anotherGenericInformer := factory.ForResource(resource) -// factory.Start(ctx.Done()) +// // Creating informers can also be created after Start, but then +// // Start must be called again: +// anotherGenericInformer := factory.ForResource(resource) +// factory.Start(ctx.Done()) type SharedInformerFactory interface { internalinterfaces.SharedInformerFactory diff --git a/pkg/client/informers/externalversions/generic.go b/pkg/client/informers/externalversions/generic.go index f7aaea89..4dbcda4d 100644 --- a/pkg/client/informers/externalversions/generic.go +++ b/pkg/client/informers/externalversions/generic.go @@ -17,10 +17,11 @@ package externalversions import ( "fmt" - v1 "huawei-csi-driver/client/apis/xuanwu/v1" schema "k8s.io/apimachinery/pkg/runtime/schema" cache "k8s.io/client-go/tools/cache" + + v1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" ) // GenericInformer is type of SharedIndexInformer which will locate and delegate to other diff --git a/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go b/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go index dc3faef7..5f17870c 100644 --- a/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go +++ b/pkg/client/informers/externalversions/internalinterfaces/factory_interfaces.go @@ -16,12 +16,13 @@ package internalinterfaces import ( - versioned "huawei-csi-driver/pkg/client/clientset/versioned" - time "time" + "time" v1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" cache "k8s.io/client-go/tools/cache" + + versioned "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/clientset/versioned" ) // NewInformerFunc takes versioned.Interface and time.Duration to return a SharedIndexInformer. diff --git a/pkg/client/informers/externalversions/xuanwu/interface.go b/pkg/client/informers/externalversions/xuanwu/interface.go index 4f2cfada..8beab100 100644 --- a/pkg/client/informers/externalversions/xuanwu/interface.go +++ b/pkg/client/informers/externalversions/xuanwu/interface.go @@ -16,8 +16,8 @@ package xuanwu import ( - internalinterfaces "huawei-csi-driver/pkg/client/informers/externalversions/internalinterfaces" - v1 "huawei-csi-driver/pkg/client/informers/externalversions/xuanwu/v1" + internalinterfaces "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/informers/externalversions/internalinterfaces" + v1 "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/informers/externalversions/xuanwu/v1" ) // Interface provides access to each of this group's versions. diff --git a/pkg/client/informers/externalversions/xuanwu/v1/interface.go b/pkg/client/informers/externalversions/xuanwu/v1/interface.go index 3506e218..9b165b09 100644 --- a/pkg/client/informers/externalversions/xuanwu/v1/interface.go +++ b/pkg/client/informers/externalversions/xuanwu/v1/interface.go @@ -16,7 +16,7 @@ package v1 import ( - internalinterfaces "huawei-csi-driver/pkg/client/informers/externalversions/internalinterfaces" + internalinterfaces "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/informers/externalversions/internalinterfaces" ) // Interface provides access to all the informers in this group version. diff --git a/pkg/client/informers/externalversions/xuanwu/v1/storagebackendclaim.go b/pkg/client/informers/externalversions/xuanwu/v1/storagebackendclaim.go index 609c4676..21b3a012 100644 --- a/pkg/client/informers/externalversions/xuanwu/v1/storagebackendclaim.go +++ b/pkg/client/informers/externalversions/xuanwu/v1/storagebackendclaim.go @@ -17,16 +17,17 @@ package v1 import ( "context" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" - versioned "huawei-csi-driver/pkg/client/clientset/versioned" - internalinterfaces "huawei-csi-driver/pkg/client/informers/externalversions/internalinterfaces" - v1 "huawei-csi-driver/pkg/client/listers/xuanwu/v1" - time "time" + "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" watch "k8s.io/apimachinery/pkg/watch" cache "k8s.io/client-go/tools/cache" + + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + versioned "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/clientset/versioned" + internalinterfaces "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/informers/externalversions/internalinterfaces" + v1 "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/listers/xuanwu/v1" ) // StorageBackendClaimInformer provides access to a shared informer and lister for diff --git a/pkg/client/informers/externalversions/xuanwu/v1/storagebackendcontent.go b/pkg/client/informers/externalversions/xuanwu/v1/storagebackendcontent.go index 8c99f2ff..89721244 100644 --- a/pkg/client/informers/externalversions/xuanwu/v1/storagebackendcontent.go +++ b/pkg/client/informers/externalversions/xuanwu/v1/storagebackendcontent.go @@ -17,16 +17,17 @@ package v1 import ( "context" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" - versioned "huawei-csi-driver/pkg/client/clientset/versioned" - internalinterfaces "huawei-csi-driver/pkg/client/informers/externalversions/internalinterfaces" - v1 "huawei-csi-driver/pkg/client/listers/xuanwu/v1" - time "time" + "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" watch "k8s.io/apimachinery/pkg/watch" cache "k8s.io/client-go/tools/cache" + + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + versioned "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/clientset/versioned" + internalinterfaces "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/informers/externalversions/internalinterfaces" + v1 "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/listers/xuanwu/v1" ) // StorageBackendContentInformer provides access to a shared informer and lister for diff --git a/pkg/client/informers/externalversions/xuanwu/v1/volumemodifyclaim.go b/pkg/client/informers/externalversions/xuanwu/v1/volumemodifyclaim.go index 8c29eb32..b9cff956 100644 --- a/pkg/client/informers/externalversions/xuanwu/v1/volumemodifyclaim.go +++ b/pkg/client/informers/externalversions/xuanwu/v1/volumemodifyclaim.go @@ -17,16 +17,17 @@ package v1 import ( "context" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" - versioned "huawei-csi-driver/pkg/client/clientset/versioned" - internalinterfaces "huawei-csi-driver/pkg/client/informers/externalversions/internalinterfaces" - v1 "huawei-csi-driver/pkg/client/listers/xuanwu/v1" - time "time" + "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" watch "k8s.io/apimachinery/pkg/watch" cache "k8s.io/client-go/tools/cache" + + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + versioned "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/clientset/versioned" + internalinterfaces "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/informers/externalversions/internalinterfaces" + v1 "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/listers/xuanwu/v1" ) // VolumeModifyClaimInformer provides access to a shared informer and lister for diff --git a/pkg/client/informers/externalversions/xuanwu/v1/volumemodifycontent.go b/pkg/client/informers/externalversions/xuanwu/v1/volumemodifycontent.go index bae7611d..627a62f6 100644 --- a/pkg/client/informers/externalversions/xuanwu/v1/volumemodifycontent.go +++ b/pkg/client/informers/externalversions/xuanwu/v1/volumemodifycontent.go @@ -17,16 +17,17 @@ package v1 import ( "context" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" - versioned "huawei-csi-driver/pkg/client/clientset/versioned" - internalinterfaces "huawei-csi-driver/pkg/client/informers/externalversions/internalinterfaces" - v1 "huawei-csi-driver/pkg/client/listers/xuanwu/v1" - time "time" + "time" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" runtime "k8s.io/apimachinery/pkg/runtime" watch "k8s.io/apimachinery/pkg/watch" cache "k8s.io/client-go/tools/cache" + + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + versioned "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/clientset/versioned" + internalinterfaces "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/informers/externalversions/internalinterfaces" + v1 "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/listers/xuanwu/v1" ) // VolumeModifyContentInformer provides access to a shared informer and lister for diff --git a/pkg/client/listers/xuanwu/v1/storagebackendclaim.go b/pkg/client/listers/xuanwu/v1/storagebackendclaim.go index 53af1581..6d950369 100644 --- a/pkg/client/listers/xuanwu/v1/storagebackendclaim.go +++ b/pkg/client/listers/xuanwu/v1/storagebackendclaim.go @@ -16,11 +16,11 @@ package v1 import ( - v1 "huawei-csi-driver/client/apis/xuanwu/v1" - "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/tools/cache" + + v1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" ) // StorageBackendClaimLister helps list StorageBackendClaims. diff --git a/pkg/client/listers/xuanwu/v1/storagebackendcontent.go b/pkg/client/listers/xuanwu/v1/storagebackendcontent.go index 9b70c02b..475b68e0 100644 --- a/pkg/client/listers/xuanwu/v1/storagebackendcontent.go +++ b/pkg/client/listers/xuanwu/v1/storagebackendcontent.go @@ -16,11 +16,11 @@ package v1 import ( - v1 "huawei-csi-driver/client/apis/xuanwu/v1" - "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/tools/cache" + + v1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" ) // StorageBackendContentLister helps list StorageBackendContents. diff --git a/pkg/client/listers/xuanwu/v1/volumemodifyclaim.go b/pkg/client/listers/xuanwu/v1/volumemodifyclaim.go index e60f4103..1910af6e 100644 --- a/pkg/client/listers/xuanwu/v1/volumemodifyclaim.go +++ b/pkg/client/listers/xuanwu/v1/volumemodifyclaim.go @@ -16,11 +16,11 @@ package v1 import ( - v1 "huawei-csi-driver/client/apis/xuanwu/v1" - "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/tools/cache" + + v1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" ) // VolumeModifyClaimLister helps list VolumeModifyClaims. diff --git a/pkg/client/listers/xuanwu/v1/volumemodifycontent.go b/pkg/client/listers/xuanwu/v1/volumemodifycontent.go index 01cd2d25..cf387caa 100644 --- a/pkg/client/listers/xuanwu/v1/volumemodifycontent.go +++ b/pkg/client/listers/xuanwu/v1/volumemodifycontent.go @@ -16,11 +16,11 @@ package v1 import ( - v1 "huawei-csi-driver/client/apis/xuanwu/v1" - "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" "k8s.io/client-go/tools/cache" + + v1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" ) // VolumeModifyContentLister helps list VolumeModifyContents. diff --git a/pkg/constants/constants.go b/pkg/constants/constants.go index c8f798e2..ebf91e88 100644 --- a/pkg/constants/constants.go +++ b/pkg/constants/constants.go @@ -22,7 +22,7 @@ type FileType string const ( // ProviderVersion defines provider version - ProviderVersion = "4.5.0" + ProviderVersion = "4.6.0" // ProviderVendorName defines provider vendor name ProviderVendorName = "Huawei" // EndpointDirPermission defines permission of endpoint dir @@ -74,8 +74,12 @@ const ( // DefaultKubeletVolumeDevicesDirName default kubelet volumeDevice name DefaultKubeletVolumeDevicesDirName = "/volumeDevices/" - // AllocationUnitBytes default is 512 + // AllocationUnitBytes default is 512 Bytes, it is the allocation and capacity unit for OceanStor AllocationUnitBytes = 512 + // FusionAllocUnitBytes default is 1Mi, it is the allocation unit for FusionStorage + FusionAllocUnitBytes = 1024 * 1024 + // FusionFileCapacityUnit default is 1024 Bytes, it is the capacity unit for FusionStorage + FusionFileCapacityUnit int64 = 1024 // DefaultIntBase is the default value of int base DefaultIntBase = 10 diff --git a/pkg/constants/storage.go b/pkg/constants/storage.go index f01d545d..a674b90d 100644 --- a/pkg/constants/storage.go +++ b/pkg/constants/storage.go @@ -17,23 +17,65 @@ // Package constants is storage-related constants package constants +// OceanstorVersion defines Oceantor storage's version +type OceanstorVersion string + +// OceanDiskVersion defines OceanDisk storage's version +type OceanDiskVersion string + +// IsDorado checks whether the version is Dorado +func (ver OceanstorVersion) IsDorado() bool { + return ver == "Dorado" +} + +// IsDoradoV6OrV7 checks whether the version is Dorado v6 or Dorado v7 +func (ver OceanstorVersion) IsDoradoV6OrV7() bool { + return ver.IsDoradoV6() || ver.IsDoradoV7() +} + +// IsDoradoV6 checks whether the version is Dorado v6 +func (ver OceanstorVersion) IsDoradoV6() bool { + return ver == OceanStorDoradoV6 +} + +// IsDoradoV7 checks whether the version is Dorado v7 +func (ver OceanstorVersion) IsDoradoV7() bool { + return ver == OceanStorDoradoV7 +} + const ( + // OceanStorDoradoV7 is dorado v7 + OceanStorDoradoV7 OceanstorVersion = "DoradoV7" // OceanStorDoradoV6 Dorado V6 and OceanStor V6 are exactly the same - OceanStorDoradoV6 = "DoradoV6" + OceanStorDoradoV6 OceanstorVersion = "DoradoV6" // OceanStorDoradoV3 is dorado v3 - OceanStorDoradoV3 = "DoradoV3" + OceanStorDoradoV3 OceanstorVersion = "DoradoV3" // OceanStorV3 is oceanstor v3 - OceanStorV3 = "OceanStorV3" + OceanStorV3 OceanstorVersion = "OceanStorV3" // OceanStorV5 is oceanstor v5 - OceanStorV5 = "OceanStorV5" + OceanStorV5 OceanstorVersion = "OceanStorV5" +) +const ( // DoradoV615 is Dorado V6.1.5 DoradoV615 = "6.1.5" // MinVersionSupportNfsPlus version gte 6.1.7 support label function MinVersionSupportNfsPlus = "6.1.7" + // OceanStor9000 storage type is oceanstor-9000 + OceanStor9000 = "oceanstor-9000" + // OceanStorDtree storage type is oceanstor-dtree + OceanStorDtree = "oceanstor-dtree" // OceanStorNas storage type is oceanstor-nas OceanStorNas = "oceanstor-nas" + // OceanStorSan storage type is oceanstor-san + OceanStorSan = "oceanstor-san" + // OceandiskSan storage type is oceandisk-san + OceandiskSan = "oceandisk-san" + // FusionSan storage type is fusionstorage-san + FusionSan = "fusionstorage-san" + // FusionNas storage type is fusionstorage-nas + FusionNas = "fusionstorage-nas" // CloneSpeedLevel1 means level1 of the clone speed CloneSpeedLevel1 = 1 diff --git a/pkg/finalizers/finalizers_test.go b/pkg/finalizers/finalizers_test.go index 01507ebf..8274a622 100644 --- a/pkg/finalizers/finalizers_test.go +++ b/pkg/finalizers/finalizers_test.go @@ -20,7 +20,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" ) const ( diff --git a/pkg/modify/claim_worker.go b/pkg/modify/claim_worker.go index 1b0e2fd4..59436faa 100644 --- a/pkg/modify/claim_worker.go +++ b/pkg/modify/claim_worker.go @@ -32,11 +32,11 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/labels" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" - pkgutils "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/storage/oceanstor/client" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + pkgutils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceanstor/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/pkg/modify/claim_worker_test.go b/pkg/modify/claim_worker_test.go index efc11a62..e2670502 100644 --- a/pkg/modify/claim_worker_test.go +++ b/pkg/modify/claim_worker_test.go @@ -28,11 +28,11 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/record" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" - clientSet "huawei-csi-driver/pkg/client/clientset/versioned" - "huawei-csi-driver/pkg/client/clientset/versioned/fake" - backendInformers "huawei-csi-driver/pkg/client/informers/externalversions" - "huawei-csi-driver/utils" + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + clientSet "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/clientset/versioned" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/clientset/versioned/fake" + backendInformers "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/informers/externalversions" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" ) func TestModifyClaimController_syncClaimWork_WhenGetClaimFromListerFailed(t *testing.T) { diff --git a/pkg/modify/content_worker.go b/pkg/modify/content_worker.go index e40767b5..912a7222 100644 --- a/pkg/modify/content_worker.go +++ b/pkg/modify/content_worker.go @@ -29,10 +29,10 @@ import ( apiErrors "k8s.io/apimachinery/pkg/api/errors" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" - "huawei-csi-driver/lib/drcsi" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/lib/drcsi" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/pkg/modify/controller.go b/pkg/modify/controller.go index 7784d196..d5b72363 100644 --- a/pkg/modify/controller.go +++ b/pkg/modify/controller.go @@ -29,15 +29,15 @@ import ( "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" - "huawei-csi-driver/lib/drcsi" - clientset "huawei-csi-driver/pkg/client/clientset/versioned" - "huawei-csi-driver/pkg/client/clientset/versioned/scheme" - external "huawei-csi-driver/pkg/client/informers/externalversions" - modifyinformers "huawei-csi-driver/pkg/client/informers/externalversions/xuanwu/v1" - modifylisters "huawei-csi-driver/pkg/client/listers/xuanwu/v1" - "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/utils/log" + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/lib/drcsi" + clientset "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/clientset/versioned" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/clientset/versioned/scheme" + external "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/informers/externalversions" + modifyinformers "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/informers/externalversions/xuanwu/v1" + modifylisters "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/listers/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/pkg/modify/object_worker.go b/pkg/modify/object_worker.go index 2ee53212..274ce936 100644 --- a/pkg/modify/object_worker.go +++ b/pkg/modify/object_worker.go @@ -25,7 +25,7 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // syncObjectFunc is a sync object function diff --git a/pkg/modify/object_worker_test.go b/pkg/modify/object_worker_test.go index 0e248aa5..c6c8734d 100644 --- a/pkg/modify/object_worker_test.go +++ b/pkg/modify/object_worker_test.go @@ -27,7 +27,7 @@ import ( "k8s.io/client-go/tools/cache" "k8s.io/client-go/util/workqueue" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) var logName = "volume" diff --git a/pkg/sidecar/controller/content_delete.go b/pkg/sidecar/controller/content_delete.go index 7e74f15e..ca772ce1 100644 --- a/pkg/sidecar/controller/content_delete.go +++ b/pkg/sidecar/controller/content_delete.go @@ -22,9 +22,9 @@ import ( v1 "k8s.io/api/core/v1" apiErrors "k8s.io/apimachinery/pkg/api/errors" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" - "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/utils/log" + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) func (ctrl *backendController) removeProviderBackend(ctx context.Context, diff --git a/pkg/sidecar/controller/content_sync.go b/pkg/sidecar/controller/content_sync.go index 059fd4c1..0efd0963 100644 --- a/pkg/sidecar/controller/content_sync.go +++ b/pkg/sidecar/controller/content_sync.go @@ -22,10 +22,10 @@ import ( coreV1 "k8s.io/api/core/v1" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" - "huawei-csi-driver/lib/drcsi" - "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/utils/log" + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/lib/drcsi" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) func (ctrl *backendController) initContentStatus(ctx context.Context, content *xuanwuv1.StorageBackendContent) ( diff --git a/pkg/sidecar/controller/sidecar_controller.go b/pkg/sidecar/controller/sidecar_controller.go index 33b2a180..7a32044c 100644 --- a/pkg/sidecar/controller/sidecar_controller.go +++ b/pkg/sidecar/controller/sidecar_controller.go @@ -31,14 +31,14 @@ import ( "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" - clientSet "huawei-csi-driver/pkg/client/clientset/versioned" - backendInformers "huawei-csi-driver/pkg/client/informers/externalversions/xuanwu/v1" - backendListers "huawei-csi-driver/pkg/client/listers/xuanwu/v1" - storageBackend "huawei-csi-driver/pkg/storage-backend/handle" - "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/utils/flow" - "huawei-csi-driver/utils/log" + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + clientSet "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/clientset/versioned" + backendInformers "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/informers/externalversions/xuanwu/v1" + backendListers "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/listers/xuanwu/v1" + storageBackend "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/storage-backend/handle" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/flow" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/pkg/sidecar/controller/sidecar_handler.go b/pkg/sidecar/controller/sidecar_handler.go index 8b4954ed..5df67aac 100644 --- a/pkg/sidecar/controller/sidecar_handler.go +++ b/pkg/sidecar/controller/sidecar_handler.go @@ -19,10 +19,10 @@ import ( "context" "time" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" - "huawei-csi-driver/lib/drcsi" - storageBackend "huawei-csi-driver/pkg/storage-backend/handle" - "huawei-csi-driver/utils/log" + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/lib/drcsi" + storageBackend "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/storage-backend/handle" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // Handler includes the interface of storage backend side diff --git a/pkg/storage-backend/controller/claim_delete.go b/pkg/storage-backend/controller/claim_delete.go index 30d5c4cf..f27f40b0 100644 --- a/pkg/storage-backend/controller/claim_delete.go +++ b/pkg/storage-backend/controller/claim_delete.go @@ -23,10 +23,10 @@ import ( coreV1 "k8s.io/api/core/v1" apiErrors "k8s.io/apimachinery/pkg/api/errors" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" - "huawei-csi-driver/pkg/finalizers" - "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/utils/log" + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/finalizers" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) func (ctrl *BackendController) deleteStorageBackendClaim(ctx context.Context, diff --git a/pkg/storage-backend/controller/claim_delete_test.go b/pkg/storage-backend/controller/claim_delete_test.go index ddb25ed8..8636d910 100644 --- a/pkg/storage-backend/controller/claim_delete_test.go +++ b/pkg/storage-backend/controller/claim_delete_test.go @@ -22,8 +22,8 @@ import ( "github.com/agiledragon/gomonkey/v2" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" - "huawei-csi-driver/pkg/utils" + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" ) func TestDeleteStorageBackendClaim(t *testing.T) { diff --git a/pkg/storage-backend/controller/claim_sync.go b/pkg/storage-backend/controller/claim_sync.go index fe5e267a..b3457fcb 100644 --- a/pkg/storage-backend/controller/claim_sync.go +++ b/pkg/storage-backend/controller/claim_sync.go @@ -25,12 +25,12 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/cache" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" - "huawei-csi-driver/csi/backend" - "huawei-csi-driver/pkg/finalizers" - "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/utils/flow" - "huawei-csi-driver/utils/log" + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/finalizers" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/flow" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // syncClaimByKey processes a StorageBackendClaim request. diff --git a/pkg/storage-backend/controller/configmap.go b/pkg/storage-backend/controller/configmap.go index 3949564b..a1efcc85 100644 --- a/pkg/storage-backend/controller/configmap.go +++ b/pkg/storage-backend/controller/configmap.go @@ -24,11 +24,11 @@ import ( apiErrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" - "huawei-csi-driver/csi/app" - "huawei-csi-driver/pkg/finalizers" - "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/utils/log" + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/finalizers" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) func (ctrl *BackendController) syncConfigmap(ctx context.Context, storageBackend *xuanwuv1.StorageBackendClaim) ( diff --git a/pkg/storage-backend/controller/content_delete.go b/pkg/storage-backend/controller/content_delete.go index f2d1f471..442beccc 100644 --- a/pkg/storage-backend/controller/content_delete.go +++ b/pkg/storage-backend/controller/content_delete.go @@ -20,8 +20,8 @@ import ( "errors" "fmt" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" - "huawei-csi-driver/utils/log" + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) func (ctrl *BackendController) deleteStorageBackendContent(ctx context.Context, diff --git a/pkg/storage-backend/controller/content_delete_test.go b/pkg/storage-backend/controller/content_delete_test.go index d859e092..473fa438 100644 --- a/pkg/storage-backend/controller/content_delete_test.go +++ b/pkg/storage-backend/controller/content_delete_test.go @@ -29,11 +29,11 @@ import ( coreV1 "k8s.io/client-go/kubernetes/typed/core/v1" "k8s.io/client-go/tools/record" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" - "huawei-csi-driver/pkg/client/clientset/versioned/fake" - backendInformers "huawei-csi-driver/pkg/client/informers/externalversions" + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/clientset/versioned/fake" + backendInformers "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/informers/externalversions" ) const ( diff --git a/pkg/storage-backend/controller/content_sync.go b/pkg/storage-backend/controller/content_sync.go index 3803b9ff..a95b8ce0 100644 --- a/pkg/storage-backend/controller/content_sync.go +++ b/pkg/storage-backend/controller/content_sync.go @@ -21,10 +21,10 @@ import ( coreV1 "k8s.io/api/core/v1" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" - "huawei-csi-driver/pkg/finalizers" - "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/utils/log" + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/finalizers" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) func (ctrl *BackendController) updateContent(ctx context.Context, content *xuanwuv1.StorageBackendContent) error { diff --git a/pkg/storage-backend/controller/content_sync_test.go b/pkg/storage-backend/controller/content_sync_test.go index 77435ec2..72b13ed0 100644 --- a/pkg/storage-backend/controller/content_sync_test.go +++ b/pkg/storage-backend/controller/content_sync_test.go @@ -20,9 +20,9 @@ import ( "context" "testing" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" - "huawei-csi-driver/pkg/finalizers" - "huawei-csi-driver/pkg/utils" + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/finalizers" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" ) func TestUpdateContentAddFinalizerFailed(t *testing.T) { diff --git a/pkg/storage-backend/controller/controller.go b/pkg/storage-backend/controller/controller.go index a0940678..b11d1305 100644 --- a/pkg/storage-backend/controller/controller.go +++ b/pkg/storage-backend/controller/controller.go @@ -31,12 +31,12 @@ import ( "k8s.io/client-go/tools/record" "k8s.io/client-go/util/workqueue" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" - clientSet "huawei-csi-driver/pkg/client/clientset/versioned" - backendInformers "huawei-csi-driver/pkg/client/informers/externalversions/xuanwu/v1" - backendListers "huawei-csi-driver/pkg/client/listers/xuanwu/v1" - "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/utils/log" + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + clientSet "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/clientset/versioned" + backendInformers "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/informers/externalversions/xuanwu/v1" + backendListers "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/listers/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/pkg/storage-backend/controller/secret.go b/pkg/storage-backend/controller/secret.go index 58df8ccb..dd195b1a 100644 --- a/pkg/storage-backend/controller/secret.go +++ b/pkg/storage-backend/controller/secret.go @@ -24,11 +24,11 @@ import ( apiErrors "k8s.io/apimachinery/pkg/api/errors" "k8s.io/apimachinery/pkg/labels" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" - "huawei-csi-driver/csi/app" - "huawei-csi-driver/pkg/finalizers" - "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/utils/log" + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/finalizers" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) func (ctrl *BackendController) syncSecret(ctx context.Context, storageBackend *xuanwuv1.StorageBackendClaim) ( diff --git a/pkg/storage-backend/handle/storage_backend.go b/pkg/storage-backend/handle/storage_backend.go index 088548f5..1c20280d 100644 --- a/pkg/storage-backend/handle/storage_backend.go +++ b/pkg/storage-backend/handle/storage_backend.go @@ -21,10 +21,10 @@ import ( "google.golang.org/grpc" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" - "huawei-csi-driver/lib/drcsi" - "huawei-csi-driver/lib/drcsi/rpc" - "huawei-csi-driver/utils/log" + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/lib/drcsi" + "github.com/Huawei/eSDK_K8S_Plugin/v4/lib/drcsi/rpc" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // BackendInterfaces includes interfaces that call provider diff --git a/pkg/storage-backend/handle/storage_backend_test.go b/pkg/storage-backend/handle/storage_backend_test.go index abbc251d..5c6b07e0 100644 --- a/pkg/storage-backend/handle/storage_backend_test.go +++ b/pkg/storage-backend/handle/storage_backend_test.go @@ -23,8 +23,8 @@ import ( "github.com/agiledragon/gomonkey/v2" "google.golang.org/grpc" - "huawei-csi-driver/lib/drcsi" - "huawei-csi-driver/lib/drcsi/rpc" + "github.com/Huawei/eSDK_K8S_Plugin/v4/lib/drcsi" + "github.com/Huawei/eSDK_K8S_Plugin/v4/lib/drcsi/rpc" ) func initBackend() *backend { diff --git a/pkg/utils/claim_utils.go b/pkg/utils/claim_utils.go index 0fd9d0a1..2f6cdb6f 100644 --- a/pkg/utils/claim_utils.go +++ b/pkg/utils/claim_utils.go @@ -20,9 +20,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" - clientSet "huawei-csi-driver/pkg/client/clientset/versioned" - "huawei-csi-driver/utils/log" + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + clientSet "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/clientset/versioned" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // GetClaim used to get claim by xuanwu client diff --git a/pkg/utils/content_utils.go b/pkg/utils/content_utils.go index 9e699983..d489a078 100644 --- a/pkg/utils/content_utils.go +++ b/pkg/utils/content_utils.go @@ -20,9 +20,9 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" - clientSet "huawei-csi-driver/pkg/client/clientset/versioned" - "huawei-csi-driver/utils/log" + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + clientSet "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/clientset/versioned" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // CreateContent used to create content by xuanwu client diff --git a/pkg/utils/k8s.go b/pkg/utils/k8s.go index 9427343c..4da735aa 100644 --- a/pkg/utils/k8s.go +++ b/pkg/utils/k8s.go @@ -33,12 +33,12 @@ import ( "k8s.io/client-go/tools/clientcmd" "k8s.io/client-go/tools/record" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" - "huawei-csi-driver/csi/app" - clientSet "huawei-csi-driver/pkg/client/clientset/versioned" - "huawei-csi-driver/pkg/constants" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + clientSet "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/client/clientset/versioned" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) func IsSBCTExist(ctx context.Context, backendID string) bool { diff --git a/pkg/utils/leader_election.go b/pkg/utils/leader_election.go index 491443ef..b01d267c 100644 --- a/pkg/utils/leader_election.go +++ b/pkg/utils/leader_election.go @@ -26,8 +26,8 @@ import ( "k8s.io/client-go/tools/leaderelection/resourcelock" "k8s.io/client-go/tools/record" - "huawei-csi-driver/csi/app" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // LeaderElectionConf include the configuration of leader election diff --git a/pkg/utils/notify_utils.go b/pkg/utils/notify_utils.go index af97e264..afcd3344 100644 --- a/pkg/utils/notify_utils.go +++ b/pkg/utils/notify_utils.go @@ -21,7 +21,7 @@ import ( "reflect" "runtime/debug" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/pkg/utils/utils.go b/pkg/utils/utils.go index ea63761d..5e1e71e3 100644 --- a/pkg/utils/utils.go +++ b/pkg/utils/utils.go @@ -29,10 +29,10 @@ import ( "k8s.io/apimachinery/pkg/api/meta" "k8s.io/client-go/tools/cache" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" - "huawei-csi-driver/csi/app" - "huawei-csi-driver/pkg/finalizers" - "huawei-csi-driver/utils/log" + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/finalizers" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) var ( diff --git a/pkg/utils/utils_test.go b/pkg/utils/utils_test.go index c7a5f08d..808dfb8e 100644 --- a/pkg/utils/utils_test.go +++ b/pkg/utils/utils_test.go @@ -23,12 +23,12 @@ import ( "time" "github.com/stretchr/testify/require" - //"huawei-csi-driver/utils/log" + //"github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/client-go/tools/cache" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" - "huawei-csi-driver/utils/log" + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/pkg/webhook/utils.go b/pkg/webhook/utils.go index 7547228b..25d12774 100644 --- a/pkg/webhook/utils.go +++ b/pkg/webhook/utils.go @@ -16,85 +16,14 @@ limitations under the License. package webhook import ( - "bytes" "context" - "crypto/ecdsa" - "crypto/elliptic" - "crypto/rand" - "crypto/tls" - "crypto/x509" - "crypto/x509/pkix" - "encoding/pem" - "math/big" - "time" "k8s.io/api/core/v1" metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "huawei-csi-driver/csi/app" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" ) -const certUntilYears = 99 - -// GenerateCertificate Self Signed certificate using given CN, returns x509 cert -// and priv key in PEM format -func GenerateCertificate(ctx context.Context, cn string, dnsName string) ([]byte, []byte, error) { - var err error - pemCert := &bytes.Buffer{} - - // generate private key - priv, err := ecdsa.GenerateKey(elliptic.P521(), rand.Reader) - if err != nil { - log.AddContext(ctx).Errorf("error generating crypt keys: %v", err) - return nil, nil, err - } - - // create certificate - template := x509.Certificate{ - SerialNumber: big.NewInt(1), - Subject: pkix.Name{ - CommonName: cn, - }, - NotBefore: time.Now(), - NotAfter: time.Now().AddDate(certUntilYears, 0, 0), - KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, - ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, - BasicConstraintsValid: true, - IsCA: true, - DNSNames: []string{dnsName}, - } - derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) - if err != nil { - log.AddContext(ctx).Errorf("Failed to create x509 certificate: %s", err) - return nil, nil, err - } - - err = pem.Encode(pemCert, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) - if err != nil { - log.AddContext(ctx).Errorf("Unable to encode x509 certificate to PEM format: %v", err) - return nil, nil, err - } - - b, err := x509.MarshalECPrivateKey(priv) - if err != nil { - log.AddContext(ctx).Errorf("Unable to marshal ECDSA private key: %v", err) - return nil, nil, err - } - privateBlock := pem.Block{ - Type: "RSA PRIVATE KEY", - Headers: nil, - Bytes: b, - } - - return pemCert.Bytes(), pem.EncodeToMemory(&privateBlock), nil -} - -// GetTLSCertificate from pub and priv key -func GetTLSCertificate(cert, priv []byte) (tls.Certificate, error) { - return tls.X509KeyPair(cert, priv) -} - // CreateCertSecrets creates k8s secret to store signed cert data func CreateCertSecrets(ctx context.Context, webHookCfg Config, cert, key []byte, ns string) (*v1.Secret, error) { secretData := make(map[string][]byte) diff --git a/pkg/webhook/validate_webhook.go b/pkg/webhook/validate_webhook.go index 8e3683b5..74f4fcca 100644 --- a/pkg/webhook/validate_webhook.go +++ b/pkg/webhook/validate_webhook.go @@ -23,8 +23,8 @@ import ( apisErrors "k8s.io/apimachinery/pkg/api/errors" metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "huawei-csi-driver/pkg/admission" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/admission" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // AdmissionWebHookCFG defines cfg of admission webhook diff --git a/pkg/webhook/webhook.go b/pkg/webhook/webhook.go index 87c5c1d8..bf3b423d 100644 --- a/pkg/webhook/webhook.go +++ b/pkg/webhook/webhook.go @@ -35,11 +35,12 @@ import ( "k8s.io/apimachinery/pkg/runtime/schema" "k8s.io/client-go/tools/record" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" - "huawei-csi-driver/csi/app" - "huawei-csi-driver/csi/backend" - "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/utils/log" + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/backend" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/cert" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // Controller include webhook resources @@ -231,14 +232,14 @@ func (c *Controller) getTlsCert(ctx context.Context, webHookCfg Config, ns strin } else if apisErrors.IsNotFound(err) { dnsName := webHookCfg.ServiceName + "." + ns + ".svc" cn := fmt.Sprintf("%s CA", webHookCfg.ServiceName) - caBundle, key, err := GenerateCertificate(ctx, cn, dnsName) + caBundle, key, err := cert.GenerateCertificate(ctx, cn, dnsName) if err != nil { log.AddContext(ctx).Errorf("Unable to generate x509 certificate: %v", err) return tls.Certificate{}, nil, err } caBytes = caBundle - tlsCert, err = GetTLSCertificate(caBundle, key) + tlsCert, err = cert.GetTLSCertificate(caBundle, key) if err != nil { log.AddContext(ctx).Errorf("Unable to create tls certificate: %v", err) return tls.Certificate{}, nil, err @@ -261,7 +262,7 @@ func (c *Controller) getTlsCert(ctx context.Context, webHookCfg Config, ns strin } caBytes = caBundle - tlsCert, err = GetTLSCertificate(caBundle, secretData) + tlsCert, err = cert.GetTLSCertificate(caBundle, secretData) if err != nil { log.AddContext(ctx).Errorf("unable to generate tls certs: %v", err) return tls.Certificate{}, nil, err diff --git a/pkg/webhook/webhook_cfg.go b/pkg/webhook/webhook_cfg.go index 7119a731..1fcd5704 100644 --- a/pkg/webhook/webhook_cfg.go +++ b/pkg/webhook/webhook_cfg.go @@ -20,8 +20,8 @@ import ( admissionV1 "k8s.io/api/admissionregistration/v1" - "huawei-csi-driver/csi/app" - "huawei-csi-driver/pkg/constants" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" ) const ( diff --git a/pkg/webhook/webhook_test.go b/pkg/webhook/webhook_test.go index a9bdc77a..0a2a8987 100644 --- a/pkg/webhook/webhook_test.go +++ b/pkg/webhook/webhook_test.go @@ -27,11 +27,11 @@ import ( "github.com/prashantv/gostub" corev1 "k8s.io/api/core/v1" - xuanwuv1 "huawei-csi-driver/client/apis/xuanwu/v1" - "huawei-csi-driver/csi/app" - cfg "huawei-csi-driver/csi/app/config" - "huawei-csi-driver/utils/k8sutils" - "huawei-csi-driver/utils/log" + xuanwuv1 "github.com/Huawei/eSDK_K8S_Plugin/v4/client/apis/xuanwu/v1" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + cfg "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/k8sutils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) var ( diff --git a/proto/proto.go b/proto/proto.go index 789cbe5b..64eca88e 100644 --- a/proto/proto.go +++ b/proto/proto.go @@ -24,8 +24,8 @@ import ( "net" "strings" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // GetISCSIInitiator used to get iscsi initiator diff --git a/proto/proto_test.go b/proto/proto_test.go index fda9d1f7..b74b7dd5 100644 --- a/proto/proto_test.go +++ b/proto/proto_test.go @@ -23,8 +23,8 @@ import ( "github.com/stretchr/testify/assert" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/storage/fusionstorage/attacher/attacher.go b/storage/fusionstorage/attacher/attacher.go index a3575f4c..77c23df0 100644 --- a/storage/fusionstorage/attacher/attacher.go +++ b/storage/fusionstorage/attacher/attacher.go @@ -24,13 +24,13 @@ import ( "net" "strings" - "huawei-csi-driver/connector" - _ "huawei-csi-driver/connector/iscsi" - _ "huawei-csi-driver/connector/local" - "huawei-csi-driver/storage/fusionstorage/client" - "huawei-csi-driver/storage/oceanstor/attacher" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector" + _ "github.com/Huawei/eSDK_K8S_Plugin/v4/connector/iscsi" + _ "github.com/Huawei/eSDK_K8S_Plugin/v4/connector/local" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/fusionstorage/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/base/attacher" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // VolumeAttacher defines attacher client diff --git a/storage/fusionstorage/client/client.go b/storage/fusionstorage/client/client.go index c5f5aff8..6182fb8f 100644 --- a/storage/fusionstorage/client/client.go +++ b/storage/fusionstorage/client/client.go @@ -32,19 +32,22 @@ import ( "sync" "time" - pkgUtils "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/storage/fusionstorage/types" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/fusionstorage/types" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( noAuthenticated int64 = 10000003 offLineCodeInt int64 = 1077949069 - defaultParallelCount int = 50 - maxParallelCount int = 1000 - minParallelCount int = 20 + // Cause of pacific storage does not have a determined parallel count, + // we set default parallel count to 30, which is consistent with the previous default value in our document. + // This configuration has run smoothly for a long time. + defaultParallelCount int = 30 + maxParallelCount int = 30 + minParallelCount int = 1 loginFailed = 1077949061 loginFailedWithArg = 1077987870 @@ -73,8 +76,7 @@ var ( }, } - debugLogRegex = map[string][]string{} - clientSemaphore *utils.Semaphore + debugLogRegex = map[string][]string{} ) func isFilterLog(method, url string) bool { @@ -98,7 +100,8 @@ type RestClient struct { authToken string client *http.Client - reloginMutex sync.Mutex + reloginMutex sync.Mutex + RequestSemaphore *utils.Semaphore } // NewClientConfig stores the information needed to create a new FusionStorage client @@ -119,28 +122,24 @@ func NewClient(ctx context.Context, clientConfig *NewClientConfig) *RestClient { var err error var parallelCount int - if len(clientConfig.ParallelNum) > 0 { - parallelCount, err = strconv.Atoi(clientConfig.ParallelNum) - if err != nil || parallelCount > maxParallelCount || parallelCount < minParallelCount { - log.AddContext(ctx).Warningf("The config parallelNum %d is invalid, set it to the default value %d", - parallelCount, defaultParallelCount) - parallelCount = defaultParallelCount - } - } else { + parallelCount, err = strconv.Atoi(clientConfig.ParallelNum) + if err != nil || parallelCount > maxParallelCount || parallelCount < minParallelCount { + log.Infof("The config parallelNum %d is invalid, set it to the default value %d", + parallelCount, defaultParallelCount) parallelCount = defaultParallelCount } log.AddContext(ctx).Infof("Init parallel count is %d", parallelCount) - clientSemaphore = utils.NewSemaphore(parallelCount) return &RestClient{ - url: clientConfig.Url, - user: clientConfig.User, - secretName: clientConfig.SecretName, - secretNamespace: clientConfig.SecretNamespace, - backendID: clientConfig.BackendID, - accountName: clientConfig.AccountName, - useCert: clientConfig.UseCert, - certSecretMeta: clientConfig.CertSecretMeta, + url: clientConfig.Url, + user: clientConfig.User, + secretName: clientConfig.SecretName, + secretNamespace: clientConfig.SecretNamespace, + backendID: clientConfig.BackendID, + accountName: clientConfig.AccountName, + useCert: clientConfig.UseCert, + certSecretMeta: clientConfig.CertSecretMeta, + RequestSemaphore: utils.NewSemaphore(parallelCount), } } @@ -360,8 +359,8 @@ func (cli *RestClient) doCall(ctx context.Context, method string, url string, da log.FilteredLog(ctx, isFilterLog(method, url), utils.IsDebugLog(method, url, debugLog, debugLogRegex), fmt.Sprintf("Request method: %s, url: %s, body: %v", method, req.URL, data)) - clientSemaphore.Acquire() - defer clientSemaphore.Release() + cli.RequestSemaphore.Acquire() + defer cli.RequestSemaphore.Release() resp, err := cli.client.Do(req) if err != nil { diff --git a/storage/fusionstorage/client/client_host.go b/storage/fusionstorage/client/client_host.go index 9ffc72df..73bd46a2 100644 --- a/storage/fusionstorage/client/client_host.go +++ b/storage/fusionstorage/client/client_host.go @@ -21,7 +21,7 @@ import ( "errors" "fmt" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/storage/fusionstorage/client/client_iscsi.go b/storage/fusionstorage/client/client_iscsi.go index b2c3c225..f98fd0ad 100644 --- a/storage/fusionstorage/client/client_iscsi.go +++ b/storage/fusionstorage/client/client_iscsi.go @@ -20,7 +20,7 @@ import ( "context" "fmt" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/storage/fusionstorage/client/client_namespace.go b/storage/fusionstorage/client/client_namespace.go index fe473317..8cae14a9 100644 --- a/storage/fusionstorage/client/client_namespace.go +++ b/storage/fusionstorage/client/client_namespace.go @@ -24,7 +24,7 @@ import ( fusionURL "net/url" "strconv" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/storage/fusionstorage/client/client_qos.go b/storage/fusionstorage/client/client_qos.go index 760eeeab..1c7fcd65 100644 --- a/storage/fusionstorage/client/client_qos.go +++ b/storage/fusionstorage/client/client_qos.go @@ -22,10 +22,10 @@ import ( "fmt" netUrl "net/url" - pkgUtils "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/storage/fusionstorage/types" - "huawei-csi-driver/storage/fusionstorage/utils" - "huawei-csi-driver/utils/log" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/fusionstorage/types" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/fusionstorage/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // GetConvergedQoSNameByID used to get qos name by id diff --git a/storage/fusionstorage/client/client_quota.go b/storage/fusionstorage/client/client_quota.go index 23319415..be3a5ceb 100644 --- a/storage/fusionstorage/client/client_quota.go +++ b/storage/fusionstorage/client/client_quota.go @@ -21,7 +21,7 @@ import ( "errors" "fmt" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/storage/fusionstorage/client/client_snapshot.go b/storage/fusionstorage/client/client_snapshot.go index 55ef1302..e110ac47 100644 --- a/storage/fusionstorage/client/client_snapshot.go +++ b/storage/fusionstorage/client/client_snapshot.go @@ -20,7 +20,7 @@ import ( "context" "fmt" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/storage/fusionstorage/client/client_system.go b/storage/fusionstorage/client/client_system.go index d565fdb2..92941c51 100644 --- a/storage/fusionstorage/client/client_system.go +++ b/storage/fusionstorage/client/client_system.go @@ -22,9 +22,9 @@ import ( "fmt" "strings" - pkgUtils "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // GetAccountIdByName gets account id by account name diff --git a/storage/fusionstorage/client/client_test.go b/storage/fusionstorage/client/client_test.go index 88047c4b..19c5129c 100644 --- a/storage/fusionstorage/client/client_test.go +++ b/storage/fusionstorage/client/client_test.go @@ -23,9 +23,9 @@ import ( "github.com/prashantv/gostub" "github.com/stretchr/testify/require" - "huawei-csi-driver/csi/app" - cfg "huawei-csi-driver/csi/app/config" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + cfg "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) var ( diff --git a/storage/fusionstorage/client/client_volume.go b/storage/fusionstorage/client/client_volume.go index e3924ce6..713efc80 100644 --- a/storage/fusionstorage/client/client_volume.go +++ b/storage/fusionstorage/client/client_volume.go @@ -22,7 +22,7 @@ import ( "fmt" "strconv" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/storage/fusionstorage/smartx/smartx.go b/storage/fusionstorage/smartx/smartx.go index b2cad7bc..be625c23 100644 --- a/storage/fusionstorage/smartx/smartx.go +++ b/storage/fusionstorage/smartx/smartx.go @@ -24,8 +24,8 @@ import ( "fmt" "time" - "huawei-csi-driver/storage/fusionstorage/client" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/fusionstorage/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) var ( diff --git a/storage/fusionstorage/utils/utils.go b/storage/fusionstorage/utils/utils.go index b69f824d..ffd50eac 100644 --- a/storage/fusionstorage/utils/utils.go +++ b/storage/fusionstorage/utils/utils.go @@ -22,7 +22,7 @@ import ( "encoding/json" "fmt" - "huawei-csi-driver/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" ) const ( @@ -88,11 +88,12 @@ func ExtractStorageQuotaParameters(ctx context.Context, storageQuotaConfig strin // CheckErrorCode used to check Response // Response data struct // Response -// data -// ... -// result: -// code: 0 -// description: "" +// +// data +// ... +// result: +// code: 0 +// description: "" func CheckErrorCode(response map[string]interface{}) error { result, ok := response["result"].(map[string]interface{}) diff --git a/storage/fusionstorage/volume/nas.go b/storage/fusionstorage/volume/nas.go index 8b6ce26e..98629443 100644 --- a/storage/fusionstorage/volume/nas.go +++ b/storage/fusionstorage/volume/nas.go @@ -27,14 +27,15 @@ import ( "strings" "time" - pkgUtils "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/storage/fusionstorage/client" - "huawei-csi-driver/storage/fusionstorage/smartx" - "huawei-csi-driver/storage/fusionstorage/types" - fsUtils "huawei-csi-driver/storage/fusionstorage/utils" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/flow" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/fusionstorage/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/fusionstorage/smartx" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/fusionstorage/types" + fsUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/fusionstorage/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/flow" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( @@ -305,17 +306,19 @@ func (p *NAS) Create(ctx context.Context, params map[string]interface{}) (utils. return nil, err } - volObj := p.prepareVolObj(ctx, params) - return volObj, nil + return p.prepareVolObj(ctx, params) } -func (p *NAS) prepareVolObj(ctx context.Context, params map[string]interface{}) utils.Volume { - volName, isStr := params["name"].(string) - if !isStr { - // Not expecting this error to happen - log.AddContext(ctx).Warningf("Expecting string for volume name, received type %T", params["name"]) +func (p *NAS) prepareVolObj(ctx context.Context, params map[string]interface{}) (utils.Volume, error) { + volName, ok := params["name"].(string) + if !ok { + return nil, utils.Errorf(ctx, "expecting string for volume name, received type %T", params["name"]) } - return utils.NewVolume(volName) + + vol := utils.NewVolume(volName) + capacity := utils.GetValueOrFallback(params, "capacity", int64(0)) + vol.SetSize(utils.TransK8SCapacity(capacity, constants.FusionFileCapacityUnit)) + return vol, nil } func (p *NAS) createFS(ctx context.Context, @@ -679,7 +682,7 @@ func (p *NAS) setSize(ctx context.Context, fsName string, quota map[string]inter if hardSize, exits := quota["space_hard_quota"].(float64); exits && hardSize != quotaInvalidValue { capacity = int64(hardSize) } else if softSize, exits := quota["space_soft_quota"].(float64); exits && softSize != quotaInvalidValue { - capacity = int64(hardSize) + capacity = int64(softSize) } else { msg := fmt.Sprintf("Quota %v does not contain space_hard_quota or space_soft_quota.", quota) log.AddContext(ctx).Errorln(msg) diff --git a/storage/fusionstorage/volume/nas_test.go b/storage/fusionstorage/volume/nas_test.go index 2b845aaf..aacaa556 100644 --- a/storage/fusionstorage/volume/nas_test.go +++ b/storage/fusionstorage/volume/nas_test.go @@ -25,9 +25,9 @@ import ( "github.com/agiledragon/gomonkey/v2" "github.com/stretchr/testify/require" - "huawei-csi-driver/storage/fusionstorage/client" - "huawei-csi-driver/storage/fusionstorage/types" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/fusionstorage/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/fusionstorage/types" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/storage/fusionstorage/volume/san.go b/storage/fusionstorage/volume/san.go index 1dc8d373..07af8604 100644 --- a/storage/fusionstorage/volume/san.go +++ b/storage/fusionstorage/volume/san.go @@ -22,12 +22,13 @@ import ( "fmt" "strconv" - pkgUtils "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/storage/fusionstorage/client" - "huawei-csi-driver/storage/fusionstorage/smartx" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/flow" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/fusionstorage/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/fusionstorage/smartx" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/flow" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( @@ -116,21 +117,25 @@ func (p *SAN) Create(ctx context.Context, params map[string]interface{}) (utils. taskflow.Revert() return nil, err } - volObj := p.prepareVolObj(ctx, params, res) - return volObj, nil + + return p.prepareVolObj(ctx, params, res) } -func (p *SAN) prepareVolObj(ctx context.Context, params, res map[string]interface{}) utils.Volume { - volName, isStr := params["name"].(string) - if !isStr { - // Not expecting this error to happen - log.AddContext(ctx).Warningf("Expecting string for volume name, received type %T", params["name"]) +func (p *SAN) prepareVolObj(ctx context.Context, params, res map[string]interface{}) (utils.Volume, error) { + volName, ok := params["name"].(string) + if !ok { + return nil, utils.Errorf(ctx, "expecting string for volume name, received type %T", params["name"]) } + volObj := utils.NewVolume(volName) if lunWWN, ok := res["lunWWN"].(string); ok { volObj.SetLunWWN(lunWWN) } - return volObj + + capacity := utils.GetValueOrFallback(params, "capacity", int64(0)) + volObj.SetSize(utils.TransK8SCapacity(capacity, constants.FusionAllocUnitBytes)) + + return volObj, nil } func (p *SAN) createLun(ctx context.Context, @@ -314,7 +319,7 @@ func (p *SAN) Query(ctx context.Context, name string) (utils.Volume, error) { // set the size, need to trans MiB to Bytes capacity := int64(vol["volSize"].(float64)) - volObj.SetSize(utils.TransK8SCapacity(capacity, 1024*1024)) + volObj.SetSize(utils.TransK8SCapacity(capacity, constants.FusionAllocUnitBytes)) return volObj, nil } diff --git a/storage/oceanstor/client/client.go b/storage/oceanstor/client/client.go deleted file mode 100644 index 2514bb32..00000000 --- a/storage/oceanstor/client/client.go +++ /dev/null @@ -1,1138 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2020-2024. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package client - -import ( - "bytes" - "context" - "crypto/tls" - "encoding/json" - "errors" - "fmt" - "io" - "io/ioutil" - "net/http" - "net/http/cookiejar" - "regexp" - "slices" - "strconv" - "sync" - "sync/atomic" - "time" - - "huawei-csi-driver/pkg/constants" - pkgUtils "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" -) - -const ( - // DefaultParallelCount defines default parallel count - DefaultParallelCount int = 50 - - // MaxParallelCount defines max parallel count - MaxParallelCount int = 1000 - - // MinParallelCount defines min parallel count - MinParallelCount int = 20 - - // GetInfoWaitInternal defines wait internal of getting info - GetInfoWaitInternal = 10 - - // QueryCountPerBatch defines query count for each circle of batch operation - QueryCountPerBatch int = 100 - - // IPLockErrorCode defines error code of ip lock - IPLockErrorCode = 1077949071 - - // UserOffline defines error code of user off line - UserOffline = 1077949069 - - // UserUnauthorized defines error code of user unauthorized - UserUnauthorized = -401 - - // SuccessCode defines error code of success - SuccessCode = int64(0) - - // UrlNotFound defines error msg of url not found - UrlNotFound = "404_NotFound" -) - -const ( - description string = "Created from huawei-csi for Kubernetes" - defaultVStore string = "System_vStore" - defaultVStoreID string = "0" - defaultHttpTimeout = 60 * time.Second -) - -var ( - // WrongPasswordErrorCodes user or password is incorrect - WrongPasswordErrorCodes = []int64{1077987870, 1077949081, 1077949061} - // AccountBeenLocked account been locked - AccountBeenLocked = []int64{1077949070, 1077987871} -) - -// BaseClientInterface defines interfaces for base client operations -type BaseClientInterface interface { - ApplicationType - Clone - FC - Filesystem - FSSnapshot - Host - HyperMetro - Iscsi - Lun - LunCopy - LunSnapshot - Mapping - Qos - Replication - RoCE - System - VStore - DTree - OceanStorQuota - LIF - - Call(ctx context.Context, method string, url string, data map[string]interface{}) (Response, error) - SafeCall(ctx context.Context, method string, url string, data map[string]interface{}) (Response, error) - BaseCall(ctx context.Context, method string, url string, data map[string]interface{}) (Response, error) - SafeBaseCall(ctx context.Context, method string, url string, data map[string]interface{}) (Response, error) - Get(ctx context.Context, url string, data map[string]interface{}) (Response, error) - Post(ctx context.Context, url string, data map[string]interface{}) (Response, error) - Put(ctx context.Context, url string, data map[string]interface{}) (Response, error) - Delete(ctx context.Context, url string, data map[string]interface{}) (Response, error) - SafeDelete(ctx context.Context, url string, data map[string]interface{}) (Response, error) - GetRequest(ctx context.Context, method string, url string, data map[string]interface{}) (*http.Request, error) - DuplicateClient() *BaseClient - Login(ctx context.Context) error - Logout(ctx context.Context) - ReLogin(ctx context.Context) error - GetBackendID() string -} - -var ( - filterLog = map[string]map[string]bool{ - "POST": { - "/xx/sessions": true, - }, - } - - filterLogRegex = map[string][]string{ - "GET": { - `/vstore_pair\?filter=ID`, - `/FsHyperMetroDomain\?RUNNINGSTATUS=0`, - `/remote_device`, - }, - } - - debugLog = map[string]map[string]bool{ - "GET": { - "/license/feature": true, - "/nfsservice": true, - "/storagepool": true, - `/vstore_pair\?RETYPE=1`: true, - `/vstore?filter=NAME`: true, - `/container_pv`: true, - `/system`: true, - }, - } - - debugLogRegex = map[string][]string{ - "GET": { - `/vstore_pair\?RETYPE=1`, - `/vstore\?filter=NAME`, - `/system`, - }, - } - - // RequestSemaphore provides semaphore of client - RequestSemaphore *utils.Semaphore -) - -func isFilterLog(method, url string) bool { - if filter, exist := filterLog[method]; exist && filter[url] { - return true - } - - if filter, exist := filterLogRegex[method]; exist { - for _, k := range filter { - match, err := regexp.MatchString(k, url) - if err == nil && match { - return true - } - } - } - - return false -} - -// BaseClient implements BaseClientInterface -type BaseClient struct { - Client HTTP - - Url string - Urls []string - - User string - SecretNamespace string - SecretName string - VStoreName string - VStoreID string - StorageVersion string - BackendID string - Storage string - CurrentSiteWwn string - CurrentLifWwn string - LastLif string - Product string - DeviceId string - Token string - - SystemInfoRefreshing uint32 - ReLoginMutex sync.Mutex -} - -// HTTP defines for http request process -type HTTP interface { - Do(req *http.Request) (*http.Response, error) -} - -func newHTTPClientByBackendID(ctx context.Context, backendID string) (HTTP, error) { - var defaultUseCert bool - client := &http.Client{ - Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: !defaultUseCert}}, - Timeout: defaultHttpTimeout, - } - - jar, err := cookiejar.New(nil) - if err != nil { - log.AddContext(ctx).Errorf("create jar failed, error: %v", err) - return client, err - } - - useCert, certMeta, err := pkgUtils.GetCertSecretFromBackendID(ctx, backendID) - if err != nil { - log.AddContext(ctx).Errorf("get cert secret from backend [%v] failed, error: %v", backendID, err) - return client, err - } - - useCert, certPool, err := pkgUtils.GetCertPool(ctx, useCert, certMeta) - if err != nil { - log.AddContext(ctx).Errorf("get cert pool failed, error: %v", err) - return client, err - } - - client.Transport = &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: !useCert, RootCAs: certPool}} - client.Jar = jar - return client, nil -} - -func newHTTPClientByCertMeta(ctx context.Context, useCert bool, certMeta string) (HTTP, error) { - jar, err := cookiejar.New(nil) - if err != nil { - log.AddContext(ctx).Errorf("create jar failed, error: %v", err) - return nil, err - } - - useCert, certPool, err := pkgUtils.GetCertPool(ctx, useCert, certMeta) - if err != nil { - return nil, err - } - - return &http.Client{ - Transport: &http.Transport{ - TLSClientConfig: &tls.Config{InsecureSkipVerify: !useCert, RootCAs: certPool}, - }, - Jar: jar, - Timeout: defaultHttpTimeout, - }, nil -} - -// Response defines response of request -type Response struct { - Error map[string]interface{} `json:"error"` - Data interface{} `json:"data,omitempty"` -} - -// AssertErrorCode asserts if error code represents success -func (resp *Response) AssertErrorCode() error { - val, exists := resp.Error["code"] - if !exists { - return fmt.Errorf("error code not exists, data: %+v", *resp) - } - - code, ok := val.(float64) - if !ok { - return fmt.Errorf("code is not float64, data: %+v", *resp) - } - - if int64(code) != SuccessCode { - return fmt.Errorf("error code is not success, data: %+v", *resp) - } - - return nil -} - -// GetData converts interface{} type data to specific type -func (resp *Response) GetData(val any) error { - data, err := json.Marshal(resp.Data) - if err != nil { - return fmt.Errorf("failed to marshal data, error %w", err) - } - - err = json.Unmarshal(data, &val) - if err != nil { - return fmt.Errorf("failed to unmarshal data as %T, error: %w", val, err) - } - - return nil -} - -// NewClientConfig stores the information needed to create a new OceanStor client -type NewClientConfig struct { - Urls []string - User string - SecretName string - SecretNamespace string - VstoreName string - ParallelNum string - BackendID string - UseCert bool - CertSecretMeta string - Storage string - Name string -} - -// NewClient inits a new base client -func NewClient(ctx context.Context, param *NewClientConfig) (*BaseClient, error) { - var err error - var parallelCount int - - if len(param.ParallelNum) > 0 { - parallelCount, err = strconv.Atoi(param.ParallelNum) - if err != nil || parallelCount > MaxParallelCount || parallelCount < MinParallelCount { - log.AddContext(ctx).Warningf("The config parallelNum %d is invalid, set it to the default value %d", - parallelCount, DefaultParallelCount) - parallelCount = DefaultParallelCount - } - } else { - parallelCount = DefaultParallelCount - } - - log.AddContext(ctx).Infof("Init parallel count is %d", parallelCount) - RequestSemaphore = utils.NewSemaphore(parallelCount) - - httpClient, err := newHTTPClientByCertMeta(ctx, param.UseCert, param.CertSecretMeta) - if err != nil { - log.AddContext(ctx).Errorf("new http client by cert meta failed, err is %v", err) - return nil, err - } - - return &BaseClient{ - Urls: param.Urls, - User: param.User, - Storage: param.Storage, - SecretName: param.SecretName, - SecretNamespace: param.SecretNamespace, - VStoreName: param.VstoreName, - Client: httpClient, - BackendID: param.BackendID, - }, nil -} - -// Call provides call for restful request -func (cli *BaseClient) Call(ctx context.Context, - method string, url string, - data map[string]interface{}) (Response, error) { - var r Response - var err error - - r, err = cli.BaseCall(ctx, method, url, data) - if !needReLogin(r, err) { - return r, err - } - - // Current connection fails, try to relogin to other Urls if exist, - // if relogin success, resend the request again. - log.AddContext(ctx).Infof("Try to relogin and resend request method: %s, Url: %s", method, url) - err = cli.ReLogin(ctx) - if err != nil { - return r, err - } - - // If the logical port changes from storage A to storage B, the system information must be updated. - if err = cli.SetSystemInfo(ctx); err != nil { - log.AddContext(ctx).Errorf("after relogin, can't get system info, error: %v", err) - return r, err - } - - return cli.BaseCall(ctx, method, url, data) -} - -// SafeCall provides call for restful request -func (cli *BaseClient) SafeCall(ctx context.Context, - method string, url string, - data map[string]interface{}) (Response, error) { - var r Response - var err error - - r, err = cli.SafeBaseCall(ctx, method, url, data) - if !needReLogin(r, err) { - return r, err - } - - // Current connection fails, try to relogin to other Urls if exist, - // if relogin success, resend the request again. - log.AddContext(ctx).Infof("Try to re-login and resend request method: %s, Url: %s", method, url) - err = cli.ReLogin(ctx) - if err != nil { - return r, err - } - - // If the logical port changes from storage A to storage B, the system information must be updated. - if err = cli.SetSystemInfo(ctx); err != nil { - log.AddContext(ctx).Errorf("after re-login, can't get system info, error: %v", err) - return r, err - } - - return cli.SafeBaseCall(ctx, method, url, data) -} - -// needReLogin determine if it is necessary to log in to the storage again -func needReLogin(r Response, err error) bool { - var unconnected, unauthorized, offline bool - if err != nil && err.Error() == "unconnected" { - unconnected = true - } - - if r.Error != nil { - if code, ok := r.Error["code"].(float64); ok { - unauthorized = int64(code) == UserUnauthorized - offline = int64(code) == UserOffline - } - } - return unconnected || unauthorized || offline -} - -// GetRequest return the request info -func (cli *BaseClient) GetRequest(ctx context.Context, - method string, url string, - data map[string]interface{}) (*http.Request, error) { - var req *http.Request - var err error - - reqUrl := cli.Url - if cli.DeviceId != "" { - reqUrl += "/" + cli.DeviceId - } - reqUrl += url - - var reqBody io.Reader - - if data != nil { - reqBytes, err := json.Marshal(data) - if err != nil { - log.AddContext(ctx).Errorf("json.Marshal data %v error: %v", maskRequestData(data), err) - return req, err - } - reqBody = bytes.NewReader(reqBytes) - } - - req, err = http.NewRequest(method, reqUrl, reqBody) - if err != nil { - log.AddContext(ctx).Errorf("Construct http request error: %s", err.Error()) - return req, err - } - - req.Header.Set("Connection", "keep-alive") - req.Header.Set("Content-Type", "application/json") - - if cli.Token != "" { - req.Header.Set("iBaseToken", cli.Token) - } - - return req, nil -} - -// BaseCall provides base call for request -func (cli *BaseClient) BaseCall(ctx context.Context, - method string, - url string, - data map[string]interface{}) (Response, error) { - var r Response - var req *http.Request - var err error - - if cli.Client == nil { - errMsg := "http client is nil" - log.AddContext(ctx).Errorf("Failed to send request method: %s, url: %s, error: %s", method, url, errMsg) - return r, errors.New(errMsg) - } - - reqUrl := cli.Url - reqUrl += url - - if url != "/xx/sessions" && url != "/sessions" { - cli.ReLoginMutex.Lock() - req, err = cli.GetRequest(ctx, method, url, data) - cli.ReLoginMutex.Unlock() - } else { - req, err = cli.GetRequest(ctx, method, url, data) - } - - if err != nil { - return r, err - } - - log.FilteredLog(ctx, isFilterLog(method, url), utils.IsDebugLog(method, url, debugLog, debugLogRegex), - fmt.Sprintf("Request method: %s, Url: %s, body: %v", method, req.URL, data)) - - if RequestSemaphore == nil { - return r, errors.New("request semaphore is nil") - } - - RequestSemaphore.Acquire() - defer RequestSemaphore.Release() - - resp, err := cli.Client.Do(req) - if err != nil { - log.AddContext(ctx).Errorf("Send request method: %s, Url: %s, error: %v", method, req.URL, err) - return r, errors.New("unconnected") - } - defer resp.Body.Close() - - body, err := ioutil.ReadAll(resp.Body) - if err != nil { - log.AddContext(ctx).Errorf("Read response data error: %v", err) - return r, err - } - - log.FilteredLog(ctx, isFilterLog(method, url), utils.IsDebugLog(method, url, debugLog, debugLogRegex), - fmt.Sprintf("Response method: %s, Url: %s, body: %s", method, req.URL, body)) - - err = json.Unmarshal(body, &r) - if err != nil { - log.AddContext(ctx).Errorf("json.Unmarshal data %s error: %v", body, err) - return r, err - } - - return r, nil -} - -// SafeBaseCall provides base call for request -func (cli *BaseClient) SafeBaseCall(ctx context.Context, - method string, - url string, - data map[string]interface{}) (Response, error) { - var req *http.Request - var err error - - if cli.Client == nil { - return Response{}, fmt.Errorf("failed to send request method: %s, url: %s,"+ - " cause by client not init", method, url) - } - - reqUrl := cli.Url - reqUrl += url - - if url != "/xx/sessions" && url != "/sessions" { - cli.ReLoginMutex.Lock() - req, err = cli.GetRequest(ctx, method, url, data) - cli.ReLoginMutex.Unlock() - } else { - req, err = cli.GetRequest(ctx, method, url, data) - } - - if err != nil || req == nil { - return Response{}, fmt.Errorf("get request failed, error: %w", err) - } - - log.FilteredLog(ctx, isFilterLog(method, url), utils.IsDebugLog(method, url, debugLog, debugLogRegex), - fmt.Sprintf("Request method: %s, Url: %s, body: %v", method, req.URL, data)) - - if RequestSemaphore != nil { - RequestSemaphore.Acquire() - defer RequestSemaphore.Release() - } - - return cli.safeDoCall(ctx, method, url, req) -} - -func (cli *BaseClient) safeDoCall(ctx context.Context, method string, url string, req *http.Request) (Response, error) { - // check whether the logical port is changed from A to B before invoking. - // The possible cause is that other invoking operations are performed for re-login. - isNotSessionUrl := url != "/xx/sessions" && url != "/sessions" - if isNotSessionUrl && cli.CurrentLifWwn != "" { - if cli.systemInfoRefreshing() { - return Response{}, fmt.Errorf("querying lif and system information... Please wait") - } - - if cli.CurrentLifWwn != cli.CurrentSiteWwn { - currentPort := cli.GetCurrentLif(ctx) - log.AddContext(ctx).Errorf("current logical port [%s] is not running on own site, "+ - "currentLifWwn: %s, currentSiteWwn: %s", currentPort, cli.CurrentLifWwn, cli.CurrentSiteWwn) - return Response{}, fmt.Errorf("current logical port [%s] is not running on own site", currentPort) - } - } - - resp, err := cli.Client.Do(req) - if err != nil { - log.AddContext(ctx).Errorf("Send request method: %s, Url: %s, error: %v", method, req.URL, err) - return Response{}, errors.New("unconnected") - } - - defer func() { - if err := resp.Body.Close(); err != nil { - log.AddContext(ctx).Infof("read close resp failed, error %v", err) - } - }() - - body, err := io.ReadAll(resp.Body) - if err != nil { - return Response{}, fmt.Errorf("read response data error: %w", err) - } - - log.FilteredLog(ctx, isFilterLog(method, url), utils.IsDebugLog(method, url, debugLog, debugLogRegex), - fmt.Sprintf("Response method: %s, Url: %s, body: %s", method, req.URL, body)) - - var r Response - err = json.Unmarshal(body, &r) - if err != nil { - return Response{}, fmt.Errorf("json.Unmarshal data %s error: %w", body, err) - } - - return r, nil -} - -// Get provides http request of GET method -func (cli *BaseClient) Get(ctx context.Context, url string, data map[string]interface{}) (Response, error) { - return cli.Call(ctx, "GET", url, data) -} - -// Post provides http request of POST method -func (cli *BaseClient) Post(ctx context.Context, url string, data map[string]interface{}) (Response, error) { - return cli.Call(ctx, "POST", url, data) -} - -// Put provides http request of PUT method -func (cli *BaseClient) Put(ctx context.Context, url string, data map[string]interface{}) (Response, error) { - return cli.Call(ctx, "PUT", url, data) -} - -// Delete provides http request of DELETE method -func (cli *BaseClient) Delete(ctx context.Context, url string, data map[string]interface{}) (Response, error) { - return cli.Call(ctx, "DELETE", url, data) -} - -// SafeDelete provides http request of DELETE method -func (cli *BaseClient) SafeDelete(ctx context.Context, url string, data map[string]interface{}) (Response, error) { - return cli.SafeCall(ctx, "DELETE", url, data) -} - -// DuplicateClient clone a base client from origin client -func (cli *BaseClient) DuplicateClient() *BaseClient { - dup := *cli - - dup.Urls = make([]string, len(cli.Urls)) - copy(dup.Urls, cli.Urls) - - dup.Client = nil - - return &dup -} - -// ValidateLogin validates the login info -func (cli *BaseClient) ValidateLogin(ctx context.Context) error { - var resp Response - var err error - - password, err := utils.GetPasswordFromSecret(ctx, cli.SecretName, cli.SecretNamespace) - if err != nil { - return err - } - - data := map[string]interface{}{ - "username": cli.User, - "password": password, - "scope": "0", - } - - if len(cli.VStoreName) > 0 && cli.VStoreName != defaultVStore { - data["vstorename"] = cli.VStoreName - } - - cli.DeviceId = "" - cli.Token = "" - for i, url := range cli.Urls { - cli.Url = url + "/deviceManager/rest" - - log.AddContext(ctx).Infof("Try to login %s", cli.Url) - resp, err = cli.BaseCall(ctx, "POST", "/xx/sessions", data) - if err == nil { - /* Sort the login Url to the last slot of san addresses, so that - if this connection error, next time will try other Url first. */ - cli.Urls[i], cli.Urls[len(cli.Urls)-1] = cli.Urls[len(cli.Urls)-1], cli.Urls[i] - break - } else if err.Error() != "unconnected" { - log.AddContext(ctx).Errorf("Login %s error", cli.Url) - break - } - - log.AddContext(ctx).Warningf("Login %s error due to connection failure, gonna try another Url", - cli.Url) - } - - if err != nil { - return err - } - - code := int64(resp.Error["code"].(float64)) - if code != 0 { - return fmt.Errorf("validate login %s error: %+v", cli.Url, resp) - } - - cli.setDeviceIdFromRespData(ctx, resp) - - log.AddContext(ctx).Infof("Validate login %s success", cli.Url) - return nil -} - -func (cli *BaseClient) setDeviceIdFromRespData(ctx context.Context, resp Response) { - respData, ok := resp.Data.(map[string]interface{}) - if !ok { - log.AddContext(ctx).Warningf("convert response data to map[string]interface{} failed, data type: [%T]", - resp.Data) - } - - cli.DeviceId, ok = respData["deviceid"].(string) - if !ok { - log.AddContext(ctx).Warningf("not found deviceId, response data is: [%v]", respData["deviceid"]) - } - - if _, exists := respData["iBaseToken"]; !exists { - log.AddContext(ctx).Warningf("not found iBaseToken, response data is: [%v]", resp.Data) - } - cli.Token, ok = respData["iBaseToken"].(string) - if !ok { - log.AddContext(ctx).Warningf("convert iBaseToken to string error, data type: [%T]", - respData["iBaseToken"]) - } -} - -// Login login and set data from response -func (cli *BaseClient) Login(ctx context.Context) error { - var resp Response - var err error - - cli.Client, err = newHTTPClientByBackendID(ctx, cli.BackendID) - if err != nil { - log.AddContext(ctx).Errorf("new http client by backend %s failed, err is %v", cli.BackendID, err) - return err - } - - data, err := cli.getRequestParams(ctx, cli.BackendID) - if err != nil { - return err - } - - cli.DeviceId = "" - cli.Token = "" - for i, url := range cli.Urls { - cli.Url = url + "/deviceManager/rest" - - log.AddContext(ctx).Infof("Try to login %s", cli.Url) - resp, err = cli.BaseCall(ctx, "POST", "/xx/sessions", data) - if err == nil { - /* Sort the login Url to the last slot of san addresses, so that - if this connection error, next time will try other Url first. */ - cli.Urls[i], cli.Urls[len(cli.Urls)-1] = cli.Urls[len(cli.Urls)-1], cli.Urls[i] - break - } else if err.Error() != "unconnected" { - log.AddContext(ctx).Errorf("Login %s error", cli.Url) - break - } - - log.AddContext(ctx).Warningf("Login %s error due to connection failure, gonna try another Url", cli.Url) - } - - if err != nil { - return err - } - - errCode, _ := resp.Error["code"].(float64) - if code := int64(errCode); code != 0 { - msg := fmt.Sprintf("Login %s error: %+v", cli.Url, resp) - if utils.Contains(WrongPasswordErrorCodes, code) || utils.Contains(AccountBeenLocked, code) || - code == IPLockErrorCode { - if err := pkgUtils.SetStorageBackendContentOnlineStatus(ctx, cli.BackendID, false); err != nil { - msg = msg + fmt.Sprintf("\nSetStorageBackendContentOffline [%s] failed. error: %v", cli.BackendID, err) - } - } - return errors.New(msg) - } - - if err = cli.setDataFromRespData(ctx, resp); err != nil { - cli.Logout(ctx) - setErr := pkgUtils.SetStorageBackendContentOnlineStatus(ctx, cli.BackendID, false) - if setErr != nil { - log.AddContext(ctx).Errorf("SetStorageBackendContentOffline [%s] failed. error: %v", cli.BackendID, setErr) - } - return err - } - return nil -} - -func (cli *BaseClient) setDataFromRespData(ctx context.Context, resp Response) error { - respData, ok := resp.Data.(map[string]interface{}) - if !ok { - return pkgUtils.Errorln(ctx, fmt.Sprintf("convert resp.Data to map[string]interface{} failed,"+ - " data type: [%T]", resp.Data)) - } - cli.DeviceId, ok = respData["deviceid"].(string) - if !ok { - return pkgUtils.Errorln(ctx, fmt.Sprintf("convert respData[\"deviceid\"]: [%v] to string failed", - respData["deviceid"])) - } - cli.Token, ok = respData["iBaseToken"].(string) - if !ok { - return pkgUtils.Errorln(ctx, fmt.Sprintf("convert respData[\"iBaseToken\"]: [%T] to string failed", - respData["iBaseToken"])) - } - - vStoreName, exist := respData["vstoreName"].(string) - vStoreID, idExist := respData["vstoreId"].(string) - if !exist && !idExist { - log.AddContext(ctx).Infof("storage client login response vstoreName is empty, set it to default %s", - defaultVStore) - cli.VStoreName = defaultVStore - } else if exist { - cli.VStoreName = vStoreName - } - - if !idExist { - log.AddContext(ctx).Infof("storage client login response vstoreID is empty, set it to default %s", - defaultVStoreID) - cli.VStoreID = defaultVStoreID - } else { - cli.VStoreID = vStoreID - } - - log.AddContext(ctx).Infof("Login %s success", cli.Url) - return nil -} - -// Logout logout -func (cli *BaseClient) Logout(ctx context.Context) { - resp, err := cli.BaseCall(ctx, "DELETE", "/sessions", nil) - if err != nil { - log.AddContext(ctx).Warningf("Logout %s error: %v", cli.Url, err) - return - } - - code := int64(resp.Error["code"].(float64)) - if code != 0 { - log.AddContext(ctx).Warningf("Logout %s error: %d", cli.Url, code) - return - } - - log.AddContext(ctx).Infof("Logout %s success", cli.Url) -} - -// ReLogin logout and login again -func (cli *BaseClient) ReLogin(ctx context.Context) error { - oldToken := cli.Token - - cli.ReLoginMutex.Lock() - defer cli.ReLoginMutex.Unlock() - - if cli.Token != "" && oldToken != cli.Token { - // Coming here indicates other thread had already done relogin, so no need to relogin again - return nil - } else if cli.Token != "" { - cli.Logout(ctx) - } - - err := cli.Login(ctx) - if err != nil { - log.AddContext(ctx).Errorf("Try to relogin error: %v", err) - return err - } - - return nil -} - -// GetBackendID get backend id of client -func (cli *BaseClient) GetBackendID() string { - return cli.BackendID -} - -func (cli *BaseClient) getResponseDataMap(ctx context.Context, data interface{}) (map[string]interface{}, error) { - respData, ok := data.(map[string]interface{}) - if !ok { - return nil, utils.Errorf(ctx, "the response data is not a map[string]interface{}") - } - return respData, nil -} - -func (cli *BaseClient) getResponseDataList(ctx context.Context, data interface{}) ([]interface{}, error) { - respData, ok := data.([]interface{}) - if !ok { - return nil, utils.Errorf(ctx, "the response data is not a []interface{}") - } - return respData, nil -} - -func (cli *BaseClient) getCountFromResponse(ctx context.Context, data interface{}) (int64, error) { - respData, err := cli.getResponseDataMap(ctx, data) - if err != nil { - return 0, err - } - countStr, ok := respData["COUNT"].(string) - if !ok { - return 0, utils.Errorf(ctx, "The COUNT is not in respData %v", respData) - } - count, err := strconv.ParseInt(countStr, constants.DefaultIntBase, constants.DefaultIntBitSize) - if err != nil { - return 0, err - } - return count, nil -} - -func (cli *BaseClient) getSystemUTCTime(ctx context.Context) (int64, error) { - resp, err := cli.Get(ctx, "/system_utc_time", nil) - if err != nil { - return 0, err - } - - code := int64(resp.Error["code"].(float64)) - if code != 0 { - return 0, utils.Errorf(ctx, "get system UTC time error: %d", code) - } - - if resp.Data == nil { - return 0, utils.Errorf(ctx, "can not get the system UTC time") - } - - respData, err := cli.getResponseDataMap(ctx, resp.Data) - if err != nil { - return 0, err - } - - utcTime, ok := respData["CMO_SYS_UTC_TIME"].(string) - if !ok { - return 0, utils.Errorf(ctx, "The CMO_SYS_UTC_TIME is not in respData %v", respData) - } - - time, err := strconv.ParseInt(utcTime, constants.DefaultIntBase, constants.DefaultIntBitSize) - if err != nil { - return 0, err - } - return time, nil -} - -func (cli *BaseClient) getObjByvStoreName(objList []interface{}) map[string]interface{} { - for _, data := range objList { - obj, ok := data.(map[string]interface{}) - if !ok || obj == nil { - continue - } - - vStoreName, ok := obj["vstoreName"].(string) - if !ok { - vStoreName = defaultVStore - } - - if vStoreName == cli.GetvStoreName() { - return obj - } - continue - - } - return nil -} - -func (cli *BaseClient) getObj(ctx context.Context, url string, start, end int, filterLog bool) ( - []map[string]interface{}, error) { - objUrl := fmt.Sprintf("%s?range=[%d-%d]", url, start, end) - resp, err := cli.Get(ctx, objUrl, nil) - if err != nil { - return nil, err - } - - code := int64(resp.Error["code"].(float64)) - if code != 0 { - return nil, fmt.Errorf("get batch obj list error: %d", code) - } - - if !filterLog { - log.AddContext(ctx).Infoln("There is no obj in storage.") - } - - if resp.Data == nil { - return nil, nil - } - - var objList []map[string]interface{} - respData, ok := resp.Data.([]interface{}) - if !ok { - return nil, errors.New("convert resp.Data to []interface{} failed") - } - for _, i := range respData { - device, ok := i.(map[string]interface{}) - if !ok { - log.AddContext(ctx).Warningf("convert resp.Data to map[string]interface{} failed") - continue - } - objList = append(objList, device) - } - return objList, nil -} - -func (cli *BaseClient) getBatchObjs(ctx context.Context, url string, filterLog bool) ([]map[string]interface{}, error) { - rangeStart := 0 - var objList []map[string]interface{} - for true { - rangeEnd := rangeStart + QueryCountPerBatch - objs, err := cli.getObj(ctx, url, rangeStart, rangeEnd, filterLog) - if err != nil { - return nil, err - } - - if objs == nil { - break - } - - objList = append(objList, objs...) - if len(objs) < QueryCountPerBatch { - break - } - rangeStart = rangeEnd + QueryCountPerBatch - } - return objList, nil -} - -func (cli *BaseClient) getRequestParams(ctx context.Context, backendID string) (map[string]interface{}, error) { - password, err := pkgUtils.GetPasswordFromBackendID(ctx, backendID) - if err != nil { - return nil, err - } - - data := map[string]interface{}{ - "username": cli.User, - "password": password, - "scope": "0", - } - - if len(cli.VStoreName) > 0 && cli.VStoreName != defaultVStore { - data["vstorename"] = cli.VStoreName - } - - return data, err -} - -// SetSystemInfo set system info -// the mutex lock is required for re-login. Therefore, the internal query of the login interface cannot be performed. -func (cli *BaseClient) SetSystemInfo(ctx context.Context) error { - log.AddContext(ctx).Infof("set backend [%s] system info is refreshing", cli.BackendID) - atomic.StoreUint32(&cli.SystemInfoRefreshing, 1) - defer func() { - log.AddContext(ctx).Infof("set backend [%s] system info are refreshed", cli.BackendID) - atomic.StoreUint32(&cli.SystemInfoRefreshing, 0) - }() - - err := cli.setBaseInfo(ctx) - if err != nil { - return err - } - cli.setLifInfo(ctx) - - log.AddContext(ctx).Infof("backend type [%s], backend [%s], storage product [%s], storage version [%s], "+ - "current site wwn [%s], current lif [%s], current lif wwn [%s]", cli.Storage, - cli.BackendID, cli.Product, cli.StorageVersion, cli.CurrentSiteWwn, cli.GetCurrentLif(ctx), cli.CurrentLifWwn) - return nil -} - -func (cli *BaseClient) setBaseInfo(ctx context.Context) error { - system, err := cli.GetSystem(ctx) - if err != nil { - log.AddContext(ctx).Errorf("get system info failed, error: %v", err) - return err - } - - cli.Product, err = utils.GetProductVersion(system) - if err != nil { - log.AddContext(ctx).Errorf("get product version failed, error: %v", err) - return err - } - - storagePointVersion, ok := system["pointRelease"].(string) - if ok { - cli.StorageVersion = storagePointVersion - } - - currentSiteWwn, ok := system["wwn"].(string) - if ok { - cli.CurrentSiteWwn = currentSiteWwn - } - - return nil -} - -func (cli *BaseClient) setLifInfo(ctx context.Context) { - if cli.Product != constants.OceanStorDoradoV6 || cli.Storage != constants.OceanStorNas { - log.AddContext(ctx).Infof("backend type [%s], name [%s], version [%s], not need query lif", cli.Storage, - cli.BackendID, cli.StorageVersion) - return - } - - currentLif := cli.GetCurrentLif(ctx) - if cli.LastLif == currentLif { - log.AddContext(ctx).Infof("backend [%s] last lif [%s], current lif [%s], not change", - cli.BackendID, cli.LastLif, currentLif) - return - } - - port, err := cli.GetLogicPort(ctx, currentLif) - if err != nil { - log.AddContext(ctx).Errorf("get logic port failed, error: %v", err) - return - } - - cli.LastLif = currentLif - cli.CurrentLifWwn = port.HomeSiteWwn -} - -func (cli *BaseClient) systemInfoRefreshing() bool { - return atomic.LoadUint32(&cli.SystemInfoRefreshing) == 1 -} - -func maskRequestData(data map[string]any) map[string]any { - sensitiveKey := []string{"user", "password", "iqn", "tgt", "tgtname", "initiatorname"} - - maskedData := make(map[string]any) - for k, v := range data { - if slices.Contains(sensitiveKey, k) { - maskedData[k] = "***" - } else { - maskedData[k] = v - } - } - - return maskedData -} diff --git a/storage/oceanstor/volume/creator/snapshot_fs.go b/storage/oceanstor/volume/creator/snapshot_fs.go deleted file mode 100644 index de06d5a5..00000000 --- a/storage/oceanstor/volume/creator/snapshot_fs.go +++ /dev/null @@ -1,81 +0,0 @@ -/* - * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package creator - -import ( - "context" - - "huawei-csi-driver/storage/oceanstor/client" - "huawei-csi-driver/utils" -) - -// SnapshotFsOptionFunc defines the function to change fields of SnapshotFsCreator -type SnapshotFsOptionFunc func(*SnapshotFsCreator) - -var _ VolumeCreator = (*SnapshotFsCreator)(nil) - -// SnapshotFsCreator provides the ability to create a snapshot file system. -type SnapshotFsCreator struct { - *BaseCreator - cloneCreator *CloneFsCreator - params *Parameter - - snapshotID string - snapshotName string - snapshotParentID string - snapshotParentName string - cloneSpeed int -} - -// NewSnapshotFsFromParams returns an instance of NewSnapshotFsFromParams -func NewSnapshotFsFromParams(cli client.BaseClientInterface, - params *Parameter, opts ...SnapshotFsOptionFunc) *SnapshotFsCreator { - base := &BaseCreator{cli: cli} - base.Init(params) - - creator := &SnapshotFsCreator{ - BaseCreator: base, - params: params, - snapshotID: params.SnapshotID(), - snapshotName: utils.GetFSSnapshotName(params.SourceSnapshotName()), - snapshotParentID: params.SnapshotParentId(), - snapshotParentName: params.SnapshotParentName(), - cloneSpeed: params.CloneSpeed(), - } - - for _, opt := range opts { - opt(creator) - } - - return creator -} - -// CreateVolume creates a snapshot filesystem volume on the storage backend. -func (creator *SnapshotFsCreator) CreateVolume(ctx context.Context) (utils.Volume, error) { - creator.cloneCreator = NewCloneFsCreatorByParams(creator.cli, - creator.params, - WithParentSnapshotId(creator.snapshotID), - WithIsDeleteParentSnapshot(false), - WithCloneFrom(creator.snapshotParentName)) - - creator.cloneCreator.BaseCreator = creator.BaseCreator - return creator.cloneCreator.CreateVolume(ctx) -} - -func (creator *SnapshotFsCreator) rollback(ctx context.Context) { - creator.cloneCreator.rollback(ctx) -} diff --git a/storage/oceanstorage/api/base.go b/storage/oceanstorage/api/base.go new file mode 100644 index 00000000..604eff9b --- /dev/null +++ b/storage/oceanstorage/api/base.go @@ -0,0 +1,24 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package api provides oceandisk restful urls definition +package api + +// Base interface urls +const ( + // GetAllQos is the query path for getting all qos + GetAllQos = "/ioclass" +) diff --git a/storage/oceanstorage/api/oceandisk.go b/storage/oceanstorage/api/oceandisk.go new file mode 100644 index 00000000..9c1efc8a --- /dev/null +++ b/storage/oceanstorage/api/oceandisk.go @@ -0,0 +1,52 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package api provides oceandisk restful urls definition +package api + +// Oceandisk storage interface urls +const ( + // QueryAssociateNamespaceGroup is the query path for associating a namespace group. + QueryAssociateNamespaceGroup = "/namespacegroup/associate?ASSOCIATEOBJTYPE=%d&ASSOCIATEOBJID=%s" + // GetNamespaceByName is the query path for getting a namespace by its name. + GetNamespaceByName = "/namespace?filter=NAME::%s&range=[0-100]" + // GetNamespaceByID is the query path for getting a namespace by its ID. + GetNamespaceByID = "/namespace/%s" + // AddNamespaceToGroup is the path for adding a namespace to a group. + AddNamespaceToGroup = "/namespacegroup/associate" + // RemoveNamespaceFromGroup is the path for removing a namespace from a group. + RemoveNamespaceFromGroup = "/namespacegroup/associate" + // GetNamespaceGroupByName is the query path for getting a namespace group by its name. + GetNamespaceGroupByName = "/namespacegroup?filter=NAME::%s" + // CreateNamespaceGroup is the path for creating a namespace group. + CreateNamespaceGroup = "/namespacegroup" + // DeleteNamespaceGroup is the path for deleting a namespace group. + DeleteNamespaceGroup = "/namespacegroup/%s" + // CreateNamespace is the path for creating a namespace. + CreateNamespace = "/namespace" + // DeleteNamespace is the path for deleting a namespace. + DeleteNamespace = "/namespace/%s" + // ExtendNamespace is the path for expanding a namespace. + ExtendNamespace = "/namespace/expand" + // GetNamespaceCountOfMapping is the query path for getting the number of namespaces in a mapping relationship. + GetNamespaceCountOfMapping = "/namespace/count?ASSOCIATEOBJTYPE=245&ASSOCIATEOBJID=%s" + // GetNamespaceCountOfHost is the query path for getting the number of namespaces on a host. + GetNamespaceCountOfHost = "/namespace/count?ASSOCIATEOBJTYPE=21&ASSOCIATEOBJID=%s" + // GetHostNamespaceId is the query path for getting the ID of a host namespace. + GetHostNamespaceId = "/namespace/associate?TYPE=11&ASSOCIATEOBJTYPE=21&ASSOCIATEOBJID=%s" + // UpdateNamespace is the path for updating a namespace. + UpdateNamespace = "/namespace/%s" +) diff --git a/storage/oceanstor/attacher/attacher.go b/storage/oceanstorage/base/attacher/attacher.go similarity index 52% rename from storage/oceanstor/attacher/attacher.go rename to storage/oceanstorage/base/attacher/attacher.go index 3e9d91a1..dae208c8 100644 --- a/storage/oceanstor/attacher/attacher.go +++ b/storage/oceanstorage/base/attacher/attacher.go @@ -14,7 +14,7 @@ * limitations under the License. */ -// Package attacher provide operations of volume attach +// Package attacher provide base operations for volume attach package attacher import ( @@ -24,11 +24,10 @@ import ( "net" "strings" - "huawei-csi-driver/connector/nvme" - pkgUtils "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/storage/oceanstor/client" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector/nvme" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/base" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( @@ -36,44 +35,45 @@ const ( maxHostNameLength = 31 ) -// VolumeAttacherPlugin defines interfaces of attach operations -type VolumeAttacherPlugin interface { - ControllerAttach(context.Context, string, map[string]interface{}) (map[string]interface{}, error) - ControllerDetach(context.Context, string, map[string]interface{}) (string, error) - getTargetRoCEPortals(context.Context) ([]string, error) - getLunInfo(context.Context, string) (map[string]interface{}, error) +// BaseAttacherClientInterface defines client interfaces need to be implemented for base attacher +type BaseAttacherClientInterface interface { + base.FC + base.Host + base.Iscsi + base.Mapping + base.RoCE } -// VolumeAttacher defines attacher to attach volume -type VolumeAttacher struct { - cli client.BaseClientInterface - protocol string - invoker string - portals []string - alua map[string]interface{} +// AttachmentManager provides base operations for attach +type AttachmentManager struct { + Cli BaseAttacherClientInterface + Protocol string + Invoker string + Portals []string + Alua map[string]interface{} } -// VolumeAttacherConfig defines the configurations of VolumeAttacher -type VolumeAttacherConfig struct { - Product string - Cli client.BaseClientInterface +// AttachmentManagerConfig defines the configurations of AttachmentManager +type AttachmentManagerConfig struct { + Cli BaseAttacherClientInterface Protocol string Invoker string Portals []string Alua map[string]interface{} } -// NewAttacher init a new attacher -func NewAttacher(config VolumeAttacherConfig) VolumeAttacherPlugin { - switch config.Product { - case "DoradoV6": - return newDoradoV6Attacher(config) - default: - return newOceanStorAttacher(config) +// NewAttachmentManager init a new AttachmentManager +func NewAttachmentManager(config AttachmentManagerConfig) *AttachmentManager { + return &AttachmentManager{ + Cli: config.Cli, + Protocol: config.Protocol, + Invoker: config.Invoker, + Portals: config.Portals, + Alua: config.Alua, } } -func (p *VolumeAttacher) getHostName(postfix string) string { +func (p *AttachmentManager) getHostName(postfix string) string { host := fmt.Sprintf("k8s_%s", postfix) if len(host) <= maxHostNameLength { return host @@ -82,19 +82,16 @@ func (p *VolumeAttacher) getHostName(postfix string) string { return host[:maxHostNameLength] } -func (p *VolumeAttacher) getHostGroupName(postfix string) string { - return fmt.Sprintf("k8s_%s_hostgroup_%s", p.invoker, postfix) +func (p *AttachmentManager) getHostGroupName(postfix string) string { + return fmt.Sprintf("k8s_%s_hostgroup_%s", p.Invoker, postfix) } -func (p *VolumeAttacher) getLunGroupName(postfix string) string { - return fmt.Sprintf("k8s_%s_lungroup_%s", p.invoker, postfix) +func (p *AttachmentManager) getMappingName(postfix string) string { + return fmt.Sprintf("k8s_%s_mapping_%s", p.Invoker, postfix) } -func (p *VolumeAttacher) getMappingName(postfix string) string { - return fmt.Sprintf("k8s_%s_mapping_%s", p.invoker, postfix) -} - -func (p *VolumeAttacher) getHost(ctx context.Context, +// GetHost gets an exist host or create a new host by host name from params +func (p *AttachmentManager) GetHost(ctx context.Context, parameters map[string]interface{}, toCreate bool) (map[string]interface{}, error) { var err error @@ -106,13 +103,13 @@ func (p *VolumeAttacher) getHost(ctx context.Context, } hostToQuery := p.getHostName(hostname) - host, err := p.cli.GetHostByName(ctx, hostToQuery) + host, err := p.Cli.GetHostByName(ctx, hostToQuery) if err != nil { log.AddContext(ctx).Errorf("Get host %s error: %v", hostToQuery, err) return nil, err } if host == nil && toCreate { - host, err = p.cli.CreateHost(ctx, hostToQuery) + host, err = p.Cli.CreateHost(ctx, hostToQuery) if err != nil { log.AddContext(ctx).Errorf("Create host %s error: %v", hostToQuery, err) return nil, err @@ -130,15 +127,16 @@ func (p *VolumeAttacher) getHost(ctx context.Context, return nil, nil } -func (p *VolumeAttacher) createMapping(ctx context.Context, hostID string) (string, error) { +// CreateMapping creates mapping by hostID +func (p *AttachmentManager) CreateMapping(ctx context.Context, hostID string) (string, error) { mappingName := p.getMappingName(hostID) - mapping, err := p.cli.GetMappingByName(ctx, mappingName) + mapping, err := p.Cli.GetMappingByName(ctx, mappingName) if err != nil { log.AddContext(ctx).Errorf("Get mapping by name %s error: %v", mappingName, err) return "", err } if mapping == nil { - mapping, err = p.cli.CreateMapping(ctx, mappingName) + mapping, err = p.Cli.CreateMapping(ctx, mappingName) if err != nil { log.AddContext(ctx).Errorf("Create mapping %s error: %v", mappingName, err) return "", err @@ -148,12 +146,13 @@ func (p *VolumeAttacher) createMapping(ctx context.Context, hostID string) (stri return mapping["ID"].(string), nil } -func (p *VolumeAttacher) createHostGroup(ctx context.Context, hostID, mappingID string) error { +// CreateHostGroup creates or gets a host group, add host to the group and add the group to mapping +func (p *AttachmentManager) CreateHostGroup(ctx context.Context, hostID, mappingID string) error { var err error var hostGroup map[string]interface{} var hostGroupID string - hostGroupsByHostID, err := p.cli.QueryAssociateHostGroup(ctx, client.AssociateObjTypeHost, hostID) + hostGroupsByHostID, err := p.Cli.QueryAssociateHostGroup(ctx, base.AssociateObjTypeHost, hostID) if err != nil { log.AddContext(ctx).Errorf("Query associated hostgroups of host %s error: %v", hostID, err) @@ -178,13 +177,13 @@ func (p *VolumeAttacher) createHostGroup(ctx context.Context, hostID, mappingID } } - hostGroup, err = p.cli.GetHostGroupByName(ctx, hostGroupName) + hostGroup, err = p.Cli.GetHostGroupByName(ctx, hostGroupName) if err != nil { log.AddContext(ctx).Errorf("Get hostgroup by name %s error: %v", hostGroupName, err) return err } if hostGroup == nil { - hostGroup, err = p.cli.CreateHostGroup(ctx, hostGroupName) + hostGroup, err = p.Cli.CreateHostGroup(ctx, hostGroupName) if err != nil { log.AddContext(ctx).Errorf("Create hostgroup %s error: %v", hostGroupName, err) return err @@ -196,7 +195,7 @@ func (p *VolumeAttacher) createHostGroup(ctx context.Context, hostID, mappingID return errors.New("createHostGroup failed, caused by not found hostGroup id") } - err = p.cli.AddHostToGroup(ctx, hostID, hostGroupID) + err = p.Cli.AddHostToGroup(ctx, hostID, hostGroupID) if err != nil { log.AddContext(ctx).Errorf("Add host %s to hostgroup %s error: %v", hostID, hostGroupID, err) @@ -206,8 +205,8 @@ func (p *VolumeAttacher) createHostGroup(ctx context.Context, hostID, mappingID return p.addToHostGroupMapping(ctx, hostGroupName, hostGroupID, mappingID) } -func (p *VolumeAttacher) addToHostGroupMapping(ctx context.Context, groupName, groupID, mappingID string) error { - hostGroupsByMappingID, err := p.cli.QueryAssociateHostGroup(ctx, client.AssociateObjTypeMapping, mappingID) +func (p *AttachmentManager) addToHostGroupMapping(ctx context.Context, groupName, groupID, mappingID string) error { + hostGroupsByMappingID, err := p.Cli.QueryAssociateHostGroup(ctx, base.AssociateObjTypeMapping, mappingID) if err != nil { log.AddContext(ctx).Errorf("Query associated host groups of mapping %s error: %v", mappingID, err) return err @@ -223,7 +222,7 @@ func (p *VolumeAttacher) addToHostGroupMapping(ctx context.Context, groupName, g } } - err = p.cli.AddGroupToMapping(ctx, client.AssociateObjTypeHostGroup, groupID, mappingID) + err = p.Cli.AddGroupToMapping(ctx, base.AssociateObjTypeHostGroup, groupID, mappingID) if err != nil { log.AddContext(ctx).Errorf("Add host group %s to mapping %s error: %v", groupID, mappingID, err) @@ -233,120 +232,7 @@ func (p *VolumeAttacher) addToHostGroupMapping(ctx context.Context, groupName, g return nil } -func (p *VolumeAttacher) createLunGroup(ctx context.Context, lunID, hostID, mappingID string) error { - var err error - var lunGroup map[string]interface{} - - lunGroupsByLunID, err := p.cli.QueryAssociateLunGroup(ctx, client.AssociateObjTypeLUN, lunID) - if err != nil { - log.AddContext(ctx).Errorf("Query associated lun groups of lun %s error: %v", lunID, err) - return err - } - - lunGroupName := p.getLunGroupName(hostID) - for _, i := range lunGroupsByLunID { - group, ok := i.(map[string]interface{}) - if !ok { - log.AddContext(ctx).Warningf("convert group to map failed, data: %v", i) - continue - } - if group["NAME"].(string) == lunGroupName { - lunGroupID, ok := group["ID"].(string) - if !ok { - return errors.New("convert group[\"ID\"] to string failed") - } - return p.addToLUNGroupMapping(ctx, lunGroupName, lunGroupID, mappingID) - } - } - - lunGroup, err = p.cli.GetLunGroupByName(ctx, lunGroupName) - if err != nil { - log.AddContext(ctx).Errorf("Get lungroup by name %s error: %v", lunGroupName, err) - return err - } - if lunGroup == nil { - lunGroup, err = p.cli.CreateLunGroup(ctx, lunGroupName) - if err != nil { - log.AddContext(ctx).Errorf("Create lungroup %s error: %v", lunGroupName, err) - return err - } - } - - lunGroupID, ok := lunGroup["ID"].(string) - if !ok { - return errors.New("createLunGroup failed, caused by not found lun group id") - } - err = p.cli.AddLunToGroup(ctx, lunID, lunGroupID) - if err != nil { - log.AddContext(ctx).Errorf("Add lun %s to group %s error: %v", lunID, lunGroupID, err) - return err - } - - return p.addToLUNGroupMapping(ctx, lunGroupName, lunGroupID, mappingID) -} - -func (p *VolumeAttacher) addToLUNGroupMapping(ctx context.Context, groupName, groupID, mappingID string) error { - lunGroupsByMappingID, err := p.cli.QueryAssociateLunGroup(ctx, client.AssociateObjTypeMapping, mappingID) - if err != nil { - log.AddContext(ctx).Errorf("Query associated lun groups of mapping %s error: %v", mappingID, err) - return err - } - - for _, i := range lunGroupsByMappingID { - group, ok := i.(map[string]interface{}) - if !ok { - return fmt.Errorf("invalid group type. Expected 'map[string]interface{}', found %T", i) - } - if group["NAME"].(string) == groupName { - return nil - } - } - - err = p.cli.AddGroupToMapping(ctx, client.AssociateObjTypeLUNGroup, groupID, mappingID) - if err != nil { - log.AddContext(ctx).Errorf("Add lun group %s to mapping %s error: %v", - groupID, mappingID, err) - return err - } - - return nil -} - -func (p *VolumeAttacher) needUpdateInitiatorAlua(initiator map[string]interface{}) bool { - if p.alua == nil { - return false - } - - multiPathType, ok := p.alua["MULTIPATHTYPE"] - if !ok { - return false - } - - if multiPathType != initiator["MULTIPATHTYPE"] { - return true - } else if initiator["MULTIPATHTYPE"] == MultiPathTypeDefault { - return false - } - - failoverMode, ok := p.alua["FAILOVERMODE"] - if ok && failoverMode != initiator["FAILOVERMODE"] { - return true - } - - specialModeType, ok := p.alua["SPECIALMODETYPE"] - if ok && specialModeType != initiator["SPECIALMODETYPE"] { - return true - } - - pathType, ok := p.alua["PATHTYPE"] - if ok && pathType != initiator["PATHTYPE"] { - return true - } - - return false -} - -func (p *VolumeAttacher) getISCSIProperties(ctx context.Context, wwn, hostLunId string, parameters map[string]any) ( +func (p *AttachmentManager) getISCSIProperties(ctx context.Context, wwn, hostLunId string, parameters map[string]any) ( map[string]interface{}, error) { tgtPortals, tgtIQNs, err := p.getTargetISCSIProperties(ctx) if err != nil { @@ -367,7 +253,7 @@ func (p *VolumeAttacher) getISCSIProperties(ctx context.Context, wwn, hostLunId }, nil } -func (p *VolumeAttacher) getFCProperties(ctx context.Context, wwn, hostLunId string, parameters map[string]any) ( +func (p *AttachmentManager) getFCProperties(ctx context.Context, wwn, hostLunId string, parameters map[string]any) ( map[string]interface{}, error) { tgtWWNs, err := p.getTargetFCProperties(ctx, parameters) if err != nil { @@ -387,7 +273,7 @@ func (p *VolumeAttacher) getFCProperties(ctx context.Context, wwn, hostLunId str }, nil } -func (p *VolumeAttacher) getFCNVMeProperties(ctx context.Context, wwn, hostLunId string, parameters map[string]any) ( +func (p *AttachmentManager) getFCNVMeProperties(ctx context.Context, wwn, hostLunId string, parameters map[string]any) ( map[string]interface{}, error) { portWWNList, err := p.getTargetFCNVMeProperties(ctx, parameters) if err != nil { @@ -400,9 +286,9 @@ func (p *VolumeAttacher) getFCNVMeProperties(ctx context.Context, wwn, hostLunId }, nil } -func (p *VolumeAttacher) getRoCEProperties(ctx context.Context, wwn, hostLunId string, parameters map[string]any) ( +func (p *AttachmentManager) getRoCEProperties(ctx context.Context, wwn, hostLunId string, parameters map[string]any) ( map[string]interface{}, error) { - tgtPortals, err := p.getTargetRoCEPortals(ctx) + tgtPortals, err := p.GetTargetRoCEPortals(ctx) if err != nil { return nil, err } @@ -413,23 +299,24 @@ func (p *VolumeAttacher) getRoCEProperties(ctx context.Context, wwn, hostLunId s }, nil } -func (p *VolumeAttacher) getMappingProperties(ctx context.Context, +// GetMappingProperties gets the mapping properties +func (p *AttachmentManager) GetMappingProperties(ctx context.Context, wwn, hostLunId string, parameters map[string]interface{}) (map[string]interface{}, error) { - if p.protocol == "iscsi" { + if p.Protocol == "iscsi" { return p.getISCSIProperties(ctx, wwn, hostLunId, parameters) - } else if p.protocol == "fc" { + } else if p.Protocol == "fc" { return p.getFCProperties(ctx, wwn, hostLunId, parameters) - } else if p.protocol == "fc-nvme" { + } else if p.Protocol == "fc-nvme" { return p.getFCNVMeProperties(ctx, wwn, hostLunId, parameters) - } else if p.protocol == "roce" { + } else if p.Protocol == "roce" { return p.getRoCEProperties(ctx, wwn, hostLunId, parameters) } - return nil, utils.Errorf(ctx, "UnSupport protocol %s", p.protocol) + return nil, utils.Errorf(ctx, "UnSupport protocol %s", p.Protocol) } -func (p *VolumeAttacher) getTargetISCSIProperties(ctx context.Context) ([]string, []string, error) { - ports, err := p.cli.GetIscsiTgtPort(ctx) +func (p *AttachmentManager) getTargetISCSIProperties(ctx context.Context) ([]string, []string, error) { + ports, err := p.Cli.GetIscsiTgtPort(ctx) if err != nil { log.AddContext(ctx).Errorf("Get iSCSI tgt port error: %v", err) return nil, nil, err @@ -466,7 +353,7 @@ func (p *VolumeAttacher) getTargetISCSIProperties(ctx context.Context) ([]string var tgtPortals []string var tgtIQNs []string - for _, portal := range p.portals { + for _, portal := range p.Portals { ip := net.ParseIP(portal).String() if !validIPs[ip] { log.AddContext(ctx).Warningf("ISCSI portal %s is not valid", ip) @@ -479,7 +366,7 @@ func (p *VolumeAttacher) getTargetISCSIProperties(ctx context.Context) ([]string } if len(tgtPortals) == 0 { - msg := fmt.Sprintf("All config portal %s is not valid", p.portals) + msg := fmt.Sprintf("All config portal %s is not valid", p.Portals) log.AddContext(ctx).Errorln(msg) return nil, nil, errors.New(msg) } @@ -487,11 +374,12 @@ func (p *VolumeAttacher) getTargetISCSIProperties(ctx context.Context) ([]string return tgtPortals, tgtIQNs, nil } -func (p *VolumeAttacher) getTargetRoCEPortals(ctx context.Context) ([]string, error) { +// GetTargetRoCEPortals gets target roce portals +func (p *AttachmentManager) GetTargetRoCEPortals(ctx context.Context) ([]string, error) { var availablePortals []string - for _, portal := range p.portals { + for _, portal := range p.Portals { ip := net.ParseIP(portal).String() - rocePortal, err := p.cli.GetRoCEPortalByIP(ctx, ip) + rocePortal, err := p.Cli.GetRoCEPortalByIP(ctx, ip) if err != nil { log.AddContext(ctx).Errorf("Get RoCE tgt portal error: %v", err) return nil, err @@ -518,7 +406,7 @@ func (p *VolumeAttacher) getTargetRoCEPortals(ctx context.Context) ([]string, er } if len(availablePortals) == 0 { - msg := fmt.Sprintf("All config portal %s is not valid", p.portals) + msg := fmt.Sprintf("All config portal %s is not valid", p.Portals) log.AddContext(ctx).Errorln(msg) return nil, errors.New(msg) } @@ -526,7 +414,7 @@ func (p *VolumeAttacher) getTargetRoCEPortals(ctx context.Context) ([]string, er return availablePortals, nil } -func (p *VolumeAttacher) getTargetFCNVMeProperties(ctx context.Context, +func (p *AttachmentManager) getTargetFCNVMeProperties(ctx context.Context, parameters map[string]interface{}) ([]nvme.PortWWNPair, error) { fcInitiators, err := GetMultipleInitiators(ctx, FC, parameters) if err != nil { @@ -536,7 +424,7 @@ func (p *VolumeAttacher) getTargetFCNVMeProperties(ctx context.Context, var ret []nvme.PortWWNPair for _, hostInitiator := range fcInitiators { - tgtWWNs, err := p.cli.GetFCTargetWWNs(ctx, hostInitiator) + tgtWWNs, err := p.Cli.GetFCTargetWWNs(ctx, hostInitiator) if err != nil { return nil, err } @@ -550,7 +438,7 @@ func (p *VolumeAttacher) getTargetFCNVMeProperties(ctx context.Context, return ret, nil } -func (p *VolumeAttacher) getTargetFCProperties(ctx context.Context, parameters map[string]any) ([]string, error) { +func (p *AttachmentManager) getTargetFCProperties(ctx context.Context, parameters map[string]any) ([]string, error) { fcInitiators, err := GetMultipleInitiators(ctx, FC, parameters) if err != nil { log.AddContext(ctx).Errorf("Get fc initiator error: %v", err) @@ -559,7 +447,7 @@ func (p *VolumeAttacher) getTargetFCProperties(ctx context.Context, parameters m validTgtWWNs := make(map[string]bool) for _, wwn := range fcInitiators { - tgtWWNs, err := p.cli.GetFCTargetWWNs(ctx, wwn) + tgtWWNs, err := p.Cli.GetFCTargetWWNs(ctx, wwn) if err != nil { return nil, err } @@ -587,7 +475,8 @@ func (p *VolumeAttacher) getTargetFCProperties(ctx context.Context, parameters m return tgtWWNs, nil } -func (p *VolumeAttacher) attachISCSI(ctx context.Context, +// AttachISCSI generates the relationship between iscsi initiator and host +func (p *AttachmentManager) AttachISCSI(ctx context.Context, hostID string, parameters map[string]interface{}) (map[string]interface{}, error) { name, err := GetSingleInitiator(ctx, ISCSI, parameters) if err != nil { @@ -595,14 +484,14 @@ func (p *VolumeAttacher) attachISCSI(ctx context.Context, return nil, err } - initiator, err := p.cli.GetIscsiInitiator(ctx, name) + initiator, err := p.Cli.GetIscsiInitiator(ctx, name) if err != nil { log.AddContext(ctx).Errorf("Get ISCSI initiator %s error: %v", name, err) return nil, err } if initiator == nil { - initiator, err = p.cli.AddIscsiInitiator(ctx, name) + initiator, err = p.Cli.AddIscsiInitiator(ctx, name) if err != nil { log.AddContext(ctx).Errorf("Add initiator %s error: %v", name, err) return nil, err @@ -618,7 +507,7 @@ func (p *VolumeAttacher) attachISCSI(ctx context.Context, log.AddContext(ctx).Warningf("convert parentID to string failed, data: %v", initiator["PARENTID"]) } if freeExist && isFree == "true" { - err := p.cli.AddIscsiInitiatorToHost(ctx, name, hostID) + err := p.Cli.AddIscsiInitiatorToHost(ctx, name, hostID) if err != nil { log.AddContext(ctx).Errorf("Add ISCSI initiator %s to host %s error: %v", name, hostID, err) return nil, err @@ -632,7 +521,8 @@ func (p *VolumeAttacher) attachISCSI(ctx context.Context, return initiator, nil } -func (p *VolumeAttacher) attachFC(ctx context.Context, +// AttachFC generates the relationship between fc initiator and host +func (p *AttachmentManager) AttachFC(ctx context.Context, hostID string, parameters map[string]interface{}) ([]map[string]interface{}, error) { fcInitiators, err := GetMultipleInitiators(ctx, FC, parameters) if err != nil { @@ -644,7 +534,7 @@ func (p *VolumeAttacher) attachFC(ctx context.Context, var hostInitiators []map[string]interface{} for _, wwn := range fcInitiators { - initiator, err := p.cli.GetFCInitiator(ctx, wwn) + initiator, err := p.Cli.GetFCInitiator(ctx, wwn) if err != nil { log.AddContext(ctx).Errorf("Get FC initiator %s error: %v", wwn, err) return nil, err @@ -681,7 +571,7 @@ func (p *VolumeAttacher) attachFC(ctx context.Context, } for _, wwn := range addWWNs { - err := p.cli.AddFCInitiatorToHost(ctx, wwn, hostID) + err := p.Cli.AddFCInitiatorToHost(ctx, wwn, hostID) if err != nil { log.AddContext(ctx).Errorf("Add initiator %s to host %s error: %v", wwn, hostID, err) return nil, err @@ -691,7 +581,8 @@ func (p *VolumeAttacher) attachFC(ctx context.Context, return hostInitiators, nil } -func (p *VolumeAttacher) attachRoCE(ctx context.Context, +// AttachRoCE generates the relationship between roce initiator and host +func (p *AttachmentManager) AttachRoCE(ctx context.Context, hostID string, parameters map[string]interface{}) (map[string]interface{}, error) { name, err := GetSingleInitiator(ctx, ROCE, parameters) if err != nil { @@ -699,14 +590,14 @@ func (p *VolumeAttacher) attachRoCE(ctx context.Context, return nil, err } - initiator, err := p.cli.GetRoCEInitiator(ctx, name) + initiator, err := p.Cli.GetRoCEInitiator(ctx, name) if err != nil { log.AddContext(ctx).Errorf("Get RoCE initiator %s error: %v", name, err) return nil, err } if initiator == nil { - initiator, err = p.cli.AddRoCEInitiator(ctx, name) + initiator, err = p.Cli.AddRoCEInitiator(ctx, name) if err != nil { log.AddContext(ctx).Errorf("Add initiator %s error: %v", name, err) return nil, err @@ -722,7 +613,7 @@ func (p *VolumeAttacher) attachRoCE(ctx context.Context, log.AddContext(ctx).Warningf("convert parentID to string failed, data: %v", initiator["PARENTID"]) } if freeExist && isFree == "true" { - err := p.cli.AddRoCEInitiatorToHost(ctx, name, hostID) + err := p.Cli.AddRoCEInitiatorToHost(ctx, name, hostID) if err != nil { log.AddContext(ctx).Errorf("Add RoCE initiator %s to host %s error: %v", name, hostID, err) return nil, err @@ -735,138 +626,3 @@ func (p *VolumeAttacher) attachRoCE(ctx context.Context, return initiator, nil } - -func (p *VolumeAttacher) doMapping(ctx context.Context, hostID, lunName string) (string, string, error) { - lun, err := p.cli.GetLunByName(ctx, lunName) - if err != nil { - log.AddContext(ctx).Errorf("Get lun %s error: %v", lunName, err) - return "", "", err - } - if lun == nil { - msg := fmt.Sprintf("Lun %s not exist for attaching", lunName) - log.AddContext(ctx).Errorln(msg) - return "", "", errors.New(msg) - } - - lunID, ok := lun["ID"].(string) - if !ok { - return "", "", pkgUtils.Errorf(ctx, "convert lunID to string failed, data: %v", lun["ID"]) - } - mappingID, err := p.createMapping(ctx, hostID) - if err != nil { - log.AddContext(ctx).Errorf("Create mapping for host %s error: %v", hostID, err) - return "", "", err - } - - err = p.createHostGroup(ctx, hostID, mappingID) - if err != nil { - log.AddContext(ctx).Errorf("Create host group for host %s error: %v", hostID, err) - return "", "", err - } - - err = p.createLunGroup(ctx, lunID, hostID, mappingID) - if err != nil { - log.AddContext(ctx).Errorf("Create lun group for host %s error: %v", hostID, err) - return "", "", err - } - - lunUniqueId, err := utils.GetLunUniqueId(ctx, p.protocol, lun) - if err != nil { - return "", "", err - } - - hostLunId, err := p.cli.GetHostLunId(ctx, hostID, lunID) - if err != nil { - return "", "", err - } - - return lunUniqueId, hostLunId, nil -} - -func (p *VolumeAttacher) doUnmapping(ctx context.Context, hostID, lunName string) (string, error) { - lun, err := p.cli.GetLunByName(ctx, lunName) - if err != nil { - log.AddContext(ctx).Errorf("Get lun %s info error: %v", lunName, err) - return "", err - } - if lun == nil { - log.AddContext(ctx).Infof("LUN %s doesn't exist while detaching", lunName) - return "", nil - } - lunID, ok := lun["ID"].(string) - if !ok { - return "", pkgUtils.Errorf(ctx, "convert lunID to string failed, data: %v", lun["ID"]) - } - lunGroupsByLunID, err := p.cli.QueryAssociateLunGroup(ctx, client.AssociateObjTypeLUN, lunID) - if err != nil { - log.AddContext(ctx).Errorf("Query associated lungroups of lun %s error: %v", lunID, err) - return "", err - } - - lunGroupName := p.getLunGroupName(hostID) - for _, i := range lunGroupsByLunID { - group, ok := i.(map[string]interface{}) - if !ok { - log.AddContext(ctx).Warningf("convert group to map failed, data: %v", i) - continue - } - if group["NAME"].(string) == lunGroupName { - lunGroupID, ok := group["ID"].(string) - if !ok { - return "", pkgUtils.Errorf(ctx, "convert lunGroupID to string failed, data: %v", group["ID"]) - } - err = p.cli.RemoveLunFromGroup(ctx, lunID, lunGroupID) - if err != nil { - log.AddContext(ctx).Errorf("Remove lun %s from group %s error: %v", - lunID, lunGroupID, err) - return "", err - } - } - } - - lunUniqueId, err := utils.GetLunUniqueId(ctx, p.protocol, lun) - if err != nil { - return "", err - } - return lunUniqueId, nil -} - -// ControllerDetach detaches volume and unmaps lun from host -func (p *VolumeAttacher) ControllerDetach(ctx context.Context, - lunName string, - parameters map[string]interface{}) (string, error) { - host, err := p.getHost(ctx, parameters, false) - if err != nil { - log.AddContext(ctx).Infof("Get host ID error: %v", err) - return "", err - } - if host == nil { - log.AddContext(ctx).Infof("Host doesn't exist while detaching %s", lunName) - return "", nil - } - - hostID, ok := host["ID"].(string) - if !ok { - return "", pkgUtils.Errorf(ctx, "convert hostID to string failed, data: %v", host["ID"]) - } - wwn, err := p.doUnmapping(ctx, hostID, lunName) - if err != nil { - log.AddContext(ctx).Errorf("Unmapping LUN %s from host %s error: %v", lunName, hostID, err) - return "", err - } - - return wwn, nil -} - -func (p *VolumeAttacher) getLunInfo(ctx context.Context, lunName string) (map[string]interface{}, error) { - lun, err := p.cli.GetLunByName(ctx, lunName) - if err != nil { - log.AddContext(ctx).Errorf("Get lun %s info error: %v", lunName, err) - return nil, err - } - if lun == nil { - log.AddContext(ctx).Infof("LUN %s doesn't exist while detaching", lunName) - return nil, nil - } - return lun, nil -} diff --git a/storage/oceanstor/attacher/attacher_utils.go b/storage/oceanstorage/base/attacher/attacher_utils.go similarity index 88% rename from storage/oceanstor/attacher/attacher_utils.go rename to storage/oceanstorage/base/attacher/attacher_utils.go index 687bc41e..de26917b 100644 --- a/storage/oceanstor/attacher/attacher_utils.go +++ b/storage/oceanstorage/base/attacher/attacher_utils.go @@ -19,12 +19,12 @@ package attacher import ( "context" - _ "huawei-csi-driver/connector/fibrechannel" - "huawei-csi-driver/connector/host" - _ "huawei-csi-driver/connector/iscsi" - _ "huawei-csi-driver/connector/nvme" - _ "huawei-csi-driver/connector/roce" - "huawei-csi-driver/utils" + _ "github.com/Huawei/eSDK_K8S_Plugin/v4/connector/fibrechannel" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector/host" + _ "github.com/Huawei/eSDK_K8S_Plugin/v4/connector/iscsi" + _ "github.com/Huawei/eSDK_K8S_Plugin/v4/connector/nvme" + _ "github.com/Huawei/eSDK_K8S_Plugin/v4/connector/roce" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" ) // InitiatorType defines the initiator type diff --git a/storage/oceanstor/attacher/attacher_utils_test.go b/storage/oceanstorage/base/attacher/attacher_utils_test.go similarity index 97% rename from storage/oceanstor/attacher/attacher_utils_test.go rename to storage/oceanstorage/base/attacher/attacher_utils_test.go index 6a310cee..207393b7 100644 --- a/storage/oceanstor/attacher/attacher_utils_test.go +++ b/storage/oceanstorage/base/attacher/attacher_utils_test.go @@ -23,8 +23,8 @@ import ( "github.com/agiledragon/gomonkey/v2" - "huawei-csi-driver/connector/host" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/connector/host" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/storage/oceanstorage/base/client.go b/storage/oceanstorage/base/client.go new file mode 100644 index 00000000..36685575 --- /dev/null +++ b/storage/oceanstorage/base/client.go @@ -0,0 +1,280 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package base provide base operations for oceanstor and oceandisk storage +package base + +import ( + "context" + "crypto/tls" + "encoding/json" + "errors" + "fmt" + "net/http" + "net/http/cookiejar" + "slices" + "time" + + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" +) + +const ( + // SuccessCode defines error code of success + SuccessCode = int64(0) + + // QueryCountPerBatch defines query count for each circle of batch operation + QueryCountPerBatch int = 100 + + // UserOffline defines error code of user off line + UserOffline = 1077949069 + + // IPLockErrorCode defines error code of ip lock + IPLockErrorCode = 1077949071 + + // UserUnauthorized defines error code of user unauthorized + UserUnauthorized = -401 + + // Unconnected defines the error msg of unconnected + Unconnected = "unconnected" + + // LocalUserType defines the user type of local + LocalUserType = "0" + + // MaxStorageThreads defines max threads of each storage + MaxStorageThreads = 100 + + // UninitializedStorage defines uninitialized storage + UninitializedStorage = "UninitializedStorage" + + defaultHttpTimeout = 60 * time.Second +) + +var ( + // WrongPasswordErrorCodes user or password is incorrect + WrongPasswordErrorCodes = []int64{1077987870, 1077949081, 1077949061} + // AccountBeenLocked account been locked + AccountBeenLocked = []int64{1077949070, 1077987871} + // RequestSemaphoreMap stores the total connection num of each storage + RequestSemaphoreMap = map[string]*utils.Semaphore{UninitializedStorage: utils.NewSemaphore(MaxStorageThreads)} +) + +// Response defines response of request +type Response struct { + Error map[string]interface{} `json:"error"` + Data interface{} `json:"data,omitempty"` +} + +// AssertErrorCode asserts if error code represents success +func (resp *Response) AssertErrorCode() error { + val, exists := resp.Error["code"] + if !exists { + return fmt.Errorf("error code not exists, data: %+v", *resp) + } + + code, ok := val.(float64) + if !ok { + return fmt.Errorf("code is not float64, data: %+v", *resp) + } + + if int64(code) != SuccessCode { + return fmt.Errorf("error code is not success, data: %+v", *resp) + } + + return nil +} + +// GetData converts interface{} type data to specific type +func (resp *Response) GetData(val any) error { + data, err := json.Marshal(resp.Data) + if err != nil { + return fmt.Errorf("failed to marshal data, error %w", err) + } + + err = json.Unmarshal(data, &val) + if err != nil { + return fmt.Errorf("failed to unmarshal data as %T, error: %w", val, err) + } + + return nil +} + +// RestClientInterface defines interfaces for base restful call +type RestClientInterface interface { + Call(ctx context.Context, method string, url string, data map[string]interface{}) (Response, error) + BaseCall(ctx context.Context, method string, url string, data map[string]interface{}) (Response, error) + Get(ctx context.Context, url string, data map[string]interface{}) (Response, error) + Post(ctx context.Context, url string, data map[string]interface{}) (Response, error) + Put(ctx context.Context, url string, data map[string]interface{}) (Response, error) + Delete(ctx context.Context, url string, data map[string]interface{}) (Response, error) + GetRequest(ctx context.Context, method string, url string, data map[string]interface{}) (*http.Request, error) + Login(ctx context.Context) error + Logout(ctx context.Context) + ReLogin(ctx context.Context) error + GetSystem(ctx context.Context) (map[string]interface{}, error) +} + +// HTTP defines for http request process +type HTTP interface { + Do(req *http.Request) (*http.Response, error) +} + +// NewHTTPClientByBackendID provides a new http client by backend id +func NewHTTPClientByBackendID(ctx context.Context, backendID string) (HTTP, error) { + var defaultUseCert bool + client := &http.Client{ + Transport: &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: !defaultUseCert}}, + Timeout: defaultHttpTimeout, + } + + jar, err := cookiejar.New(nil) + if err != nil { + log.AddContext(ctx).Errorf("create jar failed, error: %v", err) + return client, err + } + + useCert, certMeta, err := pkgUtils.GetCertSecretFromBackendID(ctx, backendID) + if err != nil { + log.AddContext(ctx).Errorf("get cert secret from backend [%v] failed, error: %v", backendID, err) + return client, err + } + + useCert, certPool, err := pkgUtils.GetCertPool(ctx, useCert, certMeta) + if err != nil { + log.AddContext(ctx).Errorf("get cert pool failed, error: %v", err) + return client, err + } + + client.Transport = &http.Transport{TLSClientConfig: &tls.Config{InsecureSkipVerify: !useCert, RootCAs: certPool}} + client.Jar = jar + return client, nil +} + +// NewHTTPClientByCertMeta provides a new http client by cert meta +func NewHTTPClientByCertMeta(ctx context.Context, useCert bool, certMeta string) (HTTP, error) { + jar, err := cookiejar.New(nil) + if err != nil { + log.AddContext(ctx).Errorf("create jar failed, error: %v", err) + return nil, err + } + + useCert, certPool, err := pkgUtils.GetCertPool(ctx, useCert, certMeta) + if err != nil { + return nil, err + } + + return &http.Client{ + Transport: &http.Transport{ + TLSClientConfig: &tls.Config{InsecureSkipVerify: !useCert, RootCAs: certPool}, + }, + Jar: jar, + Timeout: defaultHttpTimeout, + }, nil +} + +// MaskRequestData masks the sensitive data +func MaskRequestData(data map[string]any) map[string]any { + sensitiveKey := []string{"user", "password", "iqn", "tgt", "tgtname", "initiatorname"} + + maskedData := make(map[string]any) + for k, v := range data { + if slices.Contains(sensitiveKey, k) { + maskedData[k] = "***" + } else { + maskedData[k] = v + } + } + + return maskedData +} + +// NeedReLogin determine if it is necessary to log in to the storage again +func NeedReLogin(r Response, err error) bool { + var unconnected, unauthorized, offline bool + if err != nil && err.Error() == Unconnected { + unconnected = true + } + + if r.Error != nil { + if code, ok := r.Error["code"].(float64); ok { + unauthorized = int64(code) == UserUnauthorized + offline = int64(code) == UserOffline + } + } + return unconnected || unauthorized || offline +} + +// GetBatchObjs used to get batch objs by url +func GetBatchObjs(ctx context.Context, cli RestClientInterface, url string) ([]map[string]interface{}, error) { + rangeStart := 0 + var objList []map[string]interface{} + for { + rangeEnd := rangeStart + QueryCountPerBatch + objs, err := getObj(ctx, cli, url, rangeStart, rangeEnd) + if err != nil { + return nil, err + } + + if objs == nil { + break + } + + objList = append(objList, objs...) + if len(objs) < QueryCountPerBatch { + break + } + rangeStart = rangeEnd + } + return objList, nil +} + +func getObj(ctx context.Context, cli RestClientInterface, + url string, start, end int) ([]map[string]interface{}, error) { + objUrl := fmt.Sprintf("%s?range=[%d-%d]", url, start, end) + resp, err := cli.Get(ctx, objUrl, nil) + if err != nil { + return nil, err + } + + code, msg, err := utils.FormatRespErr(resp.Error) + if err != nil { + return nil, err + } + + if code != 0 { + return nil, fmt.Errorf("get batch obj list failed, error code: %d, error msg: %s", code, msg) + } + + if resp.Data == nil { + return nil, nil + } + + var objList []map[string]interface{} + respData, ok := resp.Data.([]interface{}) + if !ok { + return nil, errors.New("convert resp.Data to []interface{} failed") + } + for _, i := range respData { + obj, ok := i.(map[string]interface{}) + if !ok { + log.AddContext(ctx).Warningf("convert resp.Data to map[string]interface{} failed") + continue + } + objList = append(objList, obj) + } + return objList, nil +} diff --git a/storage/oceanstor/client/client_applicationtype.go b/storage/oceanstorage/base/client_applicationtype.go similarity index 80% rename from storage/oceanstor/client/client_applicationtype.go rename to storage/oceanstorage/base/client_applicationtype.go index 53432df9..2df5328e 100644 --- a/storage/oceanstor/client/client_applicationtype.go +++ b/storage/oceanstorage/base/client_applicationtype.go @@ -1,5 +1,5 @@ /* - * Copyright (c) Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2022-2024. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,8 @@ * limitations under the License. */ -package client +// Package base provide base operations for oceanstor and oceandisk storage +package base import ( "context" @@ -29,9 +30,14 @@ type ApplicationType interface { GetApplicationTypeByName(ctx context.Context, appType string) (string, error) } +// ApplicationTypeClient defines client implements the ApplicationType interface +type ApplicationTypeClient struct { + RestClientInterface +} + // GetApplicationTypeByName function to get the Application type ID to set the I/O size // while creating Volume -func (cli *BaseClient) GetApplicationTypeByName(ctx context.Context, appType string) (string, error) { +func (cli *ApplicationTypeClient) GetApplicationTypeByName(ctx context.Context, appType string) (string, error) { result := "" appType = URL.QueryEscape(appType) url := fmt.Sprintf("/workload_type?filter=NAME::%s", appType) @@ -56,7 +62,7 @@ func (cli *BaseClient) GetApplicationTypeByName(ctx context.Context, appType str for _, i := range respData { applicationTypes, ok := i.(map[string]interface{}) if !ok { - return result, errors.New("Data in response is not valid") + return result, errors.New("data in response is not valid") } // From the map we need the application type ID // This will be used for param to create LUN diff --git a/storage/oceanstor/client/client_fc.go b/storage/oceanstorage/base/client_fc.go similarity index 85% rename from storage/oceanstor/client/client_fc.go rename to storage/oceanstorage/base/client_fc.go index 23d4c0cd..a79ef570 100644 --- a/storage/oceanstor/client/client_fc.go +++ b/storage/oceanstorage/base/client_fc.go @@ -1,5 +1,5 @@ /* - * Copyright (c) Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2022-2024. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,14 +14,15 @@ * limitations under the License. */ -package client +// Package base provide base operations for oceanstor and oceandisk storage +package base import ( "context" "errors" "fmt" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // FC defines interfaces for fc operations @@ -42,8 +43,13 @@ type FC interface { GetFCHostLink(ctx context.Context, hostID string) ([]interface{}, error) } +// FCClient defines client implements the FC interface +type FCClient struct { + RestClientInterface +} + // QueryFCInitiatorByHost used for get fc initiator by host id -func (cli *BaseClient) QueryFCInitiatorByHost(ctx context.Context, hostID string) ([]interface{}, error) { +func (cli *FCClient) QueryFCInitiatorByHost(ctx context.Context, hostID string) ([]interface{}, error) { url := fmt.Sprintf("/fc_initiator?PARENTID=%s", hostID) resp, err := cli.Get(ctx, url, nil) if err != nil { @@ -68,7 +74,7 @@ func (cli *BaseClient) QueryFCInitiatorByHost(ctx context.Context, hostID string } // GetFCInitiator used for get fc initiator by ID::wwn -func (cli *BaseClient) GetFCInitiator(ctx context.Context, wwn string) (map[string]interface{}, error) { +func (cli *FCClient) GetFCInitiator(ctx context.Context, wwn string) (map[string]interface{}, error) { url := fmt.Sprintf("/fc_initiator?filter=ID::%s", wwn) resp, err := cli.Get(ctx, url, nil) if err != nil { @@ -97,7 +103,7 @@ func (cli *BaseClient) GetFCInitiator(ctx context.Context, wwn string) (map[stri } // GetFCInitiatorByID used for get fc initiator by id(wwn) -func (cli *BaseClient) GetFCInitiatorByID(ctx context.Context, wwn string) (map[string]interface{}, error) { +func (cli *FCClient) GetFCInitiatorByID(ctx context.Context, wwn string) (map[string]interface{}, error) { url := fmt.Sprintf("/fc_initiator/%s", wwn) resp, err := cli.Get(ctx, url, nil) if err != nil { @@ -118,7 +124,7 @@ func (cli *BaseClient) GetFCInitiatorByID(ctx context.Context, wwn string) (map[ } // UpdateFCInitiator used for update fc initiator -func (cli *BaseClient) UpdateFCInitiator(ctx context.Context, wwn string, alua map[string]interface{}) error { +func (cli *FCClient) UpdateFCInitiator(ctx context.Context, wwn string, alua map[string]interface{}) error { url := fmt.Sprintf("/fc_initiator/%s", wwn) data := map[string]interface{}{} @@ -152,7 +158,7 @@ func (cli *BaseClient) UpdateFCInitiator(ctx context.Context, wwn string, alua m } // AddFCInitiatorToHost used for add fc initiator to host -func (cli *BaseClient) AddFCInitiatorToHost(ctx context.Context, initiator, hostID string) error { +func (cli *FCClient) AddFCInitiatorToHost(ctx context.Context, initiator, hostID string) error { url := fmt.Sprintf("/fc_initiator/%s", initiator) data := map[string]interface{}{ "PARENTTYPE": 21, @@ -173,7 +179,7 @@ func (cli *BaseClient) AddFCInitiatorToHost(ctx context.Context, initiator, host } // GetFCTargetWWNs used for get fc target WWNs -func (cli *BaseClient) GetFCTargetWWNs(ctx context.Context, initiatorWWN string) ([]string, error) { +func (cli *FCClient) GetFCTargetWWNs(ctx context.Context, initiatorWWN string) ([]string, error) { url := fmt.Sprintf("/host_link?INITIATOR_TYPE=223&INITIATOR_PORT_WWN=%s", initiatorWWN) resp, err := cli.Get(ctx, url, nil) if err != nil { @@ -207,7 +213,7 @@ func (cli *BaseClient) GetFCTargetWWNs(ctx context.Context, initiatorWWN string) } // GetFCHostLink used for get fc host link by host id -func (cli *BaseClient) GetFCHostLink(ctx context.Context, hostID string) ([]interface{}, error) { +func (cli *FCClient) GetFCHostLink(ctx context.Context, hostID string) ([]interface{}, error) { url := fmt.Sprintf("/host_link?INITIATOR_TYPE=223&PARENTID=%s", hostID) resp, err := cli.Get(ctx, url, nil) if err != nil { diff --git a/storage/oceanstor/client/client_host.go b/storage/oceanstorage/base/client_host.go similarity index 90% rename from storage/oceanstor/client/client_host.go rename to storage/oceanstorage/base/client_host.go index 4cc96fd4..3d584f1f 100644 --- a/storage/oceanstor/client/client_host.go +++ b/storage/oceanstorage/base/client_host.go @@ -1,5 +1,5 @@ /* - * Copyright (c) Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2022-2024. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,14 +14,15 @@ * limitations under the License. */ -package client +// Package base provide base operations for oceanstor and oceandisk storage +package base import ( "context" "errors" "fmt" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( @@ -69,8 +70,13 @@ type Host interface { RemoveHostFromGroup(ctx context.Context, hostID, hostGroupID string) error } +// HostClient defines client implements the Host interface +type HostClient struct { + RestClientInterface +} + // AddHostToGroup used for add host to group -func (cli *BaseClient) AddHostToGroup(ctx context.Context, hostID, hostGroupID string) error { +func (cli *HostClient) AddHostToGroup(ctx context.Context, hostID, hostGroupID string) error { data := map[string]interface{}{ "ID": hostGroupID, "ASSOCIATEOBJTYPE": 21, @@ -95,7 +101,7 @@ func (cli *BaseClient) AddHostToGroup(ctx context.Context, hostID, hostGroupID s } // RemoveHostFromGroup used for remove host from group -func (cli *BaseClient) RemoveHostFromGroup(ctx context.Context, hostID, hostGroupID string) error { +func (cli *HostClient) RemoveHostFromGroup(ctx context.Context, hostID, hostGroupID string) error { data := map[string]interface{}{ "ID": hostGroupID, "ASSOCIATEOBJTYPE": 21, @@ -120,7 +126,7 @@ func (cli *BaseClient) RemoveHostFromGroup(ctx context.Context, hostID, hostGrou } // QueryAssociateHostGroup used for query associate host group -func (cli *BaseClient) QueryAssociateHostGroup(ctx context.Context, objType int, objID string) ([]interface{}, error) { +func (cli *HostClient) QueryAssociateHostGroup(ctx context.Context, objType int, objID string) ([]interface{}, error) { url := fmt.Sprintf("/hostgroup/associate?ASSOCIATEOBJTYPE=%d&ASSOCIATEOBJID=%s", objType, objID) resp, err := cli.Get(ctx, url, nil) if err != nil { @@ -145,7 +151,7 @@ func (cli *BaseClient) QueryAssociateHostGroup(ctx context.Context, objType int, } // CreateHost used for create host -func (cli *BaseClient) CreateHost(ctx context.Context, name string) (map[string]interface{}, error) { +func (cli *HostClient) CreateHost(ctx context.Context, name string) (map[string]interface{}, error) { data := map[string]interface{}{ "NAME": name, "OPERATIONSYSTEM": 0, @@ -174,7 +180,7 @@ func (cli *BaseClient) CreateHost(ctx context.Context, name string) (map[string] } // UpdateHost used for update host -func (cli *BaseClient) UpdateHost(ctx context.Context, id string, alua map[string]interface{}) error { +func (cli *HostClient) UpdateHost(ctx context.Context, id string, alua map[string]interface{}) error { url := fmt.Sprintf("/host/%s", id) data := map[string]interface{}{} @@ -200,7 +206,7 @@ func (cli *BaseClient) UpdateHost(ctx context.Context, id string, alua map[strin } // GetHostByName used to get host by name -func (cli *BaseClient) GetHostByName(ctx context.Context, name string) (map[string]interface{}, error) { +func (cli *HostClient) GetHostByName(ctx context.Context, name string) (map[string]interface{}, error) { url := fmt.Sprintf("/host?filter=NAME::%s&range=[0-100]", name) resp, err := cli.Get(ctx, url, nil) if err != nil { @@ -235,7 +241,7 @@ func (cli *BaseClient) GetHostByName(ctx context.Context, name string) (map[stri } // DeleteHost used for delete host by id -func (cli *BaseClient) DeleteHost(ctx context.Context, id string) error { +func (cli *HostClient) DeleteHost(ctx context.Context, id string) error { url := fmt.Sprintf("/host/%s", id) resp, err := cli.Delete(ctx, url, nil) if err != nil { @@ -256,7 +262,7 @@ func (cli *BaseClient) DeleteHost(ctx context.Context, id string) error { } // CreateHostGroup used for create host group -func (cli *BaseClient) CreateHostGroup(ctx context.Context, name string) (map[string]interface{}, error) { +func (cli *HostClient) CreateHostGroup(ctx context.Context, name string) (map[string]interface{}, error) { data := map[string]interface{}{ "NAME": name, } @@ -283,7 +289,7 @@ func (cli *BaseClient) CreateHostGroup(ctx context.Context, name string) (map[st } // GetHostGroupByName used for get host group by name -func (cli *BaseClient) GetHostGroupByName(ctx context.Context, name string) (map[string]interface{}, error) { +func (cli *HostClient) GetHostGroupByName(ctx context.Context, name string) (map[string]interface{}, error) { url := fmt.Sprintf("/hostgroup?filter=NAME::%s", name) resp, err := cli.Get(ctx, url, nil) if err != nil { @@ -318,7 +324,7 @@ func (cli *BaseClient) GetHostGroupByName(ctx context.Context, name string) (map } // DeleteHostGroup used for delete host group -func (cli *BaseClient) DeleteHostGroup(ctx context.Context, id string) error { +func (cli *HostClient) DeleteHostGroup(ctx context.Context, id string) error { url := fmt.Sprintf("/hostgroup/%s", id) resp, err := cli.Delete(ctx, url, nil) if err != nil { diff --git a/storage/oceanstor/client/client_iscsi.go b/storage/oceanstorage/base/client_iscsi.go similarity index 84% rename from storage/oceanstor/client/client_iscsi.go rename to storage/oceanstorage/base/client_iscsi.go index 9051cc49..d92ac63f 100644 --- a/storage/oceanstor/client/client_iscsi.go +++ b/storage/oceanstorage/base/client_iscsi.go @@ -1,5 +1,5 @@ /* - * Copyright (c) Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2022-2024. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,8 @@ * limitations under the License. */ -package client +// Package base provide base operations for oceanstor and oceandisk storage +package base import ( "context" @@ -22,10 +23,12 @@ import ( "fmt" "strings" - pkgUtils "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/utils/log" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) +const objectIdNotUnique int64 = 1077948997 + // Iscsi defines interfaces for iscsi operations type Iscsi interface { // GetIscsiInitiator used for get iscsi initiator @@ -44,8 +47,13 @@ type Iscsi interface { GetISCSIHostLink(ctx context.Context, hostID string) ([]interface{}, error) } +// IscsiClient defines client implements the Iscsi interface +type IscsiClient struct { + RestClientInterface +} + // GetISCSIHostLink used for get iscsi host link -func (cli *BaseClient) GetISCSIHostLink(ctx context.Context, hostID string) ([]interface{}, error) { +func (cli *IscsiClient) GetISCSIHostLink(ctx context.Context, hostID string) ([]interface{}, error) { url := fmt.Sprintf("/host_link?INITIATOR_TYPE=222&PARENTID=%s", hostID) resp, err := cli.Get(ctx, url, nil) if err != nil { @@ -70,7 +78,7 @@ func (cli *BaseClient) GetISCSIHostLink(ctx context.Context, hostID string) ([]i } // GetIscsiInitiator used for get iscsi initiator -func (cli *BaseClient) GetIscsiInitiator(ctx context.Context, initiator string) (map[string]interface{}, error) { +func (cli *IscsiClient) GetIscsiInitiator(ctx context.Context, initiator string) (map[string]interface{}, error) { id := strings.Replace(initiator, ":", "\\:", -1) url := fmt.Sprintf("/iscsi_initiator?filter=ID::%s", id) resp, err := cli.Get(ctx, url, nil) @@ -101,7 +109,7 @@ func (cli *BaseClient) GetIscsiInitiator(ctx context.Context, initiator string) } // GetIscsiInitiatorByID used for get iscsi initiator by id -func (cli *BaseClient) GetIscsiInitiatorByID(ctx context.Context, initiator string) (map[string]interface{}, error) { +func (cli *IscsiClient) GetIscsiInitiatorByID(ctx context.Context, initiator string) (map[string]interface{}, error) { id := strings.Replace(initiator, ":", "\\:", -1) url := fmt.Sprintf("/iscsi_initiator/%s", id) resp, err := cli.Get(ctx, url, nil) @@ -123,7 +131,7 @@ func (cli *BaseClient) GetIscsiInitiatorByID(ctx context.Context, initiator stri } // AddIscsiInitiator used for add iscsi initiator -func (cli *BaseClient) AddIscsiInitiator(ctx context.Context, initiator string) (map[string]interface{}, error) { +func (cli *IscsiClient) AddIscsiInitiator(ctx context.Context, initiator string) (map[string]interface{}, error) { data := map[string]interface{}{ "ID": initiator, } @@ -151,7 +159,7 @@ func (cli *BaseClient) AddIscsiInitiator(ctx context.Context, initiator string) } // UpdateIscsiInitiator used for update iscsi initiator -func (cli *BaseClient) UpdateIscsiInitiator(ctx context.Context, initiator string, alua map[string]interface{}) error { +func (cli *IscsiClient) UpdateIscsiInitiator(ctx context.Context, initiator string, alua map[string]interface{}) error { url := fmt.Sprintf("/iscsi_initiator/%s", initiator) data := map[string]interface{}{} @@ -185,7 +193,7 @@ func (cli *BaseClient) UpdateIscsiInitiator(ctx context.Context, initiator strin } // AddIscsiInitiatorToHost used for add iscsi initiator to host -func (cli *BaseClient) AddIscsiInitiatorToHost(ctx context.Context, initiator, hostID string) error { +func (cli *IscsiClient) AddIscsiInitiatorToHost(ctx context.Context, initiator, hostID string) error { url := fmt.Sprintf("/iscsi_initiator/%s", initiator) data := map[string]interface{}{ "PARENTTYPE": 21, @@ -206,7 +214,7 @@ func (cli *BaseClient) AddIscsiInitiatorToHost(ctx context.Context, initiator, h } // GetIscsiTgtPort used for get iscsi target port -func (cli *BaseClient) GetIscsiTgtPort(ctx context.Context) ([]interface{}, error) { +func (cli *IscsiClient) GetIscsiTgtPort(ctx context.Context) ([]interface{}, error) { resp, err := cli.Get(ctx, "/iscsi_tgt_port", nil) if err != nil { return nil, err diff --git a/storage/oceanstor/client/client_mapping.go b/storage/oceanstorage/base/client_mapping.go similarity index 85% rename from storage/oceanstor/client/client_mapping.go rename to storage/oceanstorage/base/client_mapping.go index 7bd4375b..49e431ea 100644 --- a/storage/oceanstor/client/client_mapping.go +++ b/storage/oceanstorage/base/client_mapping.go @@ -1,5 +1,5 @@ /* - * Copyright (c) Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2022-2024. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,15 +14,16 @@ * limitations under the License. */ -package client +// Package base provide base operations for oceanstor and oceandisk storage +package base import ( "context" "errors" "fmt" - pkgUtils "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/utils/log" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( @@ -47,8 +48,13 @@ type Mapping interface { AddGroupToMapping(ctx context.Context, groupType int, groupID, mappingID string) error } +// MappingClient defines client implements the Mapping interface +type MappingClient struct { + RestClientInterface +} + // CreateMapping used for create mapping -func (cli *BaseClient) CreateMapping(ctx context.Context, name string) (map[string]interface{}, error) { +func (cli *MappingClient) CreateMapping(ctx context.Context, name string) (map[string]interface{}, error) { data := map[string]interface{}{ "NAME": name, } @@ -75,7 +81,7 @@ func (cli *BaseClient) CreateMapping(ctx context.Context, name string) (map[stri } // GetMappingByName used for get mapping by name -func (cli *BaseClient) GetMappingByName(ctx context.Context, name string) (map[string]interface{}, error) { +func (cli *MappingClient) GetMappingByName(ctx context.Context, name string) (map[string]interface{}, error) { url := fmt.Sprintf("/mappingview?filter=NAME::%s", name) resp, err := cli.Get(ctx, url, nil) if err != nil { @@ -110,7 +116,7 @@ func (cli *BaseClient) GetMappingByName(ctx context.Context, name string) (map[s } // DeleteMapping used for delete mapping -func (cli *BaseClient) DeleteMapping(ctx context.Context, id string) error { +func (cli *MappingClient) DeleteMapping(ctx context.Context, id string) error { url := fmt.Sprintf("/mappingview/%s", id) resp, err := cli.Delete(ctx, url, nil) if err != nil { @@ -131,7 +137,7 @@ func (cli *BaseClient) DeleteMapping(ctx context.Context, id string) error { } // AddGroupToMapping used for add group to mapping -func (cli *BaseClient) AddGroupToMapping(ctx context.Context, groupType int, groupID, mappingID string) error { +func (cli *MappingClient) AddGroupToMapping(ctx context.Context, groupType int, groupID, mappingID string) error { data := map[string]interface{}{ "ID": mappingID, "ASSOCIATEOBJTYPE": groupType, @@ -157,7 +163,7 @@ func (cli *BaseClient) AddGroupToMapping(ctx context.Context, groupType int, gro } // RemoveGroupFromMapping used for remove group from mapping -func (cli *BaseClient) RemoveGroupFromMapping(ctx context.Context, groupType int, groupID, mappingID string) error { +func (cli *MappingClient) RemoveGroupFromMapping(ctx context.Context, groupType int, groupID, mappingID string) error { data := map[string]interface{}{ "ID": mappingID, "ASSOCIATEOBJTYPE": groupType, diff --git a/storage/oceanstor/client/client_qos.go b/storage/oceanstorage/base/client_qos.go similarity index 72% rename from storage/oceanstor/client/client_qos.go rename to storage/oceanstorage/base/client_qos.go index a17f10ee..d0b41788 100644 --- a/storage/oceanstor/client/client_qos.go +++ b/storage/oceanstorage/base/client_qos.go @@ -14,19 +14,24 @@ * limitations under the License. */ -package client +// Package base provide base operations for oceanstor and oceandisk storage +package base import ( "context" "fmt" + "strconv" "time" - pkgUtils "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/api" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( smartQosAlreadyExist int64 = 1077948993 + queryCountPerBatch = 100 ) // Qos defines interfaces for qos operations @@ -45,6 +50,14 @@ type Qos interface { ActivateQos(ctx context.Context, qosID, vStoreID string) error // DeactivateQos used for deactivate qos DeactivateQos(ctx context.Context, qosID, vStoreID string) error + // GetAllQos used for get all qos + GetAllQos(ctx context.Context) ([]map[string]interface{}, error) + getSystemUTCTime(ctx context.Context) (int64, error) +} + +// QosClient defines client implements the Qos interface +type QosClient struct { + RestClientInterface } // CreateQoSArgs is the arguments to create QoS @@ -57,7 +70,7 @@ type CreateQoSArgs struct { } // CreateQos used for create qos -func (cli *BaseClient) CreateQos(ctx context.Context, args CreateQoSArgs) (map[string]any, error) { +func (cli *QosClient) CreateQos(ctx context.Context, args CreateQoSArgs) (map[string]any, error) { utcTime, err := cli.getSystemUTCTime(ctx) if err != nil { @@ -113,7 +126,7 @@ func (cli *BaseClient) CreateQos(ctx context.Context, args CreateQoSArgs) (map[s } // ActivateQos used for active qos -func (cli *BaseClient) ActivateQos(ctx context.Context, qosID, vStoreID string) error { +func (cli *QosClient) ActivateQos(ctx context.Context, qosID, vStoreID string) error { data := map[string]interface{}{ "ID": qosID, "ENABLESTATUS": "true", @@ -137,7 +150,7 @@ func (cli *BaseClient) ActivateQos(ctx context.Context, qosID, vStoreID string) } // DeactivateQos used for deactivate qos -func (cli *BaseClient) DeactivateQos(ctx context.Context, qosID, vStoreID string) error { +func (cli *QosClient) DeactivateQos(ctx context.Context, qosID, vStoreID string) error { data := map[string]interface{}{ "ID": qosID, "ENABLESTATUS": "false", @@ -161,7 +174,7 @@ func (cli *BaseClient) DeactivateQos(ctx context.Context, qosID, vStoreID string } // DeleteQos used for delete qos -func (cli *BaseClient) DeleteQos(ctx context.Context, qosID, vStoreID string) error { +func (cli *QosClient) DeleteQos(ctx context.Context, qosID, vStoreID string) error { url := fmt.Sprintf("/ioclass/%s", qosID) var data = make(map[string]interface{}) if vStoreID != "" { @@ -182,7 +195,7 @@ func (cli *BaseClient) DeleteQos(ctx context.Context, qosID, vStoreID string) er } // GetQosByName used for get qos by name -func (cli *BaseClient) GetQosByName(ctx context.Context, name, vStoreID string) (map[string]interface{}, error) { +func (cli *QosClient) GetQosByName(ctx context.Context, name, vStoreID string) (map[string]interface{}, error) { url := fmt.Sprintf("/ioclass?filter=NAME::%s", name) var data = make(map[string]interface{}) if vStoreID != "" { @@ -219,7 +232,7 @@ func (cli *BaseClient) GetQosByName(ctx context.Context, name, vStoreID string) } // GetQosByID used for get qos by id -func (cli *BaseClient) GetQosByID(ctx context.Context, qosID, vStoreID string) (map[string]interface{}, error) { +func (cli *QosClient) GetQosByID(ctx context.Context, qosID, vStoreID string) (map[string]interface{}, error) { url := fmt.Sprintf("/ioclass/%s", qosID) var data = make(map[string]interface{}) if vStoreID != "" { @@ -243,7 +256,7 @@ func (cli *BaseClient) GetQosByID(ctx context.Context, qosID, vStoreID string) ( } // UpdateQos used for update qos -func (cli *BaseClient) UpdateQos(ctx context.Context, qosID, vStoreID string, params map[string]interface{}) error { +func (cli *QosClient) UpdateQos(ctx context.Context, qosID, vStoreID string, params map[string]interface{}) error { url := fmt.Sprintf("/ioclass/%s", qosID) if vStoreID != "" { params["vstoreId"] = vStoreID @@ -260,3 +273,40 @@ func (cli *BaseClient) UpdateQos(ctx context.Context, qosID, vStoreID string, pa return nil } + +// GetAllQos used for get all qos +func (cli *QosClient) GetAllQos(ctx context.Context) ([]map[string]interface{}, error) { + return GetBatchObjs(ctx, cli.RestClientInterface, api.GetAllQos) +} + +func (cli *QosClient) getSystemUTCTime(ctx context.Context) (int64, error) { + resp, err := cli.Get(ctx, "/system_utc_time", nil) + if err != nil { + return 0, err + } + + code := int64(resp.Error["code"].(float64)) + if code != 0 { + return 0, pkgUtils.Errorf(ctx, "get system UTC time error: %d", code) + } + + if resp.Data == nil { + return 0, pkgUtils.Errorf(ctx, "can not get the system UTC time") + } + + respData, ok := resp.Data.(map[string]interface{}) + if !ok { + return 0, pkgUtils.Errorf(ctx, "the response data is not a map[string]interface{}") + } + + utcTime, ok := respData["CMO_SYS_UTC_TIME"].(string) + if !ok { + return 0, pkgUtils.Errorf(ctx, "The CMO_SYS_UTC_TIME is not in respData %v", respData) + } + + resTime, err := strconv.ParseInt(utcTime, constants.DefaultIntBase, constants.DefaultIntBitSize) + if err != nil { + return 0, err + } + return resTime, nil +} diff --git a/storage/oceanstor/client/client_roce.go b/storage/oceanstorage/base/client_roce.go similarity index 88% rename from storage/oceanstor/client/client_roce.go rename to storage/oceanstorage/base/client_roce.go index 496db4a2..759315a0 100644 --- a/storage/oceanstor/client/client_roce.go +++ b/storage/oceanstorage/base/client_roce.go @@ -1,5 +1,5 @@ /* - * Copyright (c) Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2022-2024. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,7 +14,8 @@ * limitations under the License. */ -package client +// Package base provide base operations for oceanstor and oceandisk storage +package base import ( "context" @@ -22,8 +23,8 @@ import ( URL "net/url" "strings" - pkgUtils "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/utils/log" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // RoCE defines interfaces for RoCE operations @@ -40,8 +41,13 @@ type RoCE interface { GetRoCEPortalByIP(ctx context.Context, tgtPortal string) (map[string]interface{}, error) } +// RoCEClient defines client implements the RoCE interface +type RoCEClient struct { + RestClientInterface +} + // GetRoCEInitiator used for get RoCE initiator -func (cli *BaseClient) GetRoCEInitiator(ctx context.Context, initiator string) (map[string]interface{}, error) { +func (cli *RoCEClient) GetRoCEInitiator(ctx context.Context, initiator string) (map[string]interface{}, error) { id := URL.QueryEscape(strings.Replace(initiator, ":", "\\:", -1)) url := fmt.Sprintf("/NVMe_over_RoCE_initiator?filter=ID::%s", id) resp, err := cli.Get(ctx, url, nil) @@ -74,7 +80,7 @@ func (cli *BaseClient) GetRoCEInitiator(ctx context.Context, initiator string) ( } // GetRoCEInitiatorByID used for get RoCE initiator by id -func (cli *BaseClient) GetRoCEInitiatorByID(ctx context.Context, initiator string) (map[string]interface{}, error) { +func (cli *RoCEClient) GetRoCEInitiatorByID(ctx context.Context, initiator string) (map[string]interface{}, error) { id := URL.QueryEscape(strings.Replace(initiator, ":", "\\:", -1)) url := fmt.Sprintf("/NVMe_over_RoCE_initiator/%s", id) resp, err := cli.Get(ctx, url, nil) @@ -95,7 +101,7 @@ func (cli *BaseClient) GetRoCEInitiatorByID(ctx context.Context, initiator strin } // AddRoCEInitiator used for add RoCE initiator -func (cli *BaseClient) AddRoCEInitiator(ctx context.Context, initiator string) (map[string]interface{}, error) { +func (cli *RoCEClient) AddRoCEInitiator(ctx context.Context, initiator string) (map[string]interface{}, error) { data := map[string]interface{}{ "ID": initiator, } @@ -122,7 +128,7 @@ func (cli *BaseClient) AddRoCEInitiator(ctx context.Context, initiator string) ( } // AddRoCEInitiatorToHost used for add RoCE initiator to host -func (cli *BaseClient) AddRoCEInitiatorToHost(ctx context.Context, initiator, hostID string) error { +func (cli *RoCEClient) AddRoCEInitiatorToHost(ctx context.Context, initiator, hostID string) error { data := map[string]interface{}{ "ID": hostID, "ASSOCIATEOBJTYPE": 57870, @@ -142,7 +148,7 @@ func (cli *BaseClient) AddRoCEInitiatorToHost(ctx context.Context, initiator, ho } // GetRoCEPortalByIP used for get RoCE portal by ip -func (cli *BaseClient) GetRoCEPortalByIP(ctx context.Context, tgtPortal string) (map[string]interface{}, error) { +func (cli *RoCEClient) GetRoCEPortalByIP(ctx context.Context, tgtPortal string) (map[string]interface{}, error) { url := fmt.Sprintf("/lif?filter=IPV4ADDR::%s", tgtPortal) resp, err := cli.Get(ctx, url, nil) if err != nil { diff --git a/storage/oceanstor/client/client_system.go b/storage/oceanstorage/base/client_system.go similarity index 64% rename from storage/oceanstor/client/client_system.go rename to storage/oceanstorage/base/client_system.go index fa143a6d..90cd8aff 100644 --- a/storage/oceanstor/client/client_system.go +++ b/storage/oceanstorage/base/client_system.go @@ -1,5 +1,5 @@ /* - * Copyright (c) Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2022-2024. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -14,16 +14,16 @@ * limitations under the License. */ -package client +// Package base provide base operations for oceanstor and oceandisk storage +package base import ( "context" "errors" "fmt" - netUrl "net/url" - pkgUtils "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/utils/log" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // System defines interfaces for system operations @@ -32,26 +32,21 @@ type System interface { GetPoolByName(ctx context.Context, name string) (map[string]interface{}, error) // GetAllPools used for get all pools GetAllPools(ctx context.Context) (map[string]interface{}, error) - // GetSystem used for get system info - GetSystem(ctx context.Context) (map[string]interface{}, error) // GetLicenseFeature used for get license feature GetLicenseFeature(ctx context.Context) (map[string]int, error) // GetRemoteDeviceBySN used for get remote device by sn GetRemoteDeviceBySN(ctx context.Context, sn string) (map[string]interface{}, error) // GetAllRemoteDevices used for get all remote devices GetAllRemoteDevices(ctx context.Context) ([]map[string]interface{}, error) - // GetDeviceSN used for get device sn - GetDeviceSN() string - // GetStorageVersion used for get storage version - GetStorageVersion() string - // GetCurrentSiteWwn used for get current site wwn - GetCurrentSiteWwn() string - // SetSystemInfo set system info - SetSystemInfo(ctx context.Context) error +} + +// SystemClient defines client implements the System interface +type SystemClient struct { + RestClientInterface } // GetPoolByName used for get pool by name -func (cli *BaseClient) GetPoolByName(ctx context.Context, name string) (map[string]interface{}, error) { +func (cli *SystemClient) GetPoolByName(ctx context.Context, name string) (map[string]interface{}, error) { url := fmt.Sprintf("/storagepool?filter=NAME::%s&range=[0-100]", name) resp, err := cli.Get(ctx, url, nil) if err != nil { @@ -86,7 +81,7 @@ func (cli *BaseClient) GetPoolByName(ctx context.Context, name string) (map[stri } // GetAllPools used for get all pools -func (cli *BaseClient) GetAllPools(ctx context.Context) (map[string]interface{}, error) { +func (cli *SystemClient) GetAllPools(ctx context.Context) (map[string]interface{}, error) { resp, err := cli.Get(ctx, "/storagepool", nil) if err != nil { return nil, err @@ -127,7 +122,7 @@ func (cli *BaseClient) GetAllPools(ctx context.Context) (map[string]interface{}, } // GetLicenseFeature used for get license feature -func (cli *BaseClient) GetLicenseFeature(ctx context.Context) (map[string]int, error) { +func (cli *SystemClient) GetLicenseFeature(ctx context.Context) (map[string]int, error) { resp, err := cli.Get(ctx, "/license/feature", nil) if err != nil { return nil, err @@ -162,29 +157,8 @@ func (cli *BaseClient) GetLicenseFeature(ctx context.Context) (map[string]int, e return result, nil } -// GetSystem used for get system info -func (cli *BaseClient) GetSystem(ctx context.Context) (map[string]interface{}, error) { - resp, err := cli.Get(ctx, "/system/", nil) - if err != nil { - return nil, err - } - - code := int64(resp.Error["code"].(float64)) - if code != 0 { - msg := fmt.Sprintf("Get system info error: %d", code) - return nil, errors.New(msg) - } - - respData, ok := resp.Data.(map[string]interface{}) - if !ok { - return nil, pkgUtils.Errorf(ctx, "convert respData to map failed, data: %v", resp.Data) - } - - return respData, nil -} - // GetRemoteDeviceBySN used for get remote device by sn -func (cli *BaseClient) GetRemoteDeviceBySN(ctx context.Context, sn string) (map[string]interface{}, error) { +func (cli *SystemClient) GetRemoteDeviceBySN(ctx context.Context, sn string) (map[string]interface{}, error) { resp, err := cli.Get(ctx, "/remote_device", nil) if err != nil { return nil, err @@ -219,43 +193,6 @@ func (cli *BaseClient) GetRemoteDeviceBySN(ctx context.Context, sn string) (map[ } // GetAllRemoteDevices used for get all remote devices -func (cli *BaseClient) GetAllRemoteDevices(ctx context.Context) ([]map[string]interface{}, error) { - return cli.getBatchObjs(ctx, "/remote_device", true) -} - -// GetDeviceSN used for get device sn -func (cli *BaseClient) GetDeviceSN() string { - return cli.DeviceId -} - -func (cli *BaseClient) setStorageVersion(systemInfo map[string]interface{}) { - storagePointVersion, ok := systemInfo["pointRelease"].(string) - if ok { - cli.StorageVersion = storagePointVersion - } -} - -// GetStorageVersion used for get storage version -func (cli *BaseClient) GetStorageVersion() string { - return cli.StorageVersion -} - -// GetCurrentSiteWwn used for get current site wwn -func (cli *BaseClient) GetCurrentSiteWwn() string { - return cli.CurrentSiteWwn -} - -// GetCurrentLifWwn used for get current lif wwn -func (cli *BaseClient) GetCurrentLifWwn() string { - return cli.CurrentLifWwn -} - -// GetCurrentLif used for get current lif wwn -func (cli *BaseClient) GetCurrentLif(ctx context.Context) string { - u, err := netUrl.Parse(cli.Url) - if err != nil { - log.AddContext(ctx).Errorf("parse url failed, error: %v", err) - return "" - } - return u.Hostname() +func (cli *SystemClient) GetAllRemoteDevices(ctx context.Context) ([]map[string]interface{}, error) { + return GetBatchObjs(ctx, cli.RestClientInterface, "/remote_device") } diff --git a/storage/oceanstor/client/client_system_types.go b/storage/oceanstorage/base/client_system_types.go similarity index 97% rename from storage/oceanstor/client/client_system_types.go rename to storage/oceanstorage/base/client_system_types.go index 8d589cd3..8b4d16fe 100644 --- a/storage/oceanstor/client/client_system_types.go +++ b/storage/oceanstorage/base/client_system_types.go @@ -14,7 +14,8 @@ * limitations under the License. */ -package client +// Package base provide base operations for oceanstor and oceandisk storage +package base // OceanstorSystem holds the system information of Oceanstor storage type OceanstorSystem struct { diff --git a/storage/oceanstorage/oceandisk/attacher/attacher.go b/storage/oceanstorage/oceandisk/attacher/attacher.go new file mode 100644 index 00000000..fce82eb2 --- /dev/null +++ b/storage/oceanstorage/oceandisk/attacher/attacher.go @@ -0,0 +1,324 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package attacher provide operations of volume attach +package attacher + +import ( + "context" + "errors" + "fmt" + + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/base" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/base/attacher" + oceandisk "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceandisk/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" +) + +// VolumeAttacherPlugin defines interfaces of attach operations +type VolumeAttacherPlugin interface { + ControllerAttach(context.Context, string, map[string]interface{}) (map[string]interface{}, error) + ControllerDetach(context.Context, string, map[string]interface{}) (string, error) +} + +// OceandiskAttacher implements interface VolumeAttacherPlugin +type OceandiskAttacher struct { + *attacher.AttachmentManager + Cli oceandisk.OceandiskClientInterface +} + +// OceanDiskAttacherConfig defines the configurations of OceandiskAttacher +type OceanDiskAttacherConfig struct { + Cli oceandisk.OceandiskClientInterface + Protocol string + Invoker string + Portals []string + Alua map[string]interface{} +} + +// NewOceanDiskAttacher provides a new oceandisk attacher +func NewOceanDiskAttacher(config OceanDiskAttacherConfig) *OceandiskAttacher { + baseAttacherConfig := attacher.AttachmentManagerConfig{ + Cli: config.Cli, + Protocol: config.Protocol, + Invoker: config.Invoker, + Portals: config.Portals, + Alua: config.Alua, + } + baseAttacher := attacher.NewAttachmentManager(baseAttacherConfig) + + return &OceandiskAttacher{ + AttachmentManager: baseAttacher, + Cli: config.Cli, + } +} + +// ControllerAttach attaches volume and maps namespace to host +func (p *OceandiskAttacher) ControllerAttach(ctx context.Context, namespaceName string, + parameters map[string]interface{}) (map[string]interface{}, error) { + host, err := p.GetHost(ctx, parameters, true) + if err != nil { + log.AddContext(ctx).Errorf("get host ID error: %v", err) + return nil, err + } + + hostID, ok := utils.GetValue[string](host, "ID") + if !ok { + return nil, errors.New("convert host[\"ID\"] to string failed") + } + hostName, ok := utils.GetValue[string](host, "NAME") + if !ok { + return nil, errors.New("convert host[\"NAME\"] to string failed") + } + + hostAlua := utils.GetAlua(ctx, p.Alua, hostName) + + if hostAlua != nil && p.needUpdateHost(host, hostAlua) { + err := p.Cli.UpdateHost(ctx, hostID, hostAlua) + if err != nil { + log.AddContext(ctx).Errorf("update host %s error: %v", hostID, err) + return nil, err + } + } + + if p.Protocol == "iscsi" { + _, err = p.AttachISCSI(ctx, hostID, parameters) + } else if p.Protocol == "fc" || p.Protocol == "fc-nvme" { + _, err = p.AttachFC(ctx, hostID, parameters) + } else if p.Protocol == "roce" { + _, err = p.AttachRoCE(ctx, hostID, parameters) + } + + if err != nil { + log.AddContext(ctx).Errorf("attach %s connection error: %v", p.Protocol, err) + return nil, err + } + + wwn, hostNamespaceId, err := p.doMapping(ctx, hostID, namespaceName) + if err != nil { + log.AddContext(ctx).Errorf("mapping Namespace %s to host %s error: %v", namespaceName, hostID, err) + return nil, err + } + + return p.GetMappingProperties(ctx, wwn, hostNamespaceId, parameters) +} + +func (p *OceandiskAttacher) needUpdateHost(host map[string]interface{}, hostAlua map[string]interface{}) bool { + accessMode, ok := hostAlua["accessMode"] + if !ok { + return false + } + + if accessMode != host["accessMode"] { + return true + } + + return false +} + +func (p *OceandiskAttacher) doMapping(ctx context.Context, hostID, namespaceName string) (string, string, error) { + namespace, err := p.Cli.GetNamespaceByName(ctx, namespaceName) + if err != nil { + return "", "", err + } + if len(namespace) == 0 { + return "", "", fmt.Errorf("namespace %s not exist for attaching", namespaceName) + } + + namespaceID, ok := utils.GetValue[string](namespace, "ID") + if !ok { + return "", "", fmt.Errorf("convert namespaceID to string failed, data: %v", namespace["ID"]) + } + mappingID, err := p.CreateMapping(ctx, hostID) + if err != nil { + return "", "", fmt.Errorf("create mapping for host %s error: %v", hostID, err) + } + + err = p.CreateHostGroup(ctx, hostID, mappingID) + if err != nil { + return "", "", fmt.Errorf("create host group for host %s error: %v", hostID, err) + } + + err = p.createNamespaceGroup(ctx, namespaceID, hostID, mappingID) + if err != nil { + return "", "", fmt.Errorf("create namespace group for host %s error: %v", hostID, err) + } + + namespaceUniqueId, err := utils.GetLunUniqueId(ctx, p.Protocol, namespace) + if err != nil { + return "", "", err + } + + hostNamespaceId, err := p.Cli.GetHostNamespaceId(ctx, hostID, namespaceID) + if err != nil { + return "", "", err + } + + return namespaceUniqueId, hostNamespaceId, nil +} + +func (p *OceandiskAttacher) createNamespaceGroup(ctx context.Context, namespaceID, hostID, mappingID string) error { + var err error + var namespaceGroup map[string]interface{} + + namespaceGroupsByNamespaceID, err := p.Cli.QueryAssociateNamespaceGroup(ctx, + oceandisk.AssociateObjTypeNamespace, namespaceID) + if err != nil { + return fmt.Errorf("query associated namespace groups of namespace %s error: %v", namespaceID, err) + } + + namespaceGroupName := p.getNamespaceGroupName(hostID) + for _, i := range namespaceGroupsByNamespaceID { + group, ok := i.(map[string]interface{}) + if !ok { + log.AddContext(ctx).Warningf("convert group to map failed, data: %v", i) + continue + } + if group["NAME"].(string) == namespaceGroupName { + namespaceGroupID, ok := utils.GetValue[string](group, "ID") + if !ok { + return errors.New("convert group[\"ID\"] to string failed") + } + return p.addToNamespaceGroupMapping(ctx, namespaceGroupName, namespaceGroupID, mappingID) + } + } + + namespaceGroup, err = p.Cli.GetNamespaceGroupByName(ctx, namespaceGroupName) + if err != nil { + return fmt.Errorf("get namespacegroup by name %s error: %v", namespaceGroupName, err) + } + if len(namespaceGroup) == 0 { + namespaceGroup, err = p.Cli.CreateNamespaceGroup(ctx, namespaceGroupName) + if err != nil { + log.AddContext(ctx).Errorf("create namespacegroup %s error: %v", namespaceGroupName, err) + return err + } + } + + namespaceGroupID, ok := utils.GetValue[string](namespaceGroup, "ID") + if !ok { + return errors.New("createNamespaceGroup failed, caused by not found namespace group id") + } + err = p.Cli.AddNamespaceToGroup(ctx, namespaceID, namespaceGroupID) + if err != nil { + return fmt.Errorf("add namespace %s to group %s error: %v", namespaceID, namespaceGroupID, err) + } + + return p.addToNamespaceGroupMapping(ctx, namespaceGroupName, namespaceGroupID, mappingID) +} + +// getNamespaceGroupName generates namespace group name +func (p *OceandiskAttacher) getNamespaceGroupName(postfix string) string { + return fmt.Sprintf("k8s_%s_namespacegroup_%s", p.Invoker, postfix) +} + +func (p *OceandiskAttacher) addToNamespaceGroupMapping(ctx context.Context, + groupName, groupID, mappingID string) error { + namespaceGroupsByMappingID, err := p.Cli.QueryAssociateNamespaceGroup(ctx, + base.AssociateObjTypeMapping, mappingID) + if err != nil { + return fmt.Errorf("query associated namespace groups of mapping %s error: %v", mappingID, err) + } + + for _, i := range namespaceGroupsByMappingID { + group, ok := i.(map[string]interface{}) + if !ok { + return fmt.Errorf("invalid group type. Expected 'map[string]interface{}', found %T", i) + } + if group["NAME"].(string) == groupName { + return nil + } + } + + err = p.Cli.AddGroupToMapping(ctx, oceandisk.AssociateObjTypeNamespaceGroup, groupID, mappingID) + if err != nil { + return fmt.Errorf("add namespace group %s to mapping %s error: %v", groupID, mappingID, err) + } + + return nil +} + +// ControllerDetach detaches volume and unmaps namespace from host +func (p *OceandiskAttacher) ControllerDetach(ctx context.Context, + namespaceName string, parameters map[string]interface{}) (string, error) { + host, err := p.GetHost(ctx, parameters, false) + if err != nil { + return "", fmt.Errorf("get host ID error: %v", err) + } + if host == nil { + log.AddContext(ctx).Infof("host doesn't exist while detaching %s", namespaceName) + return "", nil + } + + hostID, ok := utils.GetValue[string](host, "ID") + if !ok { + return "", fmt.Errorf("convert hostID to string failed, data: %v", host["ID"]) + } + + wwn, err := p.doUnmapping(ctx, hostID, namespaceName) + if err != nil { + return "", err + } + + return wwn, nil +} + +func (p *OceandiskAttacher) doUnmapping(ctx context.Context, hostID, namespaceName string) (string, error) { + namespace, err := p.Cli.GetNamespaceByName(ctx, namespaceName) + if err != nil { + return "", fmt.Errorf("get namespace %s info error: %v", namespaceName, err) + } + if len(namespace) == 0 { + log.AddContext(ctx).Infof("namespace %s doesn't exist while detaching", namespaceName) + return "", nil + } + namespaceID, ok := utils.GetValue[string](namespace, "ID") + if !ok { + return "", fmt.Errorf("convert namespaceID to string failed, data: %v", namespace["ID"]) + } + namespaceGroupsByNamespaceID, err := p.Cli.QueryAssociateNamespaceGroup(ctx, + oceandisk.AssociateObjTypeNamespace, namespaceID) + if err != nil { + return "", fmt.Errorf("query associated namespacegroups of namespace %s error: %v", namespaceID, err) + } + + namespaceGroupName := p.getNamespaceGroupName(hostID) + for _, i := range namespaceGroupsByNamespaceID { + group, ok := i.(map[string]interface{}) + if !ok { + log.AddContext(ctx).Warningf("convert group to map failed, data: %v", i) + continue + } + if group["NAME"].(string) == namespaceGroupName { + namespaceGroupID, ok := utils.GetValue[string](group, "ID") + if !ok { + return "", fmt.Errorf("convert namespaceGroupID to string failed, data: %v", group["ID"]) + } + err = p.Cli.RemoveNamespaceFromGroup(ctx, namespaceID, namespaceGroupID) + if err != nil { + return "", fmt.Errorf("remove namespace %s from group %s error: %v", + namespaceID, namespaceGroupID, err) + } + } + } + + namespaceUniqueId, err := utils.GetLunUniqueId(ctx, p.Protocol, namespace) + if err != nil { + return "", fmt.Errorf("unmapping Namespace %s from host %s error: %v", namespaceName, hostID, err) + } + return namespaceUniqueId, nil +} diff --git a/storage/oceanstorage/oceandisk/attacher/attacher_test.go b/storage/oceanstorage/oceandisk/attacher/attacher_test.go new file mode 100644 index 00000000..c7eef7c4 --- /dev/null +++ b/storage/oceanstorage/oceandisk/attacher/attacher_test.go @@ -0,0 +1,505 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package attacher provide operations of volume attach +package attacher + +import ( + "context" + "errors" + "fmt" + "reflect" + "testing" + + "github.com/agiledragon/gomonkey/v2" + + baseAttacher "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/base/attacher" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceandisk/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" +) + +const ( + logName = "oceandisk_attacher.log" +) + +func TestMain(m *testing.M) { + log.MockInitLogging(logName) + defer log.MockStopLogging(logName) + + m.Run() +} + +func TestOceandiskAttacher_addToNamespaceGroupMapping_Success(t *testing.T) { + // arrange + groupName, groupID, mappingID := "group1", "1", "1" + newClient, err := client.NewClient(context.Background(), &client.NewClientConfig{}) + if err != nil { + return + } + attacher := NewOceanDiskAttacher(OceanDiskAttacherConfig{Cli: newClient}) + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodReturn(newClient, "QueryAssociateNamespaceGroup", []interface{}{}, nil). + ApplyMethodReturn(newClient, "AddGroupToMapping", nil) + + // action + getErr := attacher.addToNamespaceGroupMapping(context.Background(), groupName, groupID, mappingID) + + // assert + if getErr != nil { + t.Errorf("TestOceandiskAttacher_addToNamespaceGroupMapping_Success failed, "+ + "wantErr = nil, gotErr = %v", getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestOceandiskAttacher_addToNamespaceGroupMapping_AlreadyExistSuccess(t *testing.T) { + // arrange + groupName, groupID, mappingID := "group1", "1", "1" + newClient, err := client.NewClient(context.Background(), &client.NewClientConfig{}) + if err != nil { + return + } + attacher := NewOceanDiskAttacher(OceanDiskAttacherConfig{Cli: newClient}) + + var mockResponse []interface{} + group := map[string]interface{}{ + "NAME": groupName, + } + mockResponse = append(mockResponse, group) + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodReturn(newClient, "QueryAssociateNamespaceGroup", mockResponse, nil) + + // action + getErr := attacher.addToNamespaceGroupMapping(context.Background(), groupName, groupID, mappingID) + + // assert + if getErr != nil { + t.Errorf("TestOceandiskAttacher_addToNamespaceGroupMapping_AlreadyExistSuccess failed, "+ + "wantErr = nil, gotErr = %v", getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestOceandiskAttacher_addToNamespaceGroupMapping_QueryGroupError(t *testing.T) { + // arrange + groupName, groupID, mappingID := "group1", "1", "1" + newClient, err := client.NewClient(context.Background(), &client.NewClientConfig{}) + if err != nil { + return + } + attacher := NewOceanDiskAttacher(OceanDiskAttacherConfig{Cli: newClient}) + mockErr := errors.New("query group err") + wantErr := fmt.Errorf("query associated namespace groups of mapping %s error: %v", mappingID, mockErr) + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodReturn(newClient, "QueryAssociateNamespaceGroup", nil, mockErr) + + // action + getErr := attacher.addToNamespaceGroupMapping(context.Background(), groupName, groupID, mappingID) + + // assert + if !reflect.DeepEqual(wantErr, getErr) { + t.Errorf("TestOceandiskAttacher_addToNamespaceGroupMapping_QueryGroupError failed, "+ + "wantErr = %v, gotErr = %v", wantErr, getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestOceandiskAttacher_addToNamespaceGroupMapping_FormatGroupError(t *testing.T) { + // arrange + groupName, groupID, mappingID := "group1", "1", "1" + newClient, err := client.NewClient(context.Background(), &client.NewClientConfig{}) + if err != nil { + return + } + attacher := NewOceanDiskAttacher(OceanDiskAttacherConfig{Cli: newClient}) + + var mockResponse []interface{} + group := "invalid format group" + mockResponse = append(mockResponse, group) + + wantErr := fmt.Errorf("invalid group type. Expected 'map[string]interface{}', found %T", group) + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodReturn(newClient, "QueryAssociateNamespaceGroup", mockResponse, nil) + + // action + getErr := attacher.addToNamespaceGroupMapping(context.Background(), groupName, groupID, mappingID) + + // assert + if !reflect.DeepEqual(wantErr, getErr) { + t.Errorf("TestOceandiskAttacher_addToNamespaceGroupMapping_FormatGroupError failed, "+ + "wantErr = %v, gotErr = %v", wantErr, getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestOceandiskAttacher_addToNamespaceGroupMapping_AddGroupToMappingError(t *testing.T) { + // arrange + groupName, groupID, mappingID := "group1", "1", "1" + newClient, err := client.NewClient(context.Background(), &client.NewClientConfig{}) + if err != nil { + return + } + attacher := NewOceanDiskAttacher(OceanDiskAttacherConfig{Cli: newClient}) + mockErr := errors.New("add group to mapping err") + wantErr := fmt.Errorf("add namespace group %s to mapping %s error: %v", groupID, mappingID, mockErr) + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodReturn(newClient, "QueryAssociateNamespaceGroup", []interface{}{}, nil). + ApplyMethodReturn(newClient, "AddGroupToMapping", mockErr) + + // action + getErr := attacher.addToNamespaceGroupMapping(context.Background(), groupName, groupID, mappingID) + + // assert + if !reflect.DeepEqual(wantErr, getErr) { + t.Errorf("TestOceandiskAttacher_addToNamespaceGroupMapping_AddGroupToMappingError failed, "+ + "wantErr = %v, gotErr = %v", wantErr, getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestOceandiskAttacher_createNamespaceGroup_Success(t *testing.T) { + // arrange + namespaceId, hostID, mappingID := "1", "1", "1" + newClient, err := client.NewClient(context.Background(), &client.NewClientConfig{}) + if err != nil { + return + } + attacher := NewOceanDiskAttacher(OceanDiskAttacherConfig{Cli: newClient}) + + group := map[string]interface{}{ + "NAME": attacher.getNamespaceGroupName(hostID), + "ID": "1", + } + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodReturn(newClient, "QueryAssociateNamespaceGroup", []interface{}{}, nil). + ApplyMethodReturn(newClient, "GetNamespaceGroupByName", map[string]interface{}{}, nil). + ApplyMethodReturn(newClient, "CreateNamespaceGroup", group, nil). + ApplyMethodReturn(newClient, "AddNamespaceToGroup", nil). + ApplyPrivateMethod(&OceandiskAttacher{}, "addToNamespaceGroupMapping", + func(ctx context.Context, groupName, groupID, mappingID string) error { + return nil + }) + + // action + getErr := attacher.createNamespaceGroup(context.Background(), namespaceId, hostID, mappingID) + + // assert + if getErr != nil { + t.Errorf("TestOceandiskAttacher_createNamespaceGroup_Success failed, "+ + "wantErr = nil, gotErr = %v", getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestOceandiskAttacher_createNamespaceGroup_GroupAlreadyExistSuccess(t *testing.T) { + // arrange + namespaceId, hostID, mappingID := "1", "1", "1" + newClient, err := client.NewClient(context.Background(), &client.NewClientConfig{}) + if err != nil { + return + } + attacher := NewOceanDiskAttacher(OceanDiskAttacherConfig{Cli: newClient}) + + var mockResponse []interface{} + group := map[string]interface{}{ + "NAME": attacher.getNamespaceGroupName(hostID), + "ID": "1", + } + mockResponse = append(mockResponse, group) + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodReturn(newClient, "QueryAssociateNamespaceGroup", mockResponse, nil). + ApplyPrivateMethod(&OceandiskAttacher{}, "addToNamespaceGroupMapping", + func(ctx context.Context, groupName, groupID, mappingID string) error { + return nil + }) + + // action + getErr := attacher.createNamespaceGroup(context.Background(), namespaceId, hostID, mappingID) + + // assert + if getErr != nil { + t.Errorf("TestOceandiskAttacher_createNamespaceGroup_GroupAlreadyExistSuccess failed, "+ + "wantErr = nil, gotErr = %v", getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestOceandiskAttacher_createNamespaceGroup_GetGroupError(t *testing.T) { + // arrange + namespaceId, hostID, mappingID := "1", "1", "1" + newClient, err := client.NewClient(context.Background(), &client.NewClientConfig{}) + if err != nil { + return + } + attacher := NewOceanDiskAttacher(OceanDiskAttacherConfig{Cli: newClient}) + namespaceGroupName := attacher.getNamespaceGroupName(hostID) + mockErr := errors.New("get group err") + wantErr := fmt.Errorf("get namespacegroup by name %s error: %v", namespaceGroupName, mockErr) + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodReturn(newClient, "QueryAssociateNamespaceGroup", []interface{}{}, nil). + ApplyMethodReturn(newClient, "GetNamespaceGroupByName", + map[string]interface{}{}, mockErr) + + // action + getErr := attacher.createNamespaceGroup(context.Background(), namespaceId, hostID, mappingID) + + // assert + if !reflect.DeepEqual(wantErr, getErr) { + t.Errorf("TestOceandiskAttacher_createNamespaceGroup_GetGroupError failed, "+ + "wantErr = %v, gotErr = %v", wantErr, getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestOceandiskAttacher_doMapping_Success(t *testing.T) { + // arrange + hostID, namespaceName, uniqueId, hostNamespaceId := "1", "namespace1", "uniqueID1", "5" + newClient, err := client.NewClient(context.Background(), &client.NewClientConfig{}) + if err != nil { + return + } + attacher := NewOceanDiskAttacher(OceanDiskAttacherConfig{Cli: newClient, Protocol: "roce"}) + + namespace := map[string]interface{}{ + "NAME": namespaceName, + "ID": "1", + "NGUID": uniqueId, + } + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodReturn(newClient, "GetNamespaceByName", namespace, nil). + ApplyMethodReturn(&baseAttacher.AttachmentManager{}, "CreateMapping", "1", nil). + ApplyMethodReturn(&baseAttacher.AttachmentManager{}, "CreateHostGroup", nil). + ApplyPrivateMethod(&OceandiskAttacher{}, "createNamespaceGroup", + func(ctx context.Context, namespaceID, hostID, mappingID string) error { + return nil + }). + ApplyMethodReturn(newClient, "GetHostNamespaceId", hostNamespaceId, nil) + + // action + getNamespaceUniqueId, getHostNamespaceId, getErr := attacher.doMapping(context.Background(), hostID, namespaceName) + + // assert + if getNamespaceUniqueId != uniqueId || getHostNamespaceId != hostNamespaceId || getErr != nil { + t.Errorf("TestOceandiskAttacher_doMapping_Success failed, "+ + "wantUniqueId = %s, gotUniqueId = %s, wantHostNamespaceId = %s, gotHostNamespaceId = %s, "+ + "wantErr = nil, gotErr = %v", uniqueId, getNamespaceUniqueId, hostNamespaceId, getHostNamespaceId, getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestOceandiskAttacher_doMapping_NotExistError(t *testing.T) { + // arrange + hostID, namespaceName := "1", "namespace1" + newClient, err := client.NewClient(context.Background(), &client.NewClientConfig{}) + if err != nil { + return + } + attacher := NewOceanDiskAttacher(OceanDiskAttacherConfig{Cli: newClient}) + wantErr := fmt.Errorf("namespace %s not exist for attaching", namespaceName) + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodReturn(newClient, "GetNamespaceByName", map[string]interface{}{}, nil) + + // action + getNamespaceUniqueId, getHostNamespaceId, getErr := attacher.doMapping(context.Background(), hostID, namespaceName) + + // assert + if getNamespaceUniqueId != "" || getHostNamespaceId != "" || !reflect.DeepEqual(wantErr, getErr) { + t.Errorf("TestOceandiskAttacher_doMapping_Success failed, "+ + "wantUniqueId = , gotUniqueId = %s, wantHostNamespaceId = , gotHostNamespaceId = %s, "+ + "wantErr = %v, gotErr = %v", getNamespaceUniqueId, getHostNamespaceId, wantErr, getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestOceandiskAttacher_ControllerAttach_Success(t *testing.T) { + // arrange + namespaceName, hostName := "namespace1", "host1" + parameters := map[string]interface{}{} + newClient, err := client.NewClient(context.Background(), &client.NewClientConfig{}) + if err != nil { + return + } + attacher := NewOceanDiskAttacher(OceanDiskAttacherConfig{Cli: newClient}) + attacher.Alua = map[string]interface{}{ + hostName: map[string]interface{}{ + "accessMode": 0, + }, + } + wantRes := map[string]interface{}{} + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodReturn(&baseAttacher.AttachmentManager{}, + "GetHost", map[string]interface{}{"ID": "1", "NAME": hostName}, nil). + ApplyMethodReturn(newClient, "UpdateHost", nil). + ApplyMethodReturn(&baseAttacher.AttachmentManager{}, "AttachRoCE", map[string]interface{}{}, nil). + ApplyPrivateMethod(attacher, "doMapping", + func(ctx context.Context, hostID, namespaceName string) (string, string, error) { + return "", "", nil + }). + ApplyMethodReturn(&baseAttacher.AttachmentManager{}, "GetMappingProperties", wantRes, nil) + + // action + getRes, getErr := attacher.ControllerAttach(context.Background(), namespaceName, parameters) + + // assert + if !reflect.DeepEqual(wantRes, getRes) || getErr != nil { + t.Errorf("TestOceandiskAttacher_ControllerAttach_Success failed, "+ + "wantRes = %v, getRes = %s, wantErr = nil, gotErr = %v", wantRes, getRes, getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestOceandiskAttacher_doUnmapping_Success(t *testing.T) { + // arrange + hostID, namespaceName, uniqueId := "1", "namespace1", "uniqueID1" + newClient, err := client.NewClient(context.Background(), &client.NewClientConfig{}) + if err != nil { + return + } + attacher := NewOceanDiskAttacher(OceanDiskAttacherConfig{Cli: newClient, Protocol: "roce"}) + + namespace := map[string]interface{}{ + "NAME": namespaceName, + "ID": "1", + "NGUID": uniqueId, + } + var mockResponse []interface{} + group := map[string]interface{}{ + "NAME": attacher.getNamespaceGroupName(hostID), + "ID": "1", + } + mockResponse = append(mockResponse, group) + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodReturn(newClient, "GetNamespaceByName", namespace, nil). + ApplyMethodReturn(newClient, "QueryAssociateNamespaceGroup", mockResponse, nil). + ApplyMethodReturn(newClient, "RemoveNamespaceFromGroup", nil) + + // action + getWwn, getErr := attacher.doUnmapping(context.Background(), hostID, namespaceName) + + // assert + if getWwn != uniqueId || getErr != nil { + t.Errorf("TestOceandiskAttacher_doUnmapping_Success failed, "+ + "wantUniqueId = %s, gotUniqueId = %s, wantErr = nil, gotErr = %v", uniqueId, getWwn, getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestOceandiskAttacher_ControllerDetach_Success(t *testing.T) { + // arrange + namespaceName := "namespace1" + parameters := map[string]interface{}{} + newClient, err := client.NewClient(context.Background(), &client.NewClientConfig{}) + if err != nil { + return + } + attacher := NewOceanDiskAttacher(OceanDiskAttacherConfig{Cli: newClient}) + + wantWwn := "want wwn" + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodReturn(&baseAttacher.AttachmentManager{}, + "GetHost", map[string]interface{}{"ID": "1"}, nil). + ApplyPrivateMethod(attacher, "doUnmapping", + func(ctx context.Context, hostID, namespaceName string) (string, error) { + return wantWwn, nil + }) + + // action + getWwn, getErr := attacher.ControllerDetach(context.Background(), namespaceName, parameters) + + // assert + if wantWwn != getWwn || getErr != nil { + t.Errorf("TestOceandiskAttacher_ControllerDetach_Success failed, "+ + "wantWwn = %s, getWwn = %s, wantErr = nil, gotErr = %v", wantWwn, getWwn, getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} diff --git a/storage/oceanstorage/oceandisk/client/client.go b/storage/oceanstorage/oceandisk/client/client.go new file mode 100644 index 00000000..15e15bd4 --- /dev/null +++ b/storage/oceanstorage/oceandisk/client/client.go @@ -0,0 +1,167 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package client provides oceandisk storage client +package client + +import ( + "context" + "fmt" + + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/base" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" +) + +// OceandiskClientInterface defines interfaces for base client operations +type OceandiskClientInterface interface { + base.RestClientInterface + base.ApplicationType + base.FC + base.Host + base.Iscsi + base.Mapping + base.Qos + base.RoCE + base.System + + Namespace + NamespaceGroup + + GetBackendID() string + GetDeviceSN() string + GetStorageVersion() string +} + +// OceandiskClient implements OceandiskClientInterface +type OceandiskClient struct { + *base.ApplicationTypeClient + *base.FCClient + *base.HostClient + *base.IscsiClient + *base.MappingClient + *base.QosClient + *base.RoCEClient + *base.SystemClient + + *RestClient +} + +// NewClientConfig stores the information needed to create a new oceandisk client +type NewClientConfig struct { + Urls []string + User string + SecretName string + SecretNamespace string + ParallelNum string + BackendID string + UseCert bool + CertSecretMeta string + Storage string + Name string +} + +// NewClient inits a new client of oceandisk client +func NewClient(ctx context.Context, param *NewClientConfig) (*OceandiskClient, error) { + restClient, err := NewRestClient(ctx, param) + if err != nil { + return nil, err + } + + return &OceandiskClient{ + ApplicationTypeClient: &base.ApplicationTypeClient{RestClientInterface: restClient}, + FCClient: &base.FCClient{RestClientInterface: restClient}, + HostClient: &base.HostClient{RestClientInterface: restClient}, + IscsiClient: &base.IscsiClient{RestClientInterface: restClient}, + MappingClient: &base.MappingClient{RestClientInterface: restClient}, + QosClient: &base.QosClient{RestClientInterface: restClient}, + RoCEClient: &base.RoCEClient{RestClientInterface: restClient}, + SystemClient: &base.SystemClient{RestClientInterface: restClient}, + RestClient: restClient, + }, nil +} + +// ValidateLogin validates the login info +func (cli *OceandiskClient) ValidateLogin(ctx context.Context) error { + var resp base.Response + var err error + + password, err := utils.GetPasswordFromSecret(ctx, cli.SecretName, cli.SecretNamespace) + if err != nil { + return err + } + + data := map[string]interface{}{ + "username": cli.User, + "password": password, + "scope": base.LocalUserType, + } + + cli.DeviceId = "" + cli.Token = "" + for i, url := range cli.Urls { + cli.Url = url + "/deviceManager/rest" + log.AddContext(ctx).Infof("try to login %s", cli.Url) + resp, err = cli.BaseCall(ctx, "POST", "/xx/sessions", data) + if err == nil { + /* Sort the login Url to the last slot of san addresses, so that + if this connection error, next time will try other Url first. */ + cli.Urls[i], cli.Urls[len(cli.Urls)-1] = cli.Urls[len(cli.Urls)-1], cli.Urls[i] + break + } else if err.Error() != base.Unconnected { + log.AddContext(ctx).Errorf("login %s error", cli.Url) + break + } + + log.AddContext(ctx).Warningf("login %s error due to connection failure, gonna try another Url", cli.Url) + } + + if err != nil { + return err + } + + code, msg, err := utils.FormatRespErr(resp.Error) + if err != nil { + return fmt.Errorf("format login response data error: %v", err) + } + + if code != 0 { + return fmt.Errorf("validate login %s failed, error code: %d, error msg: %s", cli.Url, code, msg) + } + + cli.setDeviceIdFromRespData(ctx, resp) + + log.AddContext(ctx).Infof("validate login %s success", cli.Url) + return nil +} + +func (cli *OceandiskClient) setDeviceIdFromRespData(ctx context.Context, resp base.Response) { + respData, ok := resp.Data.(map[string]interface{}) + if !ok { + log.AddContext(ctx).Warningf("convert response data to map[string]interface{} failed, data type: [%T]", + resp.Data) + } + + cli.DeviceId, ok = utils.GetValue[string](respData, "deviceid") + if !ok { + log.AddContext(ctx).Warningf("can not convert deviceId type %T to string", respData["deviceid"]) + } + + cli.Token, ok = utils.GetValue[string](respData, "iBaseToken") + if !ok { + log.AddContext(ctx).Warningf("can not convert iBaseToken type %T to string", respData["iBaseToken"]) + } +} diff --git a/storage/oceanstorage/oceandisk/client/client_namespace.go b/storage/oceanstorage/oceandisk/client/client_namespace.go new file mode 100644 index 00000000..a34e62cc --- /dev/null +++ b/storage/oceanstorage/oceandisk/client/client_namespace.go @@ -0,0 +1,610 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package client provides oceandisk storage client +package client + +import ( + "context" + "encoding/json" + "fmt" + "strconv" + + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/api" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" +) + +const ( + objectNotExist int64 = 1077948996 + objectIdNotUnique int64 = 1077948997 + namespaceAlreadyInGroup int64 = 1077936862 + namespaceNotExist int64 = 1077936859 + parameterIncorrect int64 = 50331651 + objectNameAlreadyExist int64 = 1077948993 + + // NamespaceType is Associated object type of Namespace + NamespaceType = "11" + // AssociateObjTypeNamespace Namespace type + AssociateObjTypeNamespace = 11 + // AssociateObjTypeNamespaceGroup Namespace group type + AssociateObjTypeNamespaceGroup = 256 +) + +// Namespace defines interfaces for namespace operations +type Namespace interface { + // GetNamespaceByName used for get namespace by name + GetNamespaceByName(ctx context.Context, name string) (map[string]interface{}, error) + // GetNamespaceByID used for get namespace by id + GetNamespaceByID(ctx context.Context, id string) (map[string]interface{}, error) + // GetNamespaceCountOfHost used for get namespace count of host + GetNamespaceCountOfHost(ctx context.Context, hostID string) (int64, error) + // GetNamespaceCountOfMapping used for get namespace count of mapping by mapping id + GetNamespaceCountOfMapping(ctx context.Context, mappingID string) (int64, error) + // DeleteNamespace used for delete namespace by namespace id + DeleteNamespace(ctx context.Context, id string) error + // ExtendNamespace used for extend namespace + ExtendNamespace(ctx context.Context, namespaceID string, newCapacity int64) error + // CreateNamespace used for create namespace + CreateNamespace(ctx context.Context, params CreateNamespaceParams) (map[string]interface{}, error) + // GetHostNamespaceId used for get host namespace id + GetHostNamespaceId(ctx context.Context, hostID, namespaceID string) (string, error) + // UpdateNamespace used for update namespace + UpdateNamespace(ctx context.Context, namespaceID string, params map[string]interface{}) error +} + +// NamespaceGroup defines interfaces for namespacegroup operations +type NamespaceGroup interface { + // QueryAssociateNamespaceGroup used for query associate namespace group by object type and object id + QueryAssociateNamespaceGroup(ctx context.Context, objType int, objID string) ([]interface{}, error) + // GetNamespaceGroupByName used for get namespace group by name + GetNamespaceGroupByName(ctx context.Context, name string) (map[string]interface{}, error) + // DeleteNamespaceGroup used for delete namespace group by namespace group id + DeleteNamespaceGroup(ctx context.Context, id string) error + // RemoveNamespaceFromGroup used for remove namespace from group + RemoveNamespaceFromGroup(ctx context.Context, namespaceID, groupID string) error + // AddNamespaceToGroup used for add namespace to group + AddNamespaceToGroup(ctx context.Context, namespaceID string, groupID string) error + // CreateNamespaceGroup used for create namespace group + CreateNamespaceGroup(ctx context.Context, name string) (map[string]interface{}, error) +} + +// QueryAssociateNamespaceGroup used for query associate namespace group by object type and object id +func (cli *OceandiskClient) QueryAssociateNamespaceGroup(ctx context.Context, + objType int, objID string) ([]interface{}, error) { + url := fmt.Sprintf(api.QueryAssociateNamespaceGroup, objType, objID) + resp, err := cli.Get(ctx, url, nil) + if err != nil { + return nil, err + } + + code, msg, err := utils.FormatRespErr(resp.Error) + if err != nil { + return nil, err + } + + if code != 0 { + return nil, fmt.Errorf("associate query namespacegroup by obj %s of type %d failed, "+ + "error code: %d, error msg: %s", objID, objType, code, msg) + } + + if resp.Data == nil { + log.AddContext(ctx).Infof("obj %s of type %d doesn't associate to any namespacegroup", objID, objType) + return []interface{}{}, nil + } + + respData, ok := resp.Data.([]interface{}) + if !ok { + return nil, fmt.Errorf("convert respData to arr failed, data: %v", resp.Data) + } + return respData, nil +} + +// GetNamespaceByName used for get namespace by name +func (cli *OceandiskClient) GetNamespaceByName(ctx context.Context, name string) (map[string]interface{}, error) { + url := fmt.Sprintf(api.GetNamespaceByName, name) + resp, err := cli.Get(ctx, url, nil) + if err != nil { + return nil, err + } + + code, msg, err := utils.FormatRespErr(resp.Error) + if err != nil { + return nil, err + } + + if code != 0 { + return nil, fmt.Errorf("get namespace %s info failed, error code: %d, error msg: %s", name, code, msg) + } + + if resp.Data == nil { + log.AddContext(ctx).Infof("namespace %s does not exist, a nil list is got", name) + return map[string]interface{}{}, nil + } + + respData, ok := resp.Data.([]interface{}) + if !ok { + return nil, fmt.Errorf("convert respData to arr failed, data: %v", resp.Data) + } + if len(respData) <= 0 { + log.AddContext(ctx).Infof("namespace %s does not exist", name) + return map[string]interface{}{}, nil + } + + namespace, ok := respData[0].(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("convert namespace to map failed, data: %v", respData[0]) + } + return namespace, nil +} + +// GetNamespaceByID used for get namespace by id +func (cli *OceandiskClient) GetNamespaceByID(ctx context.Context, id string) (map[string]interface{}, error) { + url := fmt.Sprintf(api.GetNamespaceByID, id) + resp, err := cli.Get(ctx, url, nil) + if err != nil { + return nil, err + } + + code, msg, err := utils.FormatRespErr(resp.Error) + if err != nil { + return nil, err + } + + if code != 0 { + return nil, fmt.Errorf("get namespace %s info failed, error code: %d, error msg: %s", id, code, msg) + } + + namespace, ok := resp.Data.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("convert namespace to map failed, data: %v", resp.Data) + } + + return namespace, nil +} + +// AddNamespaceToGroup used for add namespace to group +func (cli *OceandiskClient) AddNamespaceToGroup(ctx context.Context, namespaceID string, groupID string) error { + data := map[string]interface{}{ + "ID": groupID, + "ASSOCIATEOBJTYPE": NamespaceType, + "ASSOCIATEOBJID": namespaceID, + } + + resp, err := cli.Post(ctx, api.AddNamespaceToGroup, data) + if err != nil { + return err + } + + code, msg, err := utils.FormatRespErr(resp.Error) + if err != nil { + return err + } + + if code == objectIdNotUnique || code == namespaceAlreadyInGroup { + log.AddContext(ctx).Infof("namespace %s is already in group %s", namespaceID, groupID) + return nil + } + + if code != 0 { + return fmt.Errorf("add namespace %s to group %s failed, "+ + "error code: %d, error msg: %s", namespaceID, groupID, code, msg) + } + + return nil +} + +// RemoveNamespaceFromGroup used for remove namespace from group +func (cli *OceandiskClient) RemoveNamespaceFromGroup(ctx context.Context, namespaceID, groupID string) error { + data := map[string]interface{}{ + "ID": groupID, + "ASSOCIATEOBJTYPE": NamespaceType, + "ASSOCIATEOBJID": namespaceID, + } + + resp, err := cli.Delete(ctx, api.RemoveNamespaceFromGroup, data) + if err != nil { + return err + } + + code, msg, err := utils.FormatRespErr(resp.Error) + if err != nil { + return err + } + + if code == objectNotExist { + log.AddContext(ctx).Infof("namespace %s is not in namespacegroup %s", namespaceID, groupID) + return nil + } + + if code != 0 { + return fmt.Errorf("remove namespace %s from group %s failed, "+ + "error code: %d, error msg: %s", namespaceID, groupID, code, msg) + } + + return nil +} + +// GetNamespaceGroupByName used for get namespace group by name +func (cli *OceandiskClient) GetNamespaceGroupByName(ctx context.Context, name string) (map[string]interface{}, error) { + url := fmt.Sprintf(api.GetNamespaceGroupByName, name) + resp, err := cli.Get(ctx, url, nil) + if err != nil { + return nil, err + } + + code, msg, err := utils.FormatRespErr(resp.Error) + if err != nil { + return nil, err + } + if code != 0 { + return nil, fmt.Errorf("get namespacegroup %s info failed, "+ + "error code: %d, error msg: %s", name, code, msg) + } + + if resp.Data == nil { + log.AddContext(ctx).Infof("namespacegroup %s does not exist, a nil list is got", name) + return map[string]interface{}{}, nil + } + + respData, ok := resp.Data.([]interface{}) + if !ok { + return nil, fmt.Errorf("convert respData to arr failed, data: %v", resp.Data) + } + + if len(respData) <= 0 { + log.AddContext(ctx).Infof("namespacegroup %s does not exist", name) + return map[string]interface{}{}, nil + } + + group, ok := respData[0].(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("convert group to arr failed, data: %v", respData[0]) + } + + return group, nil +} + +// CreateNamespaceGroup used for create namespace group +func (cli *OceandiskClient) CreateNamespaceGroup(ctx context.Context, name string) (map[string]interface{}, error) { + data := map[string]interface{}{ + "NAME": name, + } + resp, err := cli.Post(ctx, api.CreateNamespaceGroup, data) + if err != nil { + return nil, err + } + + code, msg, err := utils.FormatRespErr(resp.Error) + if err != nil { + return nil, err + } + + if code == objectNameAlreadyExist { + log.AddContext(ctx).Infof("namespacegroup %s already exists", name) + return cli.GetNamespaceGroupByName(ctx, name) + } + + if code != 0 { + return nil, fmt.Errorf("create namespacegroup %s failed, error code: %d, error msg: %s", name, code, msg) + } + + namespaceGroup, ok := resp.Data.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("convert namespaceGroup to map failed, data: %v", resp.Data) + } + return namespaceGroup, nil +} + +// DeleteNamespaceGroup used for delete namespace group by namespace group id +func (cli *OceandiskClient) DeleteNamespaceGroup(ctx context.Context, id string) error { + url := fmt.Sprintf(api.DeleteNamespaceGroup, id) + resp, err := cli.Delete(ctx, url, nil) + if err != nil { + return err + } + + code, msg, err := utils.FormatRespErr(resp.Error) + if err != nil { + return err + } + + if code == objectNotExist { + log.AddContext(ctx).Infof("namespacegroup %s does not exist while deleting", id) + return nil + } + if code != 0 { + return fmt.Errorf("delete namespacegroup %s failed, error code: %d, error msg: %s", id, code, msg) + } + + return nil +} + +// CreateNamespaceParams defines create namespace params +type CreateNamespaceParams struct { + Name string + ParentId string + Capacity int64 + Description string + WorkLoadTypeId string +} + +// MakeCreateNamespaceParams used to make parameters for CreateNamespace +func MakeCreateNamespaceParams(params map[string]interface{}) (*CreateNamespaceParams, error) { + namespaceName, ok := params["name"].(string) + if !ok { + return nil, fmt.Errorf("assert namespace name: %v to string failed", params["name"]) + } + + parentId, ok := params["poolID"].(string) + if !ok { + return nil, fmt.Errorf("assert poolID: %v to string failed", params["poolID"]) + } + + capacity, ok := params["capacity"].(int64) + if !ok { + return nil, fmt.Errorf("assert capacity: %v to int64 failed", params["capacity"]) + } + + description, ok := params["description"].(string) + if !ok { + return nil, fmt.Errorf("assert description: %v to string failed", params["capacity"]) + } + + // params["workloadTypeID"] may not exist. + // In this case, the value of workLoadTypeId is an empty string, which meets the expectation. + workLoadTypeId, _ := utils.GetValue[string](params, "workloadTypeID") + + return &CreateNamespaceParams{ + Name: namespaceName, + ParentId: parentId, + Capacity: capacity, + Description: description, + WorkLoadTypeId: workLoadTypeId, + }, nil +} + +// CreateNamespace used for create namespace +func (cli *OceandiskClient) CreateNamespace(ctx context.Context, + params CreateNamespaceParams) (map[string]interface{}, error) { + data := map[string]interface{}{ + "NAME": params.Name, + "PARENTID": params.ParentId, + "CAPACITY": params.Capacity, + "DESCRIPTION": params.Description, + } + + if params.WorkLoadTypeId != "" { + data["WORKLOADTYPEID"] = params.WorkLoadTypeId + } + + resp, err := cli.Post(ctx, api.CreateNamespace, data) + if err != nil { + return nil, err + } + + code, msg, err := utils.FormatRespErr(resp.Error) + if err != nil { + return nil, err + } + + if code == parameterIncorrect { + return nil, fmt.Errorf("create Namespace with incorrect parameters %v, "+ + "err code: %d, err msg: %s", data, code, msg) + } + + if code != 0 { + return nil, fmt.Errorf("create volume %v failed, error code: %d, error msg: %s", data, code, msg) + } + + respData, ok := resp.Data.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("convert respData to map failed, data: %v", resp.Data) + } + return respData, nil +} + +// DeleteNamespace used for delete namespace by namespace id +func (cli *OceandiskClient) DeleteNamespace(ctx context.Context, id string) error { + url := fmt.Sprintf(api.DeleteNamespace, id) + resp, err := cli.Delete(ctx, url, nil) + if err != nil { + return err + } + + code, msg, err := utils.FormatRespErr(resp.Error) + if err != nil { + return err + } + + if code == namespaceNotExist { + log.AddContext(ctx).Infof("namespace %s does not exist while deleting", id) + return nil + } + + if code != 0 { + return fmt.Errorf("delete namespace %s failed, error code: %d, error msg: %s", id, code, msg) + } + + return nil +} + +// ExtendNamespace used for extend namespace +func (cli *OceandiskClient) ExtendNamespace(ctx context.Context, namespaceID string, newCapacity int64) error { + data := map[string]interface{}{ + "CAPACITY": newCapacity, + "ID": namespaceID, + } + + resp, err := cli.Put(ctx, api.ExtendNamespace, data) + if err != nil { + return err + } + + code, msg, err := utils.FormatRespErr(resp.Error) + if err != nil { + return err + } + if code != 0 { + return fmt.Errorf("extend Namespace capacity to %d failed, "+ + "error code: %d, error msg: %s", newCapacity, code, msg) + } + + return nil +} + +// GetNamespaceCountOfMapping used for get namespace count of mapping by mapping id +func (cli *OceandiskClient) GetNamespaceCountOfMapping(ctx context.Context, mappingID string) (int64, error) { + url := fmt.Sprintf(api.GetNamespaceCountOfMapping, mappingID) + resp, err := cli.Get(ctx, url, nil) + if err != nil { + return 0, err + } + + code, msg, err := utils.FormatRespErr(resp.Error) + if err != nil { + return 0, err + } + if code != 0 { + return 0, fmt.Errorf("get mapped namespace count of mapping %s failed, "+ + "error code: %d, error msg: %s", mappingID, code, msg) + } + + respData, ok := resp.Data.(map[string]interface{}) + if !ok { + return 0, fmt.Errorf("convert respData to map failed, data: %v", resp.Data) + } + + countStr, ok := respData["COUNT"].(string) + if !ok { + return 0, fmt.Errorf("convert countStr to string failed, data: %v", respData["COUNT"]) + } + + count := utils.ParseIntWithDefault(countStr, constants.DefaultIntBase, constants.DefaultIntBitSize, 0) + return count, nil +} + +// GetNamespaceCountOfHost used for get namespace count of host +func (cli *OceandiskClient) GetNamespaceCountOfHost(ctx context.Context, hostID string) (int64, error) { + url := fmt.Sprintf(api.GetNamespaceCountOfHost, hostID) + resp, err := cli.Get(ctx, url, nil) + if err != nil { + return 0, err + } + + code, msg, err := utils.FormatRespErr(resp.Error) + if err != nil { + return 0, err + } + if code != 0 { + return 0, fmt.Errorf("get mapped namespace count of host %s failed, "+ + "error code: %d, error msg: %s", hostID, code, msg) + } + + respData, ok := resp.Data.(map[string]interface{}) + if !ok { + return 0, fmt.Errorf("convert respData to map failed, data: %v", resp.Data) + } + + countStr, ok := respData["COUNT"].(string) + if !ok { + return 0, fmt.Errorf("convert countStr to string failed, data: %v", respData["COUNT"]) + } + + count := utils.ParseIntWithDefault(countStr, constants.DefaultIntBase, constants.DefaultIntBitSize, 0) + return count, nil +} + +// GetHostNamespaceId used for get host namespace id +func (cli *OceandiskClient) GetHostNamespaceId(ctx context.Context, hostID, namespaceID string) (string, error) { + url := fmt.Sprintf(api.GetHostNamespaceId, hostID) + resp, err := cli.Get(ctx, url, nil) + if err != nil { + return "", err + } + + code, msg, err := utils.FormatRespErr(resp.Error) + if err != nil { + return "", err + } + if code != 0 { + return "", fmt.Errorf("get hostNamespaceId of host %s, namespace %s failed, "+ + "error code: %d, error msg: %s", hostID, namespaceID, code, msg) + } + + respData, ok := resp.Data.([]interface{}) + if !ok { + return "", fmt.Errorf("convert respData to arr failed, data: %v", resp.Data) + } + + var hostNamespaceId string + for _, i := range respData { + hostNamespaceInfo, ok := i.(map[string]interface{}) + if !ok { + log.AddContext(ctx).Warningf("convert hostNamespaceInfo to map failed, data: %v", i) + continue + } + + if hostNamespaceInfo["ID"].(string) == namespaceID { + var associateData map[string]interface{} + associateDataBytes := []byte(hostNamespaceInfo["ASSOCIATEMETADATA"].(string)) + err := json.Unmarshal(associateDataBytes, &associateData) + if err != nil { + return "", fmt.Errorf("unmarshal associateData fail while "+ + "getting the hostNamespaceId of host %s, namespace %s, error: %v", hostID, namespaceID, err) + } + + hostNamespaceID, ok := associateData["hostNamespaceID"] + if !ok { + return "", fmt.Errorf("hostNamesapceID field is not exist "+ + "in unmarshaled associateData of host %s, namepsace %s", hostID, namespaceID) + } + hostNamespaceIdFloat, ok := hostNamespaceID.(float64) + if ok { + hostNamespaceId = strconv.FormatInt(int64(hostNamespaceIdFloat), constants.DefaultIntBase) + break + } + } + } + + if hostNamespaceId == "" { + return "", fmt.Errorf("can not get the hostNamespaceId of host %s, namespace %s", hostID, namespaceID) + } + + return hostNamespaceId, nil +} + +// UpdateNamespace used for update namespace +func (cli *OceandiskClient) UpdateNamespace(ctx context.Context, + namespaceID string, params map[string]interface{}) error { + url := fmt.Sprintf(api.UpdateNamespace, namespaceID) + resp, err := cli.Put(ctx, url, params) + if err != nil { + return err + } + + code, msg, err := utils.FormatRespErr(resp.Error) + if err != nil { + return err + } + if code != 0 { + return fmt.Errorf("update Namespace %s by params %v failed, "+ + "error code: %d, error msg: %s", namespaceID, params, code, msg) + } + + return nil +} diff --git a/storage/oceanstorage/oceandisk/client/client_namespace_test.go b/storage/oceanstorage/oceandisk/client/client_namespace_test.go new file mode 100644 index 00000000..6152f8bf --- /dev/null +++ b/storage/oceanstorage/oceandisk/client/client_namespace_test.go @@ -0,0 +1,1513 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package client provides oceandisk storage client +package client + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "reflect" + "testing" + + "github.com/agiledragon/gomonkey/v2" + + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/base" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" +) + +func TestBaseClient_QueryAssociateNamespaceGroup_Success(t *testing.T) { + // arrange + objType := 11 + objID := "1" + client, err := NewClient(context.Background(), &NewClientConfig{}) + if err != nil { + return + } + + namespaceGroups := make([]interface{}, 0) + namespaceGroup := map[string]string{ + "DESCRIPTION": "", + "ID": "1", + } + namespaceGroups = append(namespaceGroups, namespaceGroup) + + mockResponse := base.Response{ + Error: map[string]interface{}{ + "code": float64(0), + "description": "0", + }, + Data: namespaceGroups, + } + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodFunc(&RestClient{}, "Get", func(ctx context.Context, + url string, data map[string]interface{}) (base.Response, error) { + return mockResponse, nil + }) + + // action + getRes, getErr := client.QueryAssociateNamespaceGroup(context.Background(), objType, objID) + + // assert + if !reflect.DeepEqual(namespaceGroups, getRes) || getErr != nil { + t.Errorf("TestBaseClient_QueryAssociateNamespaceGroup_Success failed, "+ + "wantRes = %v, gotRes = %v, wantErr = nil, gotErr = %v", namespaceGroups, getRes, getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestBaseClient_QueryAssociateNamespaceGroup_CodeError(t *testing.T) { + // arrange + objType := 11 + objID := "1" + client, err := NewClient(context.Background(), &NewClientConfig{}) + if err != nil { + return + } + + errCode := 1 + errMsg := "unknown error" + mockResponse := base.Response{ + Error: map[string]interface{}{ + "code": float64(errCode), + "description": errMsg, + }, + Data: nil, + } + wantErr := fmt.Errorf("associate query namespacegroup by obj %s of type %d failed, "+ + "error code: %d, error msg: %s", objID, objType, errCode, errMsg) + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodFunc(&RestClient{}, "Get", func(ctx context.Context, + url string, data map[string]interface{}) (base.Response, error) { + return mockResponse, nil + }) + + // action + getRes, getErr := client.QueryAssociateNamespaceGroup(context.Background(), objType, objID) + + // assert + if getRes != nil || !reflect.DeepEqual(wantErr, getErr) { + t.Errorf("TestBaseClient_QueryAssociateNamespaceGroup_CodeError failed, "+ + "wantRes = nil, gotRes = %v, wantErr = %v, gotErr = %v", getRes, wantErr, getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestBaseClient_QueryAssociateNamespaceGroup_RespNil(t *testing.T) { + // arrange + objType := 11 + objID := "1" + client, err := NewClient(context.Background(), &NewClientConfig{}) + if err != nil { + return + } + + mockResponse := base.Response{ + Error: map[string]interface{}{ + "code": float64(0), + "description": "0", + }, + Data: nil, + } + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodFunc(&RestClient{}, "Get", func(ctx context.Context, + url string, data map[string]interface{}) (base.Response, error) { + return mockResponse, nil + }) + + // action + getRes, getErr := client.QueryAssociateNamespaceGroup(context.Background(), objType, objID) + + // assert + if len(getRes) != 0 || getErr != nil { + t.Errorf("TestBaseClient_QueryAssociateNamespaceGroup_RespNil failed, "+ + "wantRes = [], gotRes = %v, wantErr = nil, gotErr = %v", getRes, getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestBaseClient_QueryAssociateNamespaceGroup_RespFormatError(t *testing.T) { + // arrange + objType := 11 + objID := "1" + client, err := NewClient(context.Background(), &NewClientConfig{}) + if err != nil { + return + } + + mockResponse := base.Response{ + Error: map[string]interface{}{ + "code": float64(0), + "description": "0", + }, + Data: "", + } + wantErr := fmt.Errorf("convert respData to arr failed, data: %v", mockResponse.Data) + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodFunc(&RestClient{}, "Get", func(ctx context.Context, + url string, data map[string]interface{}) (base.Response, error) { + return mockResponse, nil + }) + + // action + getRes, getErr := client.QueryAssociateNamespaceGroup(context.Background(), objType, objID) + + // assert + if getRes != nil || !reflect.DeepEqual(wantErr, getErr) { + t.Errorf("TestBaseClient_QueryAssociateNamespaceGroup_RespFormatError failed, "+ + "wantRes = nil, gotRes = %v, wantErr = %v, gotErr = %v", getRes, wantErr, getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestBaseClient_GetNamespaceByName_Success(t *testing.T) { + // arrange + name := "namespace1" + client, err := NewClient(context.Background(), &NewClientConfig{}) + if err != nil { + return + } + + namespaces := make([]interface{}, 0) + namespace := map[string]interface{}{ + "name": name, + "ID": "1", + } + namespaces = append(namespaces, namespace) + + mockResponse := base.Response{ + Error: map[string]interface{}{ + "code": float64(0), + "description": "0", + }, + Data: namespaces, + } + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodFunc(&RestClient{}, "Get", func(ctx context.Context, + url string, data map[string]interface{}) (base.Response, error) { + return mockResponse, nil + }) + + // action + getRes, getErr := client.GetNamespaceByName(context.Background(), name) + + // assert + if !reflect.DeepEqual(namespace, getRes) || getErr != nil { + t.Errorf("TestBaseClient_GetNamespaceByName_Success failed, "+ + "wantRes = %v, gotRes = %v, wantErr = nil, gotErr = %v", namespace, getRes, getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestBaseClient_GetNamespaceByName_NamespaceFormatError(t *testing.T) { + // arrange + name := "namespace1" + client, err := NewClient(context.Background(), &NewClientConfig{}) + if err != nil { + return + } + + namespaces := make([]interface{}, 0) + namespace := map[string]string{ + "name": name, + "ID": "1", + } + namespaces = append(namespaces, namespace) + + mockResponse := base.Response{ + Error: map[string]interface{}{ + "code": float64(0), + "description": "0", + }, + Data: namespaces, + } + wantErr := fmt.Errorf("convert namespace to map failed, data: %v", namespace) + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodFunc(&RestClient{}, "Get", func(ctx context.Context, + url string, data map[string]interface{}) (base.Response, error) { + return mockResponse, nil + }) + + // action + getRes, getErr := client.GetNamespaceByName(context.Background(), name) + + // assert + if getRes != nil || !reflect.DeepEqual(wantErr, getErr) { + t.Errorf("TestBaseClient_GetNamespaceByName_NamespaceFormatError failed, "+ + "wantRes = nil, gotRes = %v, wantErr = %v, gotErr = %v", getRes, wantErr, getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestBaseClient_GetNamespaceByID_Success(t *testing.T) { + // arrange + id := "1" + client, err := NewClient(context.Background(), &NewClientConfig{}) + if err != nil { + return + } + + namespace := map[string]interface{}{ + "name": "namespace1", + "ID": id, + } + mockResponse := base.Response{ + Error: map[string]interface{}{ + "code": float64(0), + "description": "0", + }, + Data: namespace, + } + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodFunc(&RestClient{}, "Get", func(ctx context.Context, + url string, data map[string]interface{}) (base.Response, error) { + return mockResponse, nil + }) + + // action + getRes, getErr := client.GetNamespaceByID(context.Background(), id) + + // assert + if !reflect.DeepEqual(namespace, getRes) || getErr != nil { + t.Errorf("TestBaseClient_GetNamespaceByID_Success failed, "+ + "wantRes = %v, gotRes = %v, wantErr = nil, gotErr = %v", namespace, getRes, getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestBaseClient_GetNamespaceByID_FormatNamespaceError(t *testing.T) { + // arrange + id := "1" + client, err := NewClient(context.Background(), &NewClientConfig{}) + if err != nil { + return + } + + namespace := "invalid format" + mockResponse := base.Response{ + Error: map[string]interface{}{ + "code": float64(0), + "description": "0", + }, + Data: namespace, + } + wantErr := fmt.Errorf("convert namespace to map failed, data: %v", namespace) + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodFunc(&RestClient{}, "Get", func(ctx context.Context, + url string, data map[string]interface{}) (base.Response, error) { + return mockResponse, nil + }) + + // action + getRes, getErr := client.GetNamespaceByID(context.Background(), id) + + // assert + if getRes != nil || !reflect.DeepEqual(wantErr, getErr) { + t.Errorf("TestBaseClient_GetNamespaceByID_FormatNamespaceError failed, "+ + "wantRes = nil, gotRes = %v, wantErr = %v, gotErr = %v", getRes, wantErr, getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestBaseClient_AddNamespaceToGroup_Success(t *testing.T) { + // arrange + namespaceID := "1" + groupID := "1" + client, err := NewClient(context.Background(), &NewClientConfig{}) + if err != nil { + return + } + + mockResponse := base.Response{ + Error: map[string]interface{}{ + "code": float64(0), + "description": "0", + }, + } + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodFunc(&RestClient{}, "Post", func(ctx context.Context, + url string, data map[string]interface{}) (base.Response, error) { + return mockResponse, nil + }) + + // action + getErr := client.AddNamespaceToGroup(context.Background(), namespaceID, groupID) + + // assert + if getErr != nil { + t.Errorf("TestBaseClient_AddNamespaceToGroup_Success failed, wantErr = nil, gotErr = %v", getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestBaseClient_AddNamespaceToGroup_CodeError(t *testing.T) { + // arrange + namespaceID := "1" + groupID := "1" + client, err := NewClient(context.Background(), &NewClientConfig{}) + if err != nil { + return + } + + errCode := 1 + errMsg := "unknown error" + mockResponse := base.Response{ + Error: map[string]interface{}{ + "code": float64(errCode), + "description": errMsg, + }, + } + wantErr := fmt.Errorf("add namespace %s to group %s failed, "+ + "error code: %d, error msg: %s", namespaceID, groupID, errCode, errMsg) + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodFunc(&RestClient{}, "Post", func(ctx context.Context, + url string, data map[string]interface{}) (base.Response, error) { + return mockResponse, nil + }) + + // action + getErr := client.AddNamespaceToGroup(context.Background(), namespaceID, groupID) + + // assert + if !reflect.DeepEqual(wantErr, getErr) { + t.Errorf("TestBaseClient_AddNamespaceToGroup_Success failed, wantErr = %v, gotErr = %v", wantErr, getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestBaseClient_RemoveNamespaceFromGroup_Success(t *testing.T) { + // arrange + namespaceID := "1" + groupID := "1" + client, err := NewClient(context.Background(), &NewClientConfig{}) + if err != nil { + return + } + + mockResponse := base.Response{ + Error: map[string]interface{}{ + "code": float64(0), + "description": "0", + }, + } + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodFunc(&RestClient{}, "Delete", func(ctx context.Context, + url string, data map[string]interface{}) (base.Response, error) { + return mockResponse, nil + }) + + // action + getErr := client.RemoveNamespaceFromGroup(context.Background(), namespaceID, groupID) + + // assert + if getErr != nil { + t.Errorf("TestBaseClient_RemoveNamespaceFromGroup_Success failed, wantErr = nil, gotErr = %v", getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestBaseClient_RemoveNamespaceFromGroup_CodeError(t *testing.T) { + // arrange + namespaceID := "1" + groupID := "1" + client, err := NewClient(context.Background(), &NewClientConfig{}) + if err != nil { + return + } + + errCode := 1 + errMsg := "unknown error" + mockResponse := base.Response{ + Error: map[string]interface{}{ + "code": float64(errCode), + "description": errMsg, + }, + } + wantErr := fmt.Errorf("remove namespace %s from group %s failed, "+ + "error code: %d, error msg: %s", namespaceID, groupID, errCode, errMsg) + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodFunc(&RestClient{}, "Delete", func(ctx context.Context, + url string, data map[string]interface{}) (base.Response, error) { + return mockResponse, nil + }) + + // action + getErr := client.RemoveNamespaceFromGroup(context.Background(), namespaceID, groupID) + + // assert + if !reflect.DeepEqual(wantErr, getErr) { + t.Errorf("TestBaseClient_RemoveNamespaceFromGroup_CodeError failed, "+ + "wantErr = %v, gotErr = %v", wantErr, getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestBaseClient_GetNamespaceGroupByName_Success(t *testing.T) { + // arrange + name := "group1" + client, err := NewClient(context.Background(), &NewClientConfig{}) + if err != nil { + return + } + + groups := make([]interface{}, 0) + group := map[string]interface{}{ + "name": name, + "ID": "1", + } + groups = append(groups, group) + + mockResponse := base.Response{ + Error: map[string]interface{}{ + "code": float64(0), + "description": "0", + }, + Data: groups, + } + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodFunc(&RestClient{}, "Get", func(ctx context.Context, + url string, data map[string]interface{}) (base.Response, error) { + return mockResponse, nil + }) + + // action + getRes, getErr := client.GetNamespaceGroupByName(context.Background(), name) + + // assert + if !reflect.DeepEqual(group, getRes) || getErr != nil { + t.Errorf("TestBaseClient_GetNamespaceGroupByName_Success failed, "+ + "wantRes = %v, gotRes = %v, wantErr = nil, gotErr = %v", group, getRes, getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestBaseClient_GetNamespaceGroupByName_RespFormatError(t *testing.T) { + // arrange + name := "group1" + client, err := NewClient(context.Background(), &NewClientConfig{}) + if err != nil { + return + } + + mockResponse := base.Response{ + Error: map[string]interface{}{ + "code": float64(0), + "description": "0", + }, + Data: "unknown format", + } + wantErr := fmt.Errorf("convert respData to arr failed, data: %v", mockResponse.Data) + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodFunc(&RestClient{}, "Get", func(ctx context.Context, + url string, data map[string]interface{}) (base.Response, error) { + return mockResponse, nil + }) + + // action + getRes, getErr := client.GetNamespaceGroupByName(context.Background(), name) + + // assert + if getRes != nil || !reflect.DeepEqual(wantErr, getErr) { + t.Errorf("TestBaseClient_GetNamespaceGroupByName_RespFormatError failed, "+ + "wantRes = nil, gotRes = %v, wantErr = %v, gotErr = %v", getRes, wantErr, getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestBaseClient_GetNamespaceGroupByName_GroupFormatError(t *testing.T) { + // arrange + name := "group1" + client, err := NewClient(context.Background(), &NewClientConfig{}) + if err != nil { + return + } + + groups := make([]interface{}, 0) + group := "unknown format" + groups = append(groups, group) + + mockResponse := base.Response{ + Error: map[string]interface{}{ + "code": float64(0), + "description": "0", + }, + Data: groups, + } + wantErr := fmt.Errorf("convert group to arr failed, data: %v", group) + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodFunc(&RestClient{}, "Get", func(ctx context.Context, + url string, data map[string]interface{}) (base.Response, error) { + return mockResponse, nil + }) + + // action + getRes, getErr := client.GetNamespaceGroupByName(context.Background(), name) + + // assert + if getRes != nil || !reflect.DeepEqual(wantErr, getErr) { + t.Errorf("TestBaseClient_GetNamespaceGroupByName_GroupFormatError failed, "+ + "wantRes = nil, gotRes = %v, wantErr = %v, gotErr = %v", getRes, wantErr, getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestBaseClient_CreateNamespaceGroup_Success(t *testing.T) { + // arrange + name := "group1" + client, err := NewClient(context.Background(), &NewClientConfig{}) + if err != nil { + return + } + + group := map[string]interface{}{ + "name": name, + "ID": "1", + } + + mockResponse := base.Response{ + Error: map[string]interface{}{ + "code": float64(0), + "description": "0", + }, + Data: group, + } + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodFunc(&RestClient{}, "Post", func(ctx context.Context, + url string, data map[string]interface{}) (base.Response, error) { + return mockResponse, nil + }) + + // action + getRes, getErr := client.CreateNamespaceGroup(context.Background(), name) + + // assert + if !reflect.DeepEqual(group, getRes) || getErr != nil { + t.Errorf("TestBaseClient_CreateNamespaceGroup_Success failed,"+ + "wantRes = %v, gotRes = %v, wantErr = nil, gotErr = %v", group, getRes, getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestBaseClient_CreateNamespaceGroup_ExistSuccess(t *testing.T) { + // arrange + name := "group1" + client, err := NewClient(context.Background(), &NewClientConfig{}) + if err != nil { + return + } + + group := map[string]interface{}{ + "name": name, + "ID": "1", + } + + mockResponse := base.Response{ + Error: map[string]interface{}{ + "code": float64(objectNameAlreadyExist), + "description": "0", + }, + Data: group, + } + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodFunc(&RestClient{}, "Post", func(ctx context.Context, + url string, data map[string]interface{}) (base.Response, error) { + return mockResponse, nil + }).ApplyMethodFunc(client, "GetNamespaceGroupByName", func(ctx context.Context, + name string) (map[string]interface{}, error) { + return group, nil + }) + + // action + getRes, getErr := client.CreateNamespaceGroup(context.Background(), name) + + // assert + if !reflect.DeepEqual(group, getRes) || getErr != nil { + t.Errorf("TestBaseClient_CreateNamespaceGroup_ExistSuccess failed,"+ + "wantRes = %v, gotRes = %v, wantErr = nil, gotErr = %v", group, getRes, getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestBaseClient_CreateNamespaceGroup_CodeError(t *testing.T) { + // arrange + name := "group1" + client, err := NewClient(context.Background(), &NewClientConfig{}) + if err != nil { + return + } + + errCode := 1 + errMsg := "unknown error" + mockResponse := base.Response{ + Error: map[string]interface{}{ + "code": float64(errCode), + "description": errMsg, + }, + Data: nil, + } + wantErr := fmt.Errorf("create namespacegroup %s failed, "+ + "error code: %d, error msg: %s", name, errCode, errMsg) + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodFunc(&RestClient{}, "Post", func(ctx context.Context, + url string, data map[string]interface{}) (base.Response, error) { + return mockResponse, nil + }) + + // action + getRes, getErr := client.CreateNamespaceGroup(context.Background(), name) + + // assert + if getRes != nil || !reflect.DeepEqual(wantErr, getErr) { + t.Errorf("TestBaseClient_CreateNamespaceGroup_CodeError failed,"+ + "wantRes = nil, gotRes = %v, wantErr = %v, gotErr = %v", getRes, wantErr, getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestBaseClient_CreateNamespaceGroup_GroupFormatError(t *testing.T) { + // arrange + name := "group1" + client, err := NewClient(context.Background(), &NewClientConfig{}) + if err != nil { + return + } + + mockResponse := base.Response{ + Error: map[string]interface{}{ + "code": float64(0), + "description": "0", + }, + Data: nil, + } + wantErr := fmt.Errorf("convert namespaceGroup to map failed, data: %v", mockResponse.Data) + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodFunc(&RestClient{}, "Post", func(ctx context.Context, + url string, data map[string]interface{}) (base.Response, error) { + return mockResponse, nil + }) + + // action + getRes, getErr := client.CreateNamespaceGroup(context.Background(), name) + + // assert + if getRes != nil || !reflect.DeepEqual(wantErr, getErr) { + t.Errorf("TestBaseClient_CreateNamespaceGroup_GroupFormatError failed,"+ + "wantRes = nil, gotRes = %v, wantErr = %v, gotErr = %v", getRes, wantErr, getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestBaseClient_DeleteNamespaceGroup_Success(t *testing.T) { + // arrange + id := "1" + client, err := NewClient(context.Background(), &NewClientConfig{}) + if err != nil { + return + } + + mockResponse := base.Response{ + Error: map[string]interface{}{ + "code": float64(0), + "description": "0", + }, + } + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodReturn(&RestClient{}, "Delete", mockResponse, nil) + + // action + getErr := client.DeleteNamespaceGroup(context.Background(), id) + + // assert + if getErr != nil { + t.Errorf("TestBaseClient_DeleteNamespaceGroup_Success failed, wantErr = nil, gotErr = %v", getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestBaseClient_DeleteNamespaceGroup_NotExistSuccess(t *testing.T) { + // arrange + id := "1" + client, err := NewClient(context.Background(), &NewClientConfig{}) + if err != nil { + return + } + + mockResponse := base.Response{ + Error: map[string]interface{}{ + "code": float64(objectNotExist), + "description": "namespacegroup is already exist", + }, + } + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodReturn(&RestClient{}, "Delete", mockResponse, nil) + + // action + getErr := client.DeleteNamespaceGroup(context.Background(), id) + + // assert + if getErr != nil { + t.Errorf("TestBaseClient_DeleteNamespaceGroup_NotExistSuccess failed, "+ + "wantErr = nil, gotErr = %v", getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestBaseClient_DeleteNamespaceGroup_CodeError(t *testing.T) { + // arrange + id := "1" + client, err := NewClient(context.Background(), &NewClientConfig{}) + if err != nil { + return + } + + errCode := 1 + errMsg := "unknown error" + mockResponse := base.Response{ + Error: map[string]interface{}{ + "code": float64(errCode), + "description": errMsg, + }, + } + wantErr := fmt.Errorf("delete namespacegroup %s failed, error code: %d, error msg: %s", id, errCode, errMsg) + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodReturn(&RestClient{}, "Delete", mockResponse, nil) + + // action + getErr := client.DeleteNamespaceGroup(context.Background(), id) + + // assert + if !reflect.DeepEqual(wantErr, getErr) { + t.Errorf("TestBaseClient_DeleteNamespaceGroup_CodeError failed, "+ + "wantErr = %v, gotErr = %v", wantErr, getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestBaseClient_ExtendNamespace_Success(t *testing.T) { + // arrange + namespaceId := "1" + capacity := int64(10) + client, err := NewClient(context.Background(), &NewClientConfig{}) + if err != nil { + return + } + + mockResponse := base.Response{ + Error: map[string]interface{}{ + "code": float64(0), + "description": "0", + }, + } + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodReturn(&RestClient{}, "Put", mockResponse, nil) + + // action + getErr := client.ExtendNamespace(context.Background(), namespaceId, capacity) + + // assert + if getErr != nil { + t.Errorf("TestBaseClient_ExtendNamespace_Success failed, wantErr = nil, gotErr = %v", getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestBaseClient_ExtendNamespace_CodeFormatError(t *testing.T) { + // arrange + namespaceId := "1" + capacity := int64(10) + errorCode := 0 + client, err := NewClient(context.Background(), &NewClientConfig{}) + if err != nil { + return + } + + mockResponse := base.Response{ + Error: map[string]interface{}{ + "code": errorCode, + "description": "0", + }, + } + wantErr := fmt.Errorf("can not convert resp code %v to float64", errorCode) + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodReturn(&RestClient{}, "Put", mockResponse, nil) + + // action + getErr := client.ExtendNamespace(context.Background(), namespaceId, capacity) + + // assert + if !reflect.DeepEqual(wantErr, getErr) { + t.Errorf("TestBaseClient_ExtendNamespace_CodeFormatError failed, "+ + "wantErr = %v, gotErr = %v", wantErr, getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestBaseClient_CreateNamespace_Success(t *testing.T) { + // arrange + params := CreateNamespaceParams{} + client, err := NewClient(context.Background(), &NewClientConfig{}) + if err != nil { + return + } + + data := map[string]interface{}{ + "ID": "1", + "Name": "test", + } + mockResponse := base.Response{ + Error: map[string]interface{}{ + "code": float64(0), + "description": "0", + }, + Data: data, + } + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodReturn(&RestClient{}, "Post", mockResponse, nil) + + // action + getRes, getErr := client.CreateNamespace(context.Background(), params) + + // assert + if !reflect.DeepEqual(data, getRes) || getErr != nil { + t.Errorf("TestBaseClient_CreateNamespace_Success failed, "+ + "wantRes = %v, gotRes = %v, wantErr = nil, gotErr = %v", data, getRes, getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestBaseClient_CreateNamespace_InvalidParamsError(t *testing.T) { + // arrange + params := CreateNamespaceParams{} + data := map[string]interface{}{ + "NAME": params.Name, + "PARENTID": params.ParentId, + "CAPACITY": params.Capacity, + "DESCRIPTION": params.Description, + } + client, err := NewClient(context.Background(), &NewClientConfig{}) + if err != nil { + return + } + + msg := "Enter a correct parameter." + mockResponse := base.Response{ + Error: map[string]interface{}{ + "code": float64(parameterIncorrect), + "description": msg, + }, + } + wantErr := fmt.Errorf("create Namespace with incorrect parameters %v, "+ + "err code: %d, err msg: %s", data, parameterIncorrect, msg) + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodReturn(&RestClient{}, "Post", mockResponse, nil) + + // action + getRes, getErr := client.CreateNamespace(context.Background(), params) + + // assert + if getRes != nil || !reflect.DeepEqual(wantErr, getErr) { + t.Errorf("TestBaseClient_CreateNamespace_InvalidParamsError failed, "+ + "wantRes = nil, gotRes = %v, wantErr = %v, gotErr = %v", getRes, wantErr, getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestBaseClient_DeleteNamespace_Success(t *testing.T) { + // arrange + id := "1" + client, err := NewClient(context.Background(), &NewClientConfig{}) + if err != nil { + return + } + + mockResponse := base.Response{ + Error: map[string]interface{}{ + "code": float64(0), + "description": "0", + }, + } + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodReturn(&RestClient{}, "Delete", mockResponse, nil) + + // action + getErr := client.DeleteNamespace(context.Background(), id) + + // assert + if getErr != nil { + t.Errorf("TestBaseClient_DeleteNamespace_Success failed, wantErr = nil, gotErr = %v", getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestBaseClient_DeleteNamespace_NotExistSuccess(t *testing.T) { + // arrange + id := "1" + client, err := NewClient(context.Background(), &NewClientConfig{}) + if err != nil { + return + } + + mockResponse := base.Response{ + Error: map[string]interface{}{ + "code": float64(namespaceNotExist), + "description": "namespacegroup is already exist", + }, + } + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodReturn(&RestClient{}, "Delete", mockResponse, nil) + + // action + getErr := client.DeleteNamespace(context.Background(), id) + + // assert + if getErr != nil { + t.Errorf("TestBaseClient_DeleteNamespace_NotExistSuccess failed, "+ + "wantErr = nil, gotErr = %v", getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestBaseClient_DeleteNamespace_CodeError(t *testing.T) { + // arrange + id := "1" + client, err := NewClient(context.Background(), &NewClientConfig{}) + if err != nil { + return + } + + errCode := 1 + errMsg := "unknown error" + mockResponse := base.Response{ + Error: map[string]interface{}{ + "code": float64(errCode), + "description": errMsg, + }, + } + wantErr := fmt.Errorf("delete namespace %s failed, error code: %d, error msg: %s", id, errCode, errMsg) + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodReturn(&RestClient{}, "Delete", mockResponse, nil) + + // action + getErr := client.DeleteNamespace(context.Background(), id) + + // assert + if !reflect.DeepEqual(wantErr, getErr) { + t.Errorf("TestBaseClient_DeleteNamespace_CodeError failed, "+ + "wantErr = %v, gotErr = %v", wantErr, getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestBaseClient_GetNamespaceCountOfMapping_Success(t *testing.T) { + // arrange + mappingId := "1" + countStr := "10" + client, err := NewClient(context.Background(), &NewClientConfig{}) + if err != nil { + return + } + + data := map[string]interface{}{ + "COUNT": countStr, + } + mockResponse := base.Response{ + Error: map[string]interface{}{ + "code": float64(0), + "description": "0", + }, + Data: data, + } + wantCount := utils.ParseIntWithDefault(countStr, constants.DefaultIntBase, constants.DefaultIntBitSize, 0) + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodReturn(&RestClient{}, "Get", mockResponse, nil) + + // action + getCount, getErr := client.GetNamespaceCountOfMapping(context.Background(), mappingId) + + // assert + if wantCount != getCount || getErr != nil { + t.Errorf("TestBaseClient_GetNamespaceCountOfMapping_Success failed, "+ + "wantCount = %d, gotCount = %d, wantErr = nil, gotErr = %v", wantCount, getCount, getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestBaseClient_GetNamespaceCountOfMapping_CountFormatError(t *testing.T) { + // arrange + mappingId := "1" + countStr := float64(10) + client, err := NewClient(context.Background(), &NewClientConfig{}) + if err != nil { + return + } + + data := map[string]interface{}{ + "COUNT": countStr, + } + mockResponse := base.Response{ + Error: map[string]interface{}{ + "code": float64(0), + "description": "0", + }, + Data: data, + } + wantErr := fmt.Errorf("convert countStr to string failed, data: %v", countStr) + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodReturn(&RestClient{}, "Get", mockResponse, nil) + + // action + getCount, getErr := client.GetNamespaceCountOfMapping(context.Background(), mappingId) + + // assert + if getCount != 0 || !reflect.DeepEqual(wantErr, getErr) { + t.Errorf("TestBaseClient_GetNamespaceCountOfMapping_CountFormatError failed, "+ + "wantCount = 0, gotCount = %d, wantErr = %v, gotErr = %v", getCount, wantErr, getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestBaseClient_GetNamespaceCountOfHost_Success(t *testing.T) { + // arrange + mappingId := "1" + countStr := "10" + client, err := NewClient(context.Background(), &NewClientConfig{}) + if err != nil { + return + } + + data := map[string]interface{}{ + "COUNT": countStr, + } + mockResponse := base.Response{ + Error: map[string]interface{}{ + "code": float64(0), + "description": "0", + }, + Data: data, + } + wantCount := utils.ParseIntWithDefault(countStr, constants.DefaultIntBase, constants.DefaultIntBitSize, 0) + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodReturn(&RestClient{}, "Get", mockResponse, nil) + + // action + getCount, getErr := client.GetNamespaceCountOfHost(context.Background(), mappingId) + + // assert + if wantCount != getCount || getErr != nil { + t.Errorf("TestBaseClient_GetNamespaceCountOfHost_Success failed, "+ + "wantCount = %d, gotCount = %d, wantErr = nil, gotErr = %v", wantCount, getCount, getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestBaseClient_GetNamespaceCountOfHost_CountFormatError(t *testing.T) { + // arrange + mappingId := "1" + countStr := float64(10) + client, err := NewClient(context.Background(), &NewClientConfig{}) + if err != nil { + return + } + + data := map[string]interface{}{ + "COUNT": countStr, + } + mockResponse := base.Response{ + Error: map[string]interface{}{ + "code": float64(0), + "description": "0", + }, + Data: data, + } + wantErr := fmt.Errorf("convert countStr to string failed, data: %v", countStr) + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodReturn(&RestClient{}, "Get", mockResponse, nil) + + // action + getCount, getErr := client.GetNamespaceCountOfHost(context.Background(), mappingId) + + // assert + if getCount != 0 || !reflect.DeepEqual(wantErr, getErr) { + t.Errorf("TestBaseClient_GetNamespaceCountOfHost_CountFormatError failed, "+ + "wantCount = 0, gotCount = %d, wantErr = %v, gotErr = %v", getCount, wantErr, getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestBaseClient_GetHostNamespaceId_Success(t *testing.T) { + // arrange + hostId := "1" + namespaceId := "1" + client, err := NewClient(context.Background(), &NewClientConfig{}) + if err != nil { + return + } + + namespaces := make([]interface{}, 0) + namespace := map[string]interface{}{ + "name": "namespace1", + "ID": "1", + "ASSOCIATEMETADATA": "{\"hostNamespaceID\":1}", + } + namespaces = append(namespaces, namespace) + mockResponse := base.Response{ + Error: map[string]interface{}{ + "code": float64(0), + "description": "0", + }, + Data: namespaces, + } + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodReturn(&RestClient{}, "Get", mockResponse, nil) + + // action + getId, getErr := client.GetHostNamespaceId(context.Background(), hostId, namespaceId) + + // assert + if getId != "1" || getErr != nil { + t.Errorf("TestBaseClient_GetHostNamespaceId_Success failed, "+ + "wantId = 1, getId = %s, wantErr = nil, gotErr = %v", getId, getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestBaseClient_GetHostNamespaceId_NotExistIdError(t *testing.T) { + // arrange + hostId := "1" + namespaceId := "1" + client, err := NewClient(context.Background(), &NewClientConfig{}) + if err != nil { + return + } + + namespaces := make([]interface{}, 0) + mockResponse := base.Response{ + Error: map[string]interface{}{ + "code": float64(0), + "description": "0", + }, + Data: namespaces, + } + wantErr := fmt.Errorf("can not get the hostNamespaceId of host %s, namespace %s", hostId, namespaceId) + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodReturn(&RestClient{}, "Get", mockResponse, nil) + + // action + getId, getErr := client.GetHostNamespaceId(context.Background(), hostId, namespaceId) + + // assert + if getId != "" || !reflect.DeepEqual(wantErr, getErr) { + t.Errorf("TestBaseClient_GetHostNamespaceId_NotExistIdError failed, "+ + "wantId = , getId = %s, wantErr = %v, gotErr = %v", getId, wantErr, getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestBaseClient_GetHostNamespaceId_JsonUnmarshalError(t *testing.T) { + // arrange + hostId := "1" + namespaceId := "1" + client, err := NewClient(context.Background(), &NewClientConfig{}) + if err != nil { + return + } + + namespaces := make([]interface{}, 0) + namespace := map[string]interface{}{ + "name": "namespace1", + "ID": "1", + "ASSOCIATEMETADATA": "{\"hostNamespaceID\":1}", + } + namespaces = append(namespaces, namespace) + mockResponse := base.Response{ + Error: map[string]interface{}{ + "code": float64(0), + "description": "0", + }, + Data: namespaces, + } + mockErr := errors.New("json unmarshal err") + wantErr := fmt.Errorf("unmarshal associateData fail while "+ + "getting the hostNamespaceId of host %s, namespace %s, error: %v", hostId, namespaceId, mockErr) + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodReturn(&RestClient{}, "Get", mockResponse, nil). + ApplyFunc(json.Unmarshal, func(data []byte, v any) error { + return mockErr + }) + + // action + getId, getErr := client.GetHostNamespaceId(context.Background(), hostId, namespaceId) + + // assert + if getId != "" || !reflect.DeepEqual(wantErr, getErr) { + t.Errorf("TestBaseClient_GetHostNamespaceId_JsonUnmarshalError failed, "+ + "wantId = , getId = %s, wantErr = %v, gotErr = %v", getId, wantErr, getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestBaseClient_UpdateNamespace_Success(t *testing.T) { + // arrange + namespaceID := "1" + params := map[string]interface{}{} + client, err := NewClient(context.Background(), &NewClientConfig{}) + if err != nil { + return + } + + mockResponse := base.Response{ + Error: map[string]interface{}{ + "code": float64(0), + "description": "0", + }, + } + + // mock + mock := gomonkey.NewPatches() + mock.ApplyMethodReturn(&RestClient{}, "Put", mockResponse, nil) + + // action + getErr := client.UpdateNamespace(context.Background(), namespaceID, params) + + // assert + if getErr != nil { + t.Errorf("TestBaseClient_UpdateNamespace_Success failed, wantErr = nil, gotErr = %v", getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} diff --git a/storage/oceanstorage/oceandisk/client/client_restful.go b/storage/oceanstorage/oceandisk/client/client_restful.go new file mode 100644 index 00000000..caeb737a --- /dev/null +++ b/storage/oceanstorage/oceandisk/client/client_restful.go @@ -0,0 +1,502 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package client provides oceandisk storage client +package client + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + "strconv" + "sync" + "sync/atomic" + + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/base" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" +) + +const ( + // DefaultParallelCount defines default parallel count + DefaultParallelCount int = 30 + + // MaxParallelCount defines max parallel count + MaxParallelCount int = 30 + + // MinParallelCount defines min parallel count + MinParallelCount int = 1 +) + +var ( + filterLog = map[string]map[string]bool{ + "POST": { + "/xx/sessions": true, + }, + } + debugLog = map[string]map[string]bool{ + "GET": { + "/license/feature": true, + "/storagepool": true, + `/system`: true, + }, + } + + debugLogRegex = map[string][]string{ + "GET": { + `/system`, + }, + } +) + +func isFilterLog(method, url string) bool { + if filter, exist := filterLog[method]; exist && filter[url] { + return true + } + + return false +} + +// RestClient defines client implements the rest interface +type RestClient struct { + Client base.HTTP + Url string + Urls []string + + User string + SecretNamespace string + SecretName string + StorageVersion string + BackendID string + Storage string + DeviceId string + Token string + + SystemInfoRefreshing uint32 + ReLoginMutex sync.Mutex + RequestSemaphore *utils.Semaphore +} + +// Call provides call for restful request +func (cli *RestClient) Call(ctx context.Context, + method string, url string, data map[string]interface{}) (base.Response, error) { + var r base.Response + var err error + + r, err = cli.BaseCall(ctx, method, url, data) + if !base.NeedReLogin(r, err) { + return r, err + } + + // Current connection fails, try to relogin to other Urls if exist, + // if relogin success, resend the request again. + log.AddContext(ctx).Infof("try to relogin and resend request method: %s, Url: %s", method, url) + err = cli.ReLogin(ctx) + if err != nil { + return base.Response{}, err + } + + if err = cli.SetSystemInfo(ctx); err != nil { + return base.Response{}, fmt.Errorf("after relogin, can't get system info, error: %v", err) + } + + return cli.BaseCall(ctx, method, url, data) +} + +// SetSystemInfo set system info +// the mutex lock is required for re-login. Therefore, the internal query of the login interface cannot be performed. +func (cli *RestClient) SetSystemInfo(ctx context.Context) error { + log.AddContext(ctx).Infof("set backend [%s] system info is refreshing", cli.BackendID) + atomic.StoreUint32(&cli.SystemInfoRefreshing, 1) + defer func() { + log.AddContext(ctx).Infof("set backend [%s] system info are refreshed", cli.BackendID) + atomic.StoreUint32(&cli.SystemInfoRefreshing, 0) + }() + + system, err := cli.GetSystem(ctx) + if err != nil { + return err + } + + storagePointVersion, ok := utils.GetValue[string](system, "pointRelease") + if ok { + cli.StorageVersion = storagePointVersion + } + + log.AddContext(ctx).Infof("backend type [%s], backend [%s], storage version [%s]", + cli.Storage, cli.BackendID, cli.StorageVersion) + return nil +} + +// BaseCall provides base call for request +func (cli *RestClient) BaseCall(ctx context.Context, + method string, url string, data map[string]interface{}) (base.Response, error) { + var r base.Response + var req *http.Request + var err error + + if cli.Client == nil { + errMsg := "http client is nil" + log.AddContext(ctx).Errorf("Failed to send request method: %s, url: %s, error: %s", method, url, errMsg) + return base.Response{}, errors.New(errMsg) + } + + if url != "/xx/sessions" && url != "/sessions" { + cli.ReLoginMutex.Lock() + req, err = cli.GetRequest(ctx, method, url, data) + cli.ReLoginMutex.Unlock() + } else { + req, err = cli.GetRequest(ctx, method, url, data) + } + + if err != nil { + return base.Response{}, err + } + + log.FilteredLog(ctx, isFilterLog(method, url), utils.IsDebugLog(method, url, debugLog, debugLogRegex), + fmt.Sprintf("Request method: %s, Url: %s, body: %v", method, req.URL, data)) + + if cli.RequestSemaphore == nil { + return base.Response{}, errors.New("request semaphore is nil") + } + + cli.RequestSemaphore.Acquire() + defer cli.RequestSemaphore.Release() + + if base.RequestSemaphoreMap[cli.GetDeviceSN()] != nil { + base.RequestSemaphoreMap[cli.GetDeviceSN()].Acquire() + defer base.RequestSemaphoreMap[cli.GetDeviceSN()].Release() + } else { + base.RequestSemaphoreMap[base.UninitializedStorage].Acquire() + defer base.RequestSemaphoreMap[base.UninitializedStorage].Release() + } + + resp, err := cli.Client.Do(req) + if err != nil { + log.AddContext(ctx).Errorf("Send request method: %s, Url: %s, error: %v", method, req.URL, err) + return base.Response{}, errors.New(base.Unconnected) + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + log.AddContext(ctx).Errorf("Read response data error: %v", err) + return base.Response{}, err + } + + log.FilteredLog(ctx, isFilterLog(method, url), utils.IsDebugLog(method, url, debugLog, debugLogRegex), + fmt.Sprintf("Response method: %s, Url: %s, body: %s", method, req.URL, body)) + + err = json.Unmarshal(body, &r) + if err != nil { + log.AddContext(ctx).Errorf("json.Unmarshal data %s error: %v", body, err) + return base.Response{}, err + } + + return r, nil +} + +// Get provides http request of GET method +func (cli *RestClient) Get(ctx context.Context, url string, data map[string]interface{}) (base.Response, error) { + return cli.Call(ctx, "GET", url, data) +} + +// Post provides http request of POST method +func (cli *RestClient) Post(ctx context.Context, url string, data map[string]interface{}) (base.Response, error) { + return cli.Call(ctx, "POST", url, data) +} + +// Put provides http request of PUT method +func (cli *RestClient) Put(ctx context.Context, url string, data map[string]interface{}) (base.Response, error) { + return cli.Call(ctx, "PUT", url, data) +} + +// Delete provides http request of DELETE method +func (cli *RestClient) Delete(ctx context.Context, url string, data map[string]interface{}) (base.Response, error) { + return cli.Call(ctx, "DELETE", url, data) +} + +// GetRequest return the request info +func (cli *RestClient) GetRequest(ctx context.Context, + method string, url string, data map[string]interface{}) (*http.Request, error) { + var req *http.Request + var err error + + reqUrl := cli.Url + if cli.DeviceId != "" { + reqUrl += "/" + cli.DeviceId + } + reqUrl += url + + var reqBody io.Reader + + if data != nil { + reqBytes, err := json.Marshal(data) + if err != nil { + log.AddContext(ctx).Errorf("json.Marshal data %v error: %v", base.MaskRequestData(data), err) + return nil, err + } + reqBody = bytes.NewReader(reqBytes) + } + + req, err = http.NewRequest(method, reqUrl, reqBody) + if err != nil { + log.AddContext(ctx).Errorf("construct http request error: %v", err) + return nil, err + } + + if req == nil || req.Header == nil { + log.AddContext(ctx).Errorln("construct http request error: request header init failed") + return nil, err + } + + req.Header.Set("Connection", "keep-alive") + req.Header.Set("Content-Type", "application/json") + + if cli.Token != "" { + req.Header.Set("iBaseToken", cli.Token) + } + + return req, nil +} + +// Login login and set data from response +func (cli *RestClient) Login(ctx context.Context) error { + var err error + + if cli.Client, err = base.NewHTTPClientByBackendID(ctx, cli.BackendID); err != nil { + return pkgUtils.Errorln(ctx, + fmt.Sprintf("new http client by backend %s failed, err is %v", cli.BackendID, err)) + } + + data, err := cli.getRequestParams(ctx, cli.BackendID) + if err != nil { + return pkgUtils.Errorln(ctx, fmt.Sprintf("get reuqest failed while login, error : %v", err)) + } + + cli.DeviceId, cli.Token = "", "" + + resp, err := cli.loginCall(ctx, data) + if err != nil { + return pkgUtils.Errorln(ctx, fmt.Sprintf("request storage failed while login, error : %v", err)) + } + + code, _, err := utils.FormatRespErr(resp.Error) + if code != 0 { + msg := fmt.Sprintf("login %s error: %+v", cli.Url, resp) + if utils.Contains(base.WrongPasswordErrorCodes, code) || utils.Contains(base.AccountBeenLocked, code) || + code == base.IPLockErrorCode { + if err := pkgUtils.SetStorageBackendContentOnlineStatus(ctx, cli.BackendID, false); err != nil { + msg = msg + fmt.Sprintf("\nsetStorageBackendContentOffline [%s] failed, "+ + "error: %v", cli.BackendID, err) + } + } + return pkgUtils.Errorln(ctx, msg) + } + + if err = cli.setDataFromRespData(ctx, resp); err != nil { + cli.Logout(ctx) + setErr := pkgUtils.SetStorageBackendContentOnlineStatus(ctx, cli.BackendID, false) + if setErr != nil { + log.AddContext(ctx).Errorf("setStorageBackendContentOffline [%s] failed, "+ + "error: %v", cli.BackendID, setErr) + } + return pkgUtils.Errorln(ctx, err.Error()) + } + return nil +} + +func (cli *RestClient) loginCall(ctx context.Context, data map[string]interface{}) (base.Response, error) { + var resp base.Response + var err error + for i, url := range cli.Urls { + cli.Url = url + "/deviceManager/rest" + log.AddContext(ctx).Infof("try to login %s", cli.Url) + + resp, err = cli.BaseCall(ctx, "POST", "/xx/sessions", data) + if err == nil { + /* Sort the login Url to the last slot of san addresses, so that + if this connection error, next time will try other Url first. */ + cli.Urls[i], cli.Urls[len(cli.Urls)-1] = cli.Urls[len(cli.Urls)-1], cli.Urls[i] + break + } else if err.Error() != base.Unconnected { + log.AddContext(ctx).Errorf("login %s error", cli.Url) + break + } + + log.AddContext(ctx).Warningf("login %s error due to connection failure, gonna try another Url", cli.Url) + } + + if err != nil { + return base.Response{}, err + } + return resp, err +} + +func (cli *RestClient) getRequestParams(ctx context.Context, backendID string) (map[string]interface{}, error) { + password, err := pkgUtils.GetPasswordFromBackendID(ctx, backendID) + if err != nil { + return nil, err + } + + data := map[string]interface{}{ + "username": cli.User, + "password": password, + "scope": base.LocalUserType, + } + + return data, err +} + +func (cli *RestClient) setDataFromRespData(ctx context.Context, resp base.Response) error { + respData, ok := resp.Data.(map[string]interface{}) + if !ok { + return fmt.Errorf("convert resp.Data to map[string]interface{} failed, data type: [%T]", resp.Data) + } + + cli.DeviceId, ok = utils.GetValue[string](respData, "deviceid") + if !ok { + return fmt.Errorf("convert respData[\"deviceid\"]: [%T] to string failed", respData["deviceid"]) + } + + if base.RequestSemaphoreMap[cli.DeviceId] == nil { + base.RequestSemaphoreMap[cli.DeviceId] = utils.NewSemaphore(base.MaxStorageThreads) + } + + cli.Token, ok = utils.GetValue[string](respData, "iBaseToken") + if !ok { + return fmt.Errorf("convert respData[\"iBaseToken\"]: [%T] to string failed", respData["iBaseToken"]) + } + + log.AddContext(ctx).Infof("login %s success", cli.Url) + return nil +} + +// Logout logout +func (cli *RestClient) Logout(ctx context.Context) { + resp, err := cli.BaseCall(ctx, "DELETE", "/sessions", nil) + if err != nil { + log.AddContext(ctx).Warningf("logout %s error: %v", cli.Url, err) + return + } + + code := int64(resp.Error["code"].(float64)) + if code != 0 { + log.AddContext(ctx).Warningf("logout %s error: %d", cli.Url, code) + return + } + + log.AddContext(ctx).Infof("logout %s success", cli.Url) +} + +// ReLogin logout and login again +func (cli *RestClient) ReLogin(ctx context.Context) error { + oldToken := cli.Token + + cli.ReLoginMutex.Lock() + defer cli.ReLoginMutex.Unlock() + + if cli.Token != "" && oldToken != cli.Token { + // Coming here indicates other thread had already done relogin, so no need to relogin again + return nil + } else if cli.Token != "" { + cli.Logout(ctx) + } + + err := cli.Login(ctx) + if err != nil { + log.AddContext(ctx).Errorf("try to relogin error: %v", err) + return err + } + + return nil +} + +// GetBackendID get backend id of client +func (cli *RestClient) GetBackendID() string { + return cli.DeviceId +} + +// GetDeviceSN used for get device sn +func (cli *RestClient) GetDeviceSN() string { + return cli.DeviceId +} + +// GetStorageVersion used for get storage version +func (cli *RestClient) GetStorageVersion() string { + return cli.StorageVersion +} + +// GetSystem used for get system info +func (cli *RestClient) GetSystem(ctx context.Context) (map[string]interface{}, error) { + resp, err := cli.Get(ctx, "/system/", nil) + if err != nil { + return nil, err + } + + code, msg, err := utils.FormatRespErr(resp.Error) + if err != nil { + return nil, err + } + + if code != 0 { + return nil, fmt.Errorf("get system info failed, error code: %d, error msg: %s", code, msg) + } + + respData, ok := resp.Data.(map[string]interface{}) + if !ok { + return nil, fmt.Errorf("convert respData to map failed, data: %v", resp.Data) + } + + return respData, nil +} + +// NewRestClient inits a new rest client +func NewRestClient(ctx context.Context, param *NewClientConfig) (*RestClient, error) { + var err error + var parallelCount int + + parallelCount, err = strconv.Atoi(param.ParallelNum) + if err != nil || parallelCount > MaxParallelCount || parallelCount < MinParallelCount { + log.Infof("the config parallelNum %d is invalid, set it to the default value %d", + parallelCount, DefaultParallelCount) + parallelCount = DefaultParallelCount + } + + log.AddContext(ctx).Infof("Init parallel count is %d", parallelCount) + httpClient, err := base.NewHTTPClientByCertMeta(ctx, param.UseCert, param.CertSecretMeta) + if err != nil { + log.AddContext(ctx).Errorf("new http client by cert meta failed, err is %v", err) + return nil, err + } + + return &RestClient{ + Urls: param.Urls, + User: param.User, + Storage: param.Storage, + SecretName: param.SecretName, + SecretNamespace: param.SecretNamespace, + Client: httpClient, + BackendID: param.BackendID, + RequestSemaphore: utils.NewSemaphore(parallelCount), + }, nil +} diff --git a/storage/oceanstorage/oceandisk/client/client_restful_test.go b/storage/oceanstorage/oceandisk/client/client_restful_test.go new file mode 100644 index 00000000..eed004c6 --- /dev/null +++ b/storage/oceanstorage/oceandisk/client/client_restful_test.go @@ -0,0 +1,427 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package client provides oceandisk storage client +package client + +import ( + "context" + "encoding/json" + "errors" + "io" + "net/http" + "net/http/httptest" + "reflect" + "strconv" + "sync" + "sync/atomic" + "testing" + "time" + + "github.com/agiledragon/gomonkey/v2" + "github.com/stretchr/testify/require" + + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/base" +) + +func TestBaseClient_GetRequest_Success(t *testing.T) { + // arrange + method := "GET" + url := "/mockUrl" + data := map[string]interface{}{} + client, _ := NewRestClient(context.Background(), &NewClientConfig{}) + + // act + getRequest, getErr := client.GetRequest(context.TODO(), method, url, data) + + // assert + if getErr != nil || getRequest == nil { + t.Errorf("TestBaseClient_GetRequest_Success failed, "+ + "wantErr = nil, gotErr = %v, wantRequest != nil, gotRequest = %v", getErr, getRequest) + } +} + +func TestBaseClient_GetRequest_JsonMarshalFailed(t *testing.T) { + // arrange + method := "GET" + url := "/mockUrl" + data := map[string]interface{}{} + client, _ := NewRestClient(context.Background(), &NewClientConfig{}) + wantErr := errors.New("json error") + + // mock + mock := gomonkey.NewPatches() + mock.ApplyFunc(json.Marshal, func(v any) ([]byte, error) { + return nil, wantErr + }) + + // act + _, getErr := client.GetRequest(context.TODO(), method, url, data) + + // assert + if !reflect.DeepEqual(getErr, wantErr) { + t.Errorf("TestBaseClient_GetRequest_JsonMarshalFailed failed, "+ + "wantErr = %v, gotErr = %v", wantErr, getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestBaseClient_GetRequest_newRequestFailed(t *testing.T) { + // arrange + method := "GET" + url := "/mockUrl" + data := map[string]interface{}{} + client, _ := NewRestClient(context.Background(), &NewClientConfig{}) + wantErr := errors.New("new request error") + + // mock + mock := gomonkey.NewPatches() + mock.ApplyFunc(http.NewRequest, func(method, url string, body io.Reader) (*http.Request, error) { + return nil, wantErr + }) + + // act + _, getErr := client.GetRequest(context.TODO(), method, url, data) + + // assert + if !reflect.DeepEqual(getErr, wantErr) { + t.Errorf("TestBaseClient_GetRequest_newRequestFailed failed, "+ + "wantErr = %v, gotErr = %v", wantErr, getErr) + } + + // cleanup + t.Cleanup(func() { + mock.Reset() + }) +} + +func TestBaseClient_BaseCall_Success(t *testing.T) { + // arrange + method := "GET" + data := map[string]interface{}{} + mockClient, _ := NewRestClient(context.Background(), &NewClientConfig{}) + wantResponse := base.Response{ + Error: make(map[string]interface{}), + Data: "", + } + + responseByte, err := json.Marshal(wantResponse) + if err != nil { + return + } + + // mock + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + w.WriteHeader(http.StatusOK) + _, err = w.Write(responseByte) + if err != nil { + return + } + })) + + getResponse, getErr := mockClient.BaseCall(context.TODO(), method, mockServer.URL, data) + + // assert + if !reflect.DeepEqual(getResponse, wantResponse) || getErr != nil { + t.Errorf("TestBaseClient_BaseCall_Success failed, "+ + "wantRes = %v, getRes = %v, wantErr = nil, gotErr = %v", wantResponse, getResponse, getErr) + } + + // cleanup + t.Cleanup(func() { + mockServer.Close() + }) +} + +func TestRestClient_BaseCall_Concurrency(t *testing.T) { + // arrange + method := "GET" + data := map[string]interface{}{} + client, _ := NewRestClient(context.Background(), &NewClientConfig{}) + wantResponse := base.Response{ + Error: make(map[string]interface{}), + Data: "", + } + + responseByte, err := json.Marshal(wantResponse) + if err != nil { + return + } + + requestTime := 10 + wantAvailablePermits := base.MaxStorageThreads - requestTime + wg := sync.WaitGroup{} + wg.Add(requestTime) + + // mock + mockServer := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + wg.Done() + wg.Wait() + time.Sleep(100 * time.Millisecond) + + w.WriteHeader(http.StatusOK) + _, err = w.Write(responseByte) + if err != nil { + return + } + })) + + // action + for i := 0; i < requestTime; i++ { + go func() { _, _ = client.BaseCall(context.TODO(), method, mockServer.URL, data) }() + } + wg.Wait() + + // assert + getAvailablePermits := base.RequestSemaphoreMap[base.UninitializedStorage].AvailablePermits() + if getAvailablePermits != wantAvailablePermits { + t.Errorf("TestRestClient_BaseCall_Concurrency failed, "+ + "wantAvailablePermits = %d, getAvailablePermits = %d", wantAvailablePermits, getAvailablePermits) + } + + // cleanup + t.Cleanup(func() { + mockServer.Close() + }) +} + +func TestRestClient_BaseCall_ExceedMaxConcurrency(t *testing.T) { + // arrange + wantResponse := base.Response{ + Error: make(map[string]interface{}), + Data: "", + } + + responseByte, err := json.Marshal(wantResponse) + if err != nil { + return + } + + var currentConcurrent atomic.Int32 + var errCount atomic.Int32 + wg := sync.WaitGroup{} + wg.Add(100 * DefaultParallelCount) + + // mock + mockServer1 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer wg.Done() + cur := currentConcurrent.Add(1) + if cur > int32(base.MaxStorageThreads) { + errCount.Add(1) + } + time.Sleep(10 * time.Millisecond) + currentConcurrent.Add(-1) + + w.WriteHeader(http.StatusOK) + _, err = w.Write(responseByte) + if err != nil { + return + } + })) + + // action + for num := 0; num < 100; num++ { + go func() { + client, _ := NewRestClient(context.Background(), &NewClientConfig{}) + for i := 0; i < DefaultParallelCount; i++ { + go func() { + _, _ = client.BaseCall(context.TODO(), "GET", + mockServer1.URL, map[string]interface{}{}) + }() + } + }() + } + + //assert + wg.Wait() + require.Equal(t, currentConcurrent.Load(), int32(0)) + require.Equal(t, errCount.Load(), int32(0)) + + // cleanup + t.Cleanup(func() { + mockServer1.Close() + }) +} + +func TestRestClient_BaseCall_IsReachMaxConcurrency(t *testing.T) { + // arrange + wantResponse := base.Response{ + Error: make(map[string]interface{}), + Data: "", + } + + responseByte, err := json.Marshal(wantResponse) + if err != nil { + return + } + + var currentConcurrent atomic.Int32 + var errCount atomic.Int32 + wg := sync.WaitGroup{} + wg.Add(100 * DefaultParallelCount) + + // mock + mockServer1 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer wg.Done() + cur := currentConcurrent.Add(1) + if cur > int32(base.MaxStorageThreads-1) { + errCount.Add(1) + } + time.Sleep(10 * time.Millisecond) + currentConcurrent.Add(-1) + + w.WriteHeader(http.StatusOK) + _, err = w.Write(responseByte) + if err != nil { + return + } + })) + + // action + for num := 0; num < 100; num++ { + go func() { + client, _ := NewRestClient(context.Background(), &NewClientConfig{}) + for i := 0; i < DefaultParallelCount; i++ { + go func() { + _, _ = client.BaseCall(context.TODO(), "GET", + mockServer1.URL, map[string]interface{}{}) + }() + } + }() + } + + //assert + wg.Wait() + require.Equal(t, currentConcurrent.Load(), int32(0)) + require.Greater(t, errCount.Load(), int32(0)) + + // cleanup + t.Cleanup(func() { + mockServer1.Close() + }) +} + +func TestRestClient_BaseCall_ExceedMaxBackendConcurrency(t *testing.T) { + // arrange + method := "GET" + data := map[string]interface{}{} + var currentConcurrent atomic.Int32 + var errCount atomic.Int32 + maxConcurrent := 10 + client, _ := NewRestClient(context.Background(), &NewClientConfig{ParallelNum: strconv.Itoa(int(maxConcurrent))}) + wantResponse := base.Response{ + Error: make(map[string]interface{}), + Data: "", + } + + responseByte, err := json.Marshal(wantResponse) + if err != nil { + return + } + + wg := sync.WaitGroup{} + wg.Add(maxConcurrent * 2) + + // mock + mockServer1 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer wg.Done() + cur := currentConcurrent.Add(1) + if cur > int32(maxConcurrent) { + errCount.Add(1) + } + time.Sleep(10 * time.Millisecond) + currentConcurrent.Add(-1) + + w.WriteHeader(http.StatusOK) + _, err = w.Write(responseByte) + if err != nil { + return + } + })) + + // action + for i := 0; i < maxConcurrent*2; i++ { + go func() { _, _ = client.BaseCall(context.TODO(), method, mockServer1.URL, data) }() + } + + //assert + wg.Wait() + require.Equal(t, currentConcurrent.Load(), int32(0)) + require.Equal(t, errCount.Load(), int32(0)) + + // cleanup + t.Cleanup(func() { + mockServer1.Close() + }) +} + +func TestRestClient_BaseCall_IsReachMaxBackendConcurrency(t *testing.T) { + // arrange + method := "GET" + data := map[string]interface{}{} + var currentConcurrent atomic.Int32 + var errCount atomic.Int32 + maxConcurrent := 10 + client, _ := NewRestClient(context.Background(), &NewClientConfig{ParallelNum: strconv.Itoa(int(maxConcurrent))}) + wantResponse := base.Response{ + Error: make(map[string]interface{}), + Data: "", + } + + responseByte, err := json.Marshal(wantResponse) + if err != nil { + return + } + + wg := sync.WaitGroup{} + wg.Add(maxConcurrent * 10) + + // mock + mockServer1 := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + defer wg.Done() + cur := currentConcurrent.Add(1) + if cur > int32(maxConcurrent-1) { + errCount.Add(1) + } + time.Sleep(10 * time.Millisecond) + currentConcurrent.Add(-1) + + w.WriteHeader(http.StatusOK) + _, err = w.Write(responseByte) + if err != nil { + return + } + })) + + // action + for i := 0; i < maxConcurrent*10; i++ { + go func() { _, _ = client.BaseCall(context.TODO(), method, mockServer1.URL, data) }() + } + + //assert + wg.Wait() + require.Equal(t, currentConcurrent.Load(), int32(0)) + require.Greater(t, errCount.Load(), int32(0)) + + // cleanup + t.Cleanup(func() { + mockServer1.Close() + }) +} diff --git a/storage/oceanstorage/oceandisk/client/client_test.go b/storage/oceanstorage/oceandisk/client/client_test.go new file mode 100644 index 00000000..05434a83 --- /dev/null +++ b/storage/oceanstorage/oceandisk/client/client_test.go @@ -0,0 +1,42 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package client provides oceandisk storage client +package client + +import ( + "testing" + + "github.com/prashantv/gostub" + + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + cfg "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" +) + +const ( + logName = "clientTest.log" +) + +func TestMain(m *testing.M) { + log.MockInitLogging(logName) + defer log.MockStopLogging(logName) + + getGlobalConfig := gostub.StubFunc(&app.GetGlobalConfig, cfg.MockCompletedConfig()) + defer getGlobalConfig.Reset() + + m.Run() +} diff --git a/storage/oceanstorage/oceandisk/smartx/smartx.go b/storage/oceanstorage/oceandisk/smartx/smartx.go new file mode 100644 index 00000000..e950c37d --- /dev/null +++ b/storage/oceanstorage/oceandisk/smartx/smartx.go @@ -0,0 +1,345 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package smartx provides operations for storage qos +package smartx + +import ( + "context" + "encoding/json" + "fmt" + "math/big" + "slices" + "strings" + "time" + + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/base" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceandisk/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" +) + +const ( + kilo = 1000 + minLatency, maxLatency = 500, 1500 + minIops, maxIops = 99, 999999999 + minBandwidth, maxBandwidth = 0, 999999999 + ioType = 2 +) + +var ( + validator = map[string]func(int) bool{ + "IOTYPE": func(value int) bool { + return value == ioType + }, + "MAXBANDWIDTH": func(value int) bool { + return minBandwidth < value && value <= maxBandwidth + + }, + "MINBANDWIDTH": func(value int) bool { + return minBandwidth < value && value <= maxBandwidth + + }, + "MAXIOPS": func(value int) bool { + return minIops < value && value <= maxIops + + }, + "MINIOPS": func(value int) bool { + return minIops < value && value <= maxIops + + }, + "LATENCY": func(value int) bool { + // User request Latency values in millisecond but during extraction values are converted in microsecond + // as required in Oceandisk QoS create interface + return value == minLatency || value == maxLatency + }, + } + + oceandiskParameters = []string{"MAXBANDWIDTH", "MINBANDWIDTH", "MAXIOPS", "MINIOPS", "LATENCY"} +) + +// CheckQoSParameterSupport verify QoS supported parameters and value validation +func CheckQoSParameterSupport(ctx context.Context, qosConfig string) error { + qosParam, err := ExtractQoSParameters(ctx, qosConfig) + if err != nil { + return utils.Errorln(ctx, err.Error()) + } + + err = validateQoSParametersSupport(qosParam) + if err != nil { + return utils.Errorln(ctx, err.Error()) + } + + return nil +} + +func validateQoSParametersSupport(qosParam map[string]float64) error { + // validate QoS parameters and parameter ranges + for k, v := range qosParam { + f, exist := validator[k] + if !exist { + return fmt.Errorf("%s is a invalid key for Oceandisk QoS", k) + } + + if !f(int(v)) { // silently ignoring decimal number + return fmt.Errorf("%s of qos parameter has invalid value", k) + } + } + return nil +} + +// ExtractQoSParameters unmarshal QoS configuration parameters +func ExtractQoSParameters(ctx context.Context, qosConfig string) (map[string]float64, error) { + var unmarshalParams map[string]interface{} + params := make(map[string]float64) + + err := json.Unmarshal([]byte(qosConfig), &unmarshalParams) + if err != nil { + return nil, fmt.Errorf("failed to unmarshal qos parameters[ %s ] error: %v", qosConfig, err) + } + + // translate values based on Oceandisk product's QoS create interface + for key, val := range unmarshalParams { + // all numbers are unmarshalled as float64 in unmarshalParams + // assert for other than number + value, ok := val.(float64) + if !ok { + return nil, fmt.Errorf("invalid QoS parameter [%s] with value type [%T]", key, val) + } + + if key == "LATENCY" { + // convert Latency from millisecond to microsecond + params[key] = value * kilo + } else { + params[key] = value + } + } + + return params, nil +} + +// ValidateQoSParameters check QoS parameters +func ValidateQoSParameters(qosParam map[string]float64) (map[string]int, error) { + // ensure at least one parameter + paramExist := false + for _, param := range oceandiskParameters { + if _, exist := qosParam[param]; exist { + paramExist = true + break + } + } + + if !paramExist { + return nil, fmt.Errorf("missing one of QoS parameter %v ", strings.Join(oceandiskParameters, ",")) + } + + // validate QoS param value + validatedParameters := make(map[string]int) + for key, value := range qosParam { + // check if not integer + if !big.NewFloat(value).IsInt() { + return nil, fmt.Errorf("the QoS parameter %s has invalid value type [%T]. "+ + "It should be integer", key, value) + } + validatedParameters[key] = int(value) + } + + return validatedParameters, nil +} + +// Client provides smartx client +type Client struct { + cli client.OceandiskClientInterface +} + +// NewSmartX inits a new smartx client +func NewSmartX(cli client.OceandiskClientInterface) *Client { + return &Client{ + cli: cli, + } +} + +func (p *Client) getQosName(objID string) string { + now := time.Now().Format("20060102150405") + return fmt.Sprintf("k8s_namespace%s_%s", objID, now) +} + +// CreateQos creates qos and return its id +func (p *Client) CreateQos(ctx context.Context, objID string, params map[string]int) (string, error) { + var err error + var lowerLimit bool + + for k := range params { + if strings.HasPrefix(k, "MIN") || strings.HasPrefix(k, "LATENCY") { + lowerLimit = true + } + } + + if lowerLimit { + data := map[string]interface{}{ + "IOPRIORITY": 3, + } + + err = p.cli.UpdateNamespace(ctx, objID, data) + + if err != nil { + return "", utils.Errorf(ctx, "upgrade obj %s of type %s IOPRIORITY error: %v", objID, objID, err) + } + } + + name := p.getQosName(objID) + qos, err := p.cli.CreateQos(ctx, p.getCreateQosArgs(name, objID, params)) + if err != nil { + return "", utils.Errorf(ctx, "create qos %v for obj %s of type namespace error: %v", params, objID, err) + } + + qosID, ok := utils.GetValue[string](qos, "ID") + if !ok { + return "", utils.Errorf(ctx, "qos ID is expected as string, get %T", qos["ID"]) + } + + qosStatus, ok := utils.GetValue[string](qos, "ENABLESTATUS") + if !ok { + return "", utils.Errorf(ctx, "qos ENABLESTATUS is expected as string, get %T", qos["ENABLESTATUS"]) + } + + if qosStatus == "false" { + err := p.cli.ActivateQos(ctx, qosID, "") + if err != nil { + return "", utils.Errorf(ctx, "activate qos %s error: %v", qosID, err) + } + } + + return qosID, nil +} + +// DeleteQos deletes qos by id +func (p *Client) DeleteQos(ctx context.Context, qosID, objID string) error { + qos, err := p.cli.GetQosByID(ctx, qosID, "") + if err != nil { + return utils.Errorf(ctx, "get qos by ID %s error: %v", qosID, err) + } + + objList, err := getObjIdListByQos(qos) + if err != nil { + return utils.Errorf(ctx, "delete qos %s failed, error: %v", qosID, err) + } + + var leftList []string + for _, i := range objList { + if i != objID { + leftList = append(leftList, i) + } + } + + if len(leftList) > 0 { + log.AddContext(ctx).Warningf("there are some other obj %v associated to qos %s", leftList, qosID) + params := map[string]interface{}{ + "LUNLIST": leftList, + } + err := p.cli.UpdateQos(ctx, qosID, "", params) + if err != nil { + return utils.Errorf(ctx, "remove obj %s of type namespace from qos %s error: %v", objID, qosID, err) + } + + return nil + } + + err = p.cli.DeactivateQos(ctx, qosID, "") + if err != nil { + return utils.Errorf(ctx, "deactivate qos %s error: %v", qosID, err) + } + + err = p.cli.DeleteQos(ctx, qosID, "") + if err != nil { + return utils.Errorf(ctx, "delete qos %s error: %v", qosID, err) + } + + return nil +} + +func getObjIdListByQos(qos map[string]interface{}) ([]string, error) { + listStr, ok := utils.GetValue[string](qos, "LUNLIST") + if !ok { + return nil, fmt.Errorf("qos volume list is expected as marshaled string, get %T", qos["LUNLIST"]) + } + + var objList []string + err := json.Unmarshal([]byte(listStr), &objList) + if err != nil { + return nil, fmt.Errorf("unmarshal %s error: %v", listStr, err) + } + + return objList, nil +} + +// DeleteQosByNamespaceId deletes qos by namespace id +func (p *Client) DeleteQosByNamespaceId(ctx context.Context, namespaceId string) error { + qosList, err := p.cli.GetAllQos(ctx) + if err != nil { + return utils.Errorf(ctx, "get all qos failed, error: %v", err) + } + + for _, qos := range qosList { + qosID, ok := utils.GetValue[string](qos, "ID") + if !ok { + log.AddContext(ctx).Warningf("qos ID is expected as string, get %T", qos["ID"]) + continue + } + + objList, err := getObjIdListByQos(qos) + if err != nil { + log.AddContext(ctx).Warningf("get qos %s related namespaces failed, error: %v", qosID, err) + continue + } + + index := slices.Index(objList, namespaceId) + if index == -1 { + continue + } + + objList = slices.Delete(objList, index, index+1) + if len(objList) > 0 { + log.AddContext(ctx).Warningf("there are some other obj %v associated to qos %v", objList, qos) + params := map[string]interface{}{"LUNLIST": objList} + err := p.cli.UpdateQos(ctx, qosID, "", params) + if err != nil { + return utils.Errorf(ctx, "remove namespace %s from qos %s error: %v", namespaceId, qosID, err) + } + continue + } + + err = p.cli.DeactivateQos(ctx, qosID, "") + if err != nil { + return utils.Errorf(ctx, "deactivate qos %s error: %v", qosID, err) + } + + err = p.cli.DeleteQos(ctx, qosID, "") + if err != nil { + return utils.Errorf(ctx, "delete qos %s error: %v", qosID, err) + } + } + + return nil +} + +func (p *Client) getCreateQosArgs(name, objID string, params map[string]int) base.CreateQoSArgs { + return base.CreateQoSArgs{ + Name: name, + ObjID: objID, + Params: params, + } +} diff --git a/storage/oceanstorage/oceandisk/volume/base.go b/storage/oceanstorage/oceandisk/volume/base.go new file mode 100644 index 00000000..abfbadf7 --- /dev/null +++ b/storage/oceanstorage/oceandisk/volume/base.go @@ -0,0 +1,146 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package volume is used for OceanDisk base +package volume + +import ( + "context" + "errors" + "fmt" + + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceandisk/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceandisk/smartx" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" +) + +// Base defines the base storage client +type Base struct { + // The current product does not support the HyperMetro and replication capabilities. + // The corresponding CLI will be added after the capabilities are supplemented. + cli client.OceandiskClientInterface + product constants.OceanDiskVersion +} + +func (p *Base) prepareVolObj(ctx context.Context, params, res map[string]interface{}) (utils.Volume, error) { + volName, ok := params["name"].(string) + if !ok { + return nil, utils.Errorf(ctx, "expecting string for volume name, received type %T", params["name"]) + } + + volObj := utils.NewVolume(volName) + if res != nil { + if lunWWN, ok := res["lunWWN"].(string); ok { + volObj.SetLunWWN(lunWWN) + } + } + + capacity := utils.GetValueOrFallback(params, "capacity", int64(0)) + volObj.SetSize(utils.TransK8SCapacity(capacity, constants.AllocationUnitBytes)) + + return volObj, nil +} + +func (p *Base) commonPreCreate(ctx context.Context, params map[string]interface{}) error { + analyzers := [...]func(context.Context, map[string]interface{}) error{ + p.getPoolID, + p.getQoS, + } + + for _, analyzer := range analyzers { + err := analyzer(ctx, params) + if err != nil { + return err + } + } + + return nil +} + +func (p *Base) getPoolID(ctx context.Context, params map[string]interface{}) error { + if params == nil { + return errors.New("getPoolID params is nil") + } + + poolName, ok := params["storagepool"].(string) + if !ok || poolName == "" { + return errors.New("must specify storage pool to create volume") + } + + pool, err := p.cli.GetPoolByName(ctx, poolName) + if err != nil { + return err + } + if len(pool) == 0 { + return fmt.Errorf("storage pool %s doesn't exist", poolName) + } + + params["poolID"] = pool["ID"] + return nil +} + +func (p *Base) getQoS(ctx context.Context, params map[string]interface{}) error { + if params == nil { + return errors.New("getQoS params is nil") + } + + if v, exist := params["qos"].(string); exist && v != "" { + qos, err := smartx.ExtractQoSParameters(ctx, v) + if err != nil { + return fmt.Errorf("extract qos parameter [%s] failed, error: %v", v, err) + } + + validatedQos, err := smartx.ValidateQoSParameters(qos) + if err != nil { + return fmt.Errorf("validate qos parameters [%v] failed, error: %v", qos, err) + } + params["qos"] = validatedQos + } + + return nil +} + +func (p *Base) setWorkLoadID(ctx context.Context, cli client.OceandiskClientInterface, + params map[string]interface{}) error { + if params == nil { + return errors.New("setWorkLoadID params is nil") + } + + if val, ok := params["applicationtype"].(string); ok { + workloadTypeID, err := p.getWorkLoadIDByName(ctx, cli, val) + if err != nil { + return err + } + params["workloadTypeID"] = workloadTypeID + } + + return nil +} + +func (p *Base) getWorkLoadIDByName(ctx context.Context, cli client.OceandiskClientInterface, workloadTypeName string) ( + string, error) { + workloadTypeID, err := cli.GetApplicationTypeByName(ctx, workloadTypeName) + if err != nil { + return "", err + } + + if workloadTypeID == "" { + return "", fmt.Errorf("workloadType does not exist on storage") + } + + return workloadTypeID, nil +} diff --git a/storage/oceanstorage/oceandisk/volume/base_test.go b/storage/oceanstorage/oceandisk/volume/base_test.go new file mode 100644 index 00000000..bb7fa5bb --- /dev/null +++ b/storage/oceanstorage/oceandisk/volume/base_test.go @@ -0,0 +1,55 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package volume is used for OceanDisk san test +package volume + +import ( + "context" + "testing" +) + +func TestBase_getQoS_Success(t *testing.T) { + // arrange + ctx := context.Background() + b := &Base{} + param := map[string]interface{}{ + "qos": map[string]int{"maxIOPS": 999, "maxMBPS": 999}, + } + + // action + gotErr := b.getQoS(ctx, param) + + // assert + if gotErr != nil { + t.Errorf("TestBase_getQoS_Success failed, gotErr = %v, wantErr = %v.", gotErr, nil) + } +} + +func TestBase_getQoS_Empty(t *testing.T) { + // arrange + ctx := context.Background() + b := &Base{} + param := map[string]interface{}{} + + // action + gotErr := b.getQoS(ctx, param) + + // assert + if gotErr != nil { + t.Errorf("TestBase_getQoS_Empty failed, gotErr = %v, wantErr = %v.", gotErr, nil) + } +} diff --git a/storage/oceanstorage/oceandisk/volume/san.go b/storage/oceanstorage/oceandisk/volume/san.go new file mode 100644 index 00000000..0a1aa279 --- /dev/null +++ b/storage/oceanstorage/oceandisk/volume/san.go @@ -0,0 +1,321 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package volume is used for OceanDisk san +package volume + +import ( + "context" + "fmt" + "strconv" + + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceandisk/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceandisk/smartx" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/flow" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" +) + +// SAN provides base san client +type SAN struct { + Base +} + +// NewSAN inits a new san client +func NewSAN(cli client.OceandiskClientInterface) *SAN { + return &SAN{ + Base: Base{ + cli: cli, + }, + } +} + +func (p *SAN) preCreate(ctx context.Context, params map[string]interface{}) error { + err := p.commonPreCreate(ctx, params) + if err != nil { + return err + } + + err = p.setWorkLoadID(ctx, p.cli, params) + if err != nil { + return err + } + + return nil +} + +// Create creates lun volume +func (p *SAN) Create(ctx context.Context, params map[string]interface{}) (utils.Volume, error) { + err := p.preCreate(ctx, params) + if err != nil { + return nil, err + } + + taskFlow := flow.NewTaskFlow(ctx, "Create-Namespace-Volume") + + taskFlow.AddTask("Create-Local-Namespace", p.createLocalNamespace, p.revertLocalNamespace) + taskFlow.AddTask("Create-Local-QoS", p.createLocalQoS, p.revertLocalQoS) + + res, err := taskFlow.Run(params) + if err != nil { + taskFlow.Revert() + return nil, err + } + + return p.prepareVolObj(ctx, params, res) +} + +func (p *SAN) createLocalNamespace(ctx context.Context, params, taskResult map[string]interface{}) ( + map[string]interface{}, error) { + // 1. Check whether the namespace exists. + namespaceName, ok := params["name"].(string) + if !ok { + return nil, fmt.Errorf("assert namespace name: [%v] to string failed", params["name"]) + } + + namespace, err := p.cli.GetNamespaceByName(ctx, namespaceName) + if err != nil { + return nil, err + } + + // 2. Do create. + if len(namespace) == 0 { + createNamespaceParams, err := client.MakeCreateNamespaceParams(params) + if err != nil { + return nil, err + } + + namespace, err = p.cli.CreateNamespace(ctx, *createNamespaceParams) + if err != nil { + return nil, err + } + } + + return map[string]interface{}{ + "localNamespaceID": namespace["ID"], + "namespaceWWN": namespace["WWN"], + }, nil +} + +func (p *SAN) revertLocalNamespace(ctx context.Context, taskResult map[string]interface{}) error { + namespaceID, ok := utils.GetValue[string](taskResult, "localNamespaceID") + if !ok || namespaceID == "" { + return nil + } + + return p.cli.DeleteNamespace(ctx, namespaceID) +} + +func (p *SAN) createLocalQoS(ctx context.Context, params, taskResult map[string]interface{}) ( + map[string]interface{}, error) { + qos, exist := params["qos"].(map[string]int) + if !exist { + return nil, nil + } + + // 1. Get namespace by id, check whether the namespace exists. + namespaceId, ok := taskResult["localNamespaceID"].(string) + if !ok { + return nil, fmt.Errorf("namespaceId: [%v] convert to string failed", taskResult["localNamespaceID"]) + } + namespace, err := p.cli.GetNamespaceByID(ctx, namespaceId) + if err != nil { + return nil, err + } + + // 2. Do create qos. + qosID, exist := namespace["IOCLASSID"].(string) + if !exist || qosID == "" { + smartX := smartx.NewSmartX(p.cli) + qosID, err = smartX.CreateQos(ctx, namespaceId, qos) + if err != nil { + return nil, err + } + } + + return map[string]interface{}{ + "localQosID": qosID, + }, nil +} + +func (p *SAN) revertLocalQoS(ctx context.Context, taskResult map[string]interface{}) error { + namespaceID, namespaceIdOk := utils.GetValue[string](taskResult, "localNamespaceID") + qosID, qosIdOk := utils.GetValue[string](taskResult, "localQosID") + if !namespaceIdOk || !qosIdOk { + return nil + } + smartX := smartx.NewSmartX(p.cli) + return smartX.DeleteQos(ctx, qosID, namespaceID) +} + +// Query queries volume by name +func (p *SAN) Query(ctx context.Context, name string) (utils.Volume, error) { + namespace, err := p.cli.GetNamespaceByName(ctx, name) + if err != nil { + return nil, utils.Errorf(ctx, "get lun by name %s error: %v", name, err) + } + + if len(namespace) == 0 { + return nil, utils.Errorf(ctx, "lun [%s] to query does not exist", name) + } + + volObj := utils.NewVolume(name) + if lunWWN, ok := namespace["WWN"].(string); ok { + volObj.SetLunWWN(lunWWN) + } + + // set the size, need to trans Sectors to Bytes + capacityStr, ok := namespace["CAPACITY"].(string) + if !ok { + return nil, utils.Errorf(ctx, "convert capacity: %v to string failed", namespace["CAPACITY"]) + } + + capacity, err := strconv.ParseInt(capacityStr, constants.DefaultIntBase, constants.DefaultIntBitSize) + if err == nil { + volObj.SetSize(utils.TransK8SCapacity(capacity, constants.AllocationUnitBytes)) + } + + return volObj, nil +} + +// Delete deletes volume by volume name +func (p *SAN) Delete(ctx context.Context, name string) error { + namespace, err := p.cli.GetNamespaceByName(ctx, name) + if err != nil { + return err + } + if len(namespace) == 0 { + log.AddContext(ctx).Infof("Namespace: %s to be deleted does not exist.", name) + return nil + } + + taskFlow := flow.NewTaskFlow(ctx, "Delete-Namespace-Volume") + taskFlow.AddTask("Delete-Local-Namespace", p.deleteNamespace, nil) + + params := map[string]interface{}{ + "namespaceName": name, + } + + _, err = taskFlow.Run(params) + return err +} + +func (p *SAN) deleteNamespace(ctx context.Context, params, taskResult map[string]interface{}) ( + map[string]interface{}, error) { + namespaceName, ok := params["namespaceName"].(string) + if !ok { + return nil, fmt.Errorf("assert namespaceName: [%v] to string failed", params["namespaceName"]) + } + + namespace, err := p.cli.GetNamespaceByName(ctx, namespaceName) + if err != nil { + return nil, err + } + if len(namespace) == 0 { + log.AddContext(ctx).Infof("Namespace: [%s] to be deleted does not exist.", namespaceName) + return nil, nil + } + + namespaceID, ok := namespace["ID"].(string) + if !ok { + return nil, fmt.Errorf("assert namespaceID: [%v] to string failed", namespace["ID"]) + } + + // Oceandisk storage doesn't provide the IOCLASSID field, we need to traverse all ioclass to delete related QoS. + smartX := smartx.NewSmartX(p.cli) + err = smartX.DeleteQosByNamespaceId(ctx, namespaceID) + if err != nil { + return nil, fmt.Errorf("delete namespace %s related qos failed, error: %v", namespaceID, err) + } + + return nil, p.cli.DeleteNamespace(ctx, namespaceID) +} + +// Expand expands volume to target size +func (p *SAN) Expand(ctx context.Context, name string, newSize int64) (bool, error) { + namespace, err := p.cli.GetNamespaceByName(ctx, name) + if err != nil { + return false, err + } else if len(namespace) == 0 { + return false, fmt.Errorf("namespace %s to be expanded does not exist", name) + } + + exposedToInitiator, ok := namespace["EXPOSEDTOINITIATOR"].(string) + if !ok { + return false, fmt.Errorf("assert exposedToInitiator: [%v] to string failed", namespace["EXPOSEDTOINITIATOR"]) + } + isAttached, err := strconv.ParseBool(exposedToInitiator) + if err != nil { + return isAttached, fmt.Errorf("parse exposedToInitiator: [%v] to bool failed, error: %w", + exposedToInitiator, err) + } + + capacityStr, ok := namespace["CAPACITY"].(string) + if !ok { + return false, fmt.Errorf("assert capacity: [%v] to string failed", namespace["CAPACITY"]) + } + curSize := utils.ParseIntWithDefault(capacityStr, constants.DefaultIntBase, constants.DefaultIntBitSize, 0) + if newSize == curSize { + log.AddContext(ctx).Infof("Target capacity of namespace: [%v] is the same as the current capacity: [%d].", + name, newSize) + return isAttached, nil + } else if newSize < curSize { + return false, fmt.Errorf("target capacity: [%d] of namespace: [%v] must be greater than or equal to "+ + "current capacity: [%d]", newSize, name, curSize) + } + + expandTask := flow.NewTaskFlow(ctx, "Expand-Namespace-Volume") + expandTask.AddTask("Expand-Local-Namespace", p.expandLocalNamespace, nil) + + params := map[string]interface{}{ + "name": name, + "size": newSize, + "namespaceID": namespace["ID"], + "localParentName": namespace["PARENTNAME"], + } + _, err = expandTask.Run(params) + return isAttached, err +} + +func (p *SAN) expandLocalNamespace(ctx context.Context, params, taskResult map[string]interface{}) ( + map[string]interface{}, error) { + // 1. Check whether the storage pool exists. + localParentName, ok := params["localParentName"].(string) + if !ok { + return nil, fmt.Errorf("assert localParentName: [%v] to string failed", params["localParentName"]) + } + + pool, err := p.cli.GetPoolByName(ctx, localParentName) + if err != nil { + return nil, err + } else if len(pool) == 0 { + return nil, fmt.Errorf("storage pool: [%s] dose not exist", localParentName) + } + + // 2. do expand + namespaceID, ok := params["namespaceID"].(string) + if !ok { + return nil, fmt.Errorf("assert namespaceID: [%v] to string failed", params["namespaceID"]) + } + + newSize, ok := params["size"].(int64) + if !ok { + return nil, fmt.Errorf("assert size: [%v] to int64 failed", params["size"]) + } + + return nil, p.cli.ExtendNamespace(ctx, namespaceID, newSize) +} diff --git a/storage/oceanstorage/oceandisk/volume/san_test.go b/storage/oceanstorage/oceandisk/volume/san_test.go new file mode 100644 index 00000000..7fa4175a --- /dev/null +++ b/storage/oceanstorage/oceandisk/volume/san_test.go @@ -0,0 +1,205 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package volume is used for OceanDisk san test +package volume + +import ( + "context" + "fmt" + "reflect" + "testing" + + "github.com/agiledragon/gomonkey/v2" + + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceandisk/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" +) + +const ( + logName = "OceanDisk_san_test.log" +) + +func TestMain(m *testing.M) { + log.MockInitLogging(logName) + defer log.MockStopLogging(logName) + + m.Run() +} + +func TestSAN_Create_Success(t *testing.T) { + // arrange + ctx := context.Background() + clientConfig := client.NewClientConfig{ + Urls: []string{"127.0.0.1"}, + User: "testUser", + SecretName: "testSecretName", + SecretNamespace: "testSecretNamespace", + ParallelNum: "30", + BackendID: "BackendID", + UseCert: false, + CertSecretMeta: "test/test", + Storage: "testStorage", + Name: "testName", + } + cli, _ := client.NewClient(ctx, &clientConfig) + san := NewSAN(cli) + param := map[string]interface{}{ + "name": "testVolume", + "storagepool": "testPool", + } + + // mock + m := gomonkey.NewPatches() + defer m.Reset() + m.ApplyPrivateMethod(&SAN{}, "preCreate", + func(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { + return nil, nil + }) + m.ApplyPrivateMethod(&SAN{}, "createLocalNamespace", + func(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { + return nil, nil + }) + m.ApplyPrivateMethod(&SAN{}, "createLocalQoS", + func(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { + return nil, nil + }) + + // action + _, gotErr := san.Create(ctx, param) + + // assert + if gotErr != nil { + t.Errorf("TestCreate_Success failed, gotErr = %v, wantErr = %v.", gotErr, nil) + } +} + +func TestSAN_Create_PrepareFailed(t *testing.T) { + // arrange + ctx := context.Background() + clientConfig := client.NewClientConfig{ + Urls: []string{"127.0.0.1"}, + User: "testUser", + SecretName: "testSecretName", + SecretNamespace: "testSecretNamespace", + ParallelNum: "30", + BackendID: "BackendID", + UseCert: false, + CertSecretMeta: "test/test", + Storage: "testStorage", + Name: "testName", + } + cli, _ := client.NewClient(ctx, &clientConfig) + san := NewSAN(cli) + param := map[string]interface{}{ + "name": "testVolume", + } + wantErr := fmt.Errorf("test error") + + // mock + m := gomonkey.NewPatches() + defer m.Reset() + m.ApplyPrivateMethod(&SAN{}, "preCreate", + func(ctx context.Context, params map[string]interface{}) error { + return wantErr + }) + m.ApplyPrivateMethod(&SAN{}, "createLocalNamespace", + func(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { + return nil, nil + }) + + // action + _, gotErr := san.Create(ctx, param) + + // assert + if gotErr == nil { + t.Errorf("TestSAN_Create_PrepareFailed failed, gotErr = %v, wantErr = %v.", gotErr, wantErr) + } +} + +func TestSAN_deleteNamespace_success(t *testing.T) { + // arrange + ctx := context.Background() + clientConfig := client.NewClientConfig{ + Urls: []string{"127.0.0.1"}, + User: "testUser", + SecretName: "testSecretName", + SecretNamespace: "testSecretNamespace", + ParallelNum: "30", + BackendID: "BackendID", + UseCert: false, + CertSecretMeta: "test/test", + Storage: "testStorage", + Name: "testName", + } + cli, _ := client.NewClient(ctx, &clientConfig) + san := NewSAN(cli) + param := map[string]interface{}{ + "namespaceName": "testName", + } + taskResult := map[string]interface{}{} + + // mock + m := gomonkey.NewPatches() + defer m.Reset() + m.ApplyMethodReturn(&client.OceandiskClient{}, "GetNamespaceByName", nil, nil) + + // action + _, gotErr := san.deleteNamespace(ctx, param, taskResult) + + // assert + if gotErr != nil { + t.Errorf("TestSAN_deleteNamespace_success failed, gotErr = %v, wantErr = %v.", gotErr, nil) + } +} + +func TestSAN_expandLocalNamespace_poolNotExist(t *testing.T) { + // arrange + ctx := context.Background() + clientConfig := client.NewClientConfig{ + Urls: []string{"127.0.0.1"}, + User: "testUser", + SecretName: "testSecretName", + SecretNamespace: "testSecretNamespace", + ParallelNum: "30", + BackendID: "BackendID", + UseCert: false, + CertSecretMeta: "test/test", + Storage: "testStorage", + Name: "testName", + } + cli, _ := client.NewClient(ctx, &clientConfig) + san := NewSAN(cli) + localParentName := "testPool01" + param := map[string]interface{}{ + "localParentName": localParentName, + } + taskResult := map[string]interface{}{} + wantErr := fmt.Errorf("storage pool: [%s] dose not exist", localParentName) + + // mock + m := gomonkey.NewPatches() + defer m.Reset() + m.ApplyMethodReturn(&client.OceandiskClient{}, "GetPoolByName", nil, nil) + + // action + _, gotErr := san.expandLocalNamespace(ctx, param, taskResult) + + // assert + if !reflect.DeepEqual(gotErr, wantErr) { + t.Errorf("TestSAN_expandLocalNamespace_poolNotExist failed, gotErr = %v, wantErr = %v.", gotErr, wantErr) + } +} diff --git a/storage/oceanstorage/oceanstor/attacher/attacher.go b/storage/oceanstorage/oceanstor/attacher/attacher.go new file mode 100644 index 00000000..2abc2703 --- /dev/null +++ b/storage/oceanstorage/oceanstor/attacher/attacher.go @@ -0,0 +1,317 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2020-2024. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package attacher provide operations of volume attach +package attacher + +import ( + "context" + "errors" + "fmt" + + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/base" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/base/attacher" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceanstor/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" +) + +// VolumeAttacherPlugin defines interfaces of attach operations +type VolumeAttacherPlugin interface { + ControllerAttach(context.Context, string, map[string]interface{}) (map[string]interface{}, error) + ControllerDetach(context.Context, string, map[string]interface{}) (string, error) + GetTargetRoCEPortals(context.Context) ([]string, error) + getLunInfo(context.Context, string) (map[string]interface{}, error) +} + +// VolumeAttacher defines attacher to attach volume +type VolumeAttacher struct { + *attacher.AttachmentManager + Cli client.OceanstorClientInterface +} + +// VolumeAttacherConfig defines the configurations of VolumeAttacher +type VolumeAttacherConfig struct { + Product constants.OceanstorVersion + Cli client.OceanstorClientInterface + Protocol string + Invoker string + Portals []string + Alua map[string]interface{} +} + +// NewAttacher init a new attacher +func NewAttacher(config VolumeAttacherConfig) VolumeAttacherPlugin { + if config.Product.IsDoradoV6OrV7() { + return newDoradoV6OrV7Attacher(config) + } + return newOceanStorAttacher(config) +} + +// getLunGroupName generates lun group name +func (p *VolumeAttacher) getLunGroupName(postfix string) string { + return fmt.Sprintf("k8s_%s_lungroup_%s", p.Invoker, postfix) +} + +func (p *VolumeAttacher) createLunGroup(ctx context.Context, lunID, hostID, mappingID string) error { + var err error + var lunGroup map[string]interface{} + + lunGroupsByLunID, err := p.Cli.QueryAssociateLunGroup(ctx, base.AssociateObjTypeLUN, lunID) + if err != nil { + log.AddContext(ctx).Errorf("Query associated lun groups of lun %s error: %v", lunID, err) + return err + } + + lunGroupName := p.getLunGroupName(hostID) + for _, i := range lunGroupsByLunID { + group, ok := i.(map[string]interface{}) + if !ok { + log.AddContext(ctx).Warningf("convert group to map failed, data: %v", i) + continue + } + if group["NAME"].(string) == lunGroupName { + lunGroupID, ok := group["ID"].(string) + if !ok { + return errors.New("convert group[\"ID\"] to string failed") + } + return p.addToLUNGroupMapping(ctx, lunGroupName, lunGroupID, mappingID) + } + } + + lunGroup, err = p.Cli.GetLunGroupByName(ctx, lunGroupName) + if err != nil { + log.AddContext(ctx).Errorf("Get lungroup by name %s error: %v", lunGroupName, err) + return err + } + if lunGroup == nil { + lunGroup, err = p.Cli.CreateLunGroup(ctx, lunGroupName) + if err != nil { + log.AddContext(ctx).Errorf("Create lungroup %s error: %v", lunGroupName, err) + return err + } + } + + lunGroupID, ok := lunGroup["ID"].(string) + if !ok { + return errors.New("createLunGroup failed, caused by not found lun group id") + } + err = p.Cli.AddLunToGroup(ctx, lunID, lunGroupID) + if err != nil { + log.AddContext(ctx).Errorf("Add lun %s to group %s error: %v", lunID, lunGroupID, err) + return err + } + + return p.addToLUNGroupMapping(ctx, lunGroupName, lunGroupID, mappingID) +} + +func (p *VolumeAttacher) addToLUNGroupMapping(ctx context.Context, groupName, groupID, mappingID string) error { + lunGroupsByMappingID, err := p.Cli.QueryAssociateLunGroup(ctx, base.AssociateObjTypeMapping, mappingID) + if err != nil { + log.AddContext(ctx).Errorf("Query associated lun groups of mapping %s error: %v", mappingID, err) + return err + } + + for _, i := range lunGroupsByMappingID { + group, ok := i.(map[string]interface{}) + if !ok { + return fmt.Errorf("invalid group type. Expected 'map[string]interface{}', found %T", i) + } + if group["NAME"].(string) == groupName { + return nil + } + } + + err = p.Cli.AddGroupToMapping(ctx, base.AssociateObjTypeLUNGroup, groupID, mappingID) + if err != nil { + log.AddContext(ctx).Errorf("Add lun group %s to mapping %s error: %v", + groupID, mappingID, err) + return err + } + + return nil +} + +func (p *VolumeAttacher) needUpdateInitiatorAlua(initiator map[string]interface{}) bool { + if p.Alua == nil { + return false + } + + multiPathType, ok := p.Alua["MULTIPATHTYPE"] + if !ok { + return false + } + + if multiPathType != initiator["MULTIPATHTYPE"] { + return true + } else if initiator["MULTIPATHTYPE"] == MultiPathTypeDefault { + return false + } + + failoverMode, ok := p.Alua["FAILOVERMODE"] + if ok && failoverMode != initiator["FAILOVERMODE"] { + return true + } + + specialModeType, ok := p.Alua["SPECIALMODETYPE"] + if ok && specialModeType != initiator["SPECIALMODETYPE"] { + return true + } + + pathType, ok := p.Alua["PATHTYPE"] + if ok && pathType != initiator["PATHTYPE"] { + return true + } + + return false +} + +func (p *VolumeAttacher) doMapping(ctx context.Context, hostID, lunName string) (string, string, error) { + lun, err := p.Cli.GetLunByName(ctx, lunName) + if err != nil { + log.AddContext(ctx).Errorf("Get lun %s error: %v", lunName, err) + return "", "", err + } + if lun == nil { + msg := fmt.Sprintf("Lun %s not exist for attaching", lunName) + log.AddContext(ctx).Errorln(msg) + return "", "", errors.New(msg) + } + + lunID, ok := lun["ID"].(string) + if !ok { + return "", "", pkgUtils.Errorf(ctx, "convert lunID to string failed, data: %v", lun["ID"]) + } + mappingID, err := p.CreateMapping(ctx, hostID) + if err != nil { + log.AddContext(ctx).Errorf("Create mapping for host %s error: %v", hostID, err) + return "", "", err + } + + err = p.CreateHostGroup(ctx, hostID, mappingID) + if err != nil { + log.AddContext(ctx).Errorf("Create host group for host %s error: %v", hostID, err) + return "", "", err + } + + err = p.createLunGroup(ctx, lunID, hostID, mappingID) + if err != nil { + log.AddContext(ctx).Errorf("Create lun group for host %s error: %v", hostID, err) + return "", "", err + } + + lunUniqueId, err := utils.GetLunUniqueId(ctx, p.Protocol, lun) + if err != nil { + return "", "", err + } + + hostLunId, err := p.Cli.GetHostLunId(ctx, hostID, lunID) + if err != nil { + return "", "", err + } + + return lunUniqueId, hostLunId, nil +} + +func (p *VolumeAttacher) doUnmapping(ctx context.Context, hostID, lunName string) (string, error) { + lun, err := p.Cli.GetLunByName(ctx, lunName) + if err != nil { + log.AddContext(ctx).Errorf("Get lun %s info error: %v", lunName, err) + return "", err + } + if lun == nil { + log.AddContext(ctx).Infof("LUN %s doesn't exist while detaching", lunName) + return "", nil + } + lunID, ok := lun["ID"].(string) + if !ok { + return "", pkgUtils.Errorf(ctx, "convert lunID to string failed, data: %v", lun["ID"]) + } + lunGroupsByLunID, err := p.Cli.QueryAssociateLunGroup(ctx, base.AssociateObjTypeLUN, lunID) + if err != nil { + log.AddContext(ctx).Errorf("Query associated lungroups of lun %s error: %v", lunID, err) + return "", err + } + + lunGroupName := p.getLunGroupName(hostID) + for _, i := range lunGroupsByLunID { + group, ok := i.(map[string]interface{}) + if !ok { + log.AddContext(ctx).Warningf("convert group to map failed, data: %v", i) + continue + } + if group["NAME"].(string) == lunGroupName { + lunGroupID, ok := group["ID"].(string) + if !ok { + return "", pkgUtils.Errorf(ctx, "convert lunGroupID to string failed, data: %v", group["ID"]) + } + err = p.Cli.RemoveLunFromGroup(ctx, lunID, lunGroupID) + if err != nil { + log.AddContext(ctx).Errorf("Remove lun %s from group %s error: %v", + lunID, lunGroupID, err) + return "", err + } + } + } + + lunUniqueId, err := utils.GetLunUniqueId(ctx, p.Protocol, lun) + if err != nil { + return "", err + } + return lunUniqueId, nil +} + +// ControllerDetach detaches volume and unmaps lun from host +func (p *VolumeAttacher) ControllerDetach(ctx context.Context, + lunName string, + parameters map[string]interface{}) (string, error) { + host, err := p.GetHost(ctx, parameters, false) + if err != nil { + log.AddContext(ctx).Infof("Get host ID error: %v", err) + return "", err + } + if host == nil { + log.AddContext(ctx).Infof("Host doesn't exist while detaching %s", lunName) + return "", nil + } + + hostID, ok := host["ID"].(string) + if !ok { + return "", pkgUtils.Errorf(ctx, "convert hostID to string failed, data: %v", host["ID"]) + } + wwn, err := p.doUnmapping(ctx, hostID, lunName) + if err != nil { + log.AddContext(ctx).Errorf("Unmapping LUN %s from host %s error: %v", lunName, hostID, err) + return "", err + } + + return wwn, nil +} + +func (p *VolumeAttacher) getLunInfo(ctx context.Context, lunName string) (map[string]interface{}, error) { + lun, err := p.Cli.GetLunByName(ctx, lunName) + if err != nil { + log.AddContext(ctx).Errorf("Get lun %s info error: %v", lunName, err) + return nil, err + } + if lun == nil { + log.AddContext(ctx).Infof("LUN %s doesn't exist while detaching", lunName) + return nil, nil + } + return lun, nil +} diff --git a/storage/oceanstor/attacher/dorado_v6_attacher.go b/storage/oceanstorage/oceanstor/attacher/dorado_v6_attacher.go similarity index 65% rename from storage/oceanstor/attacher/dorado_v6_attacher.go rename to storage/oceanstorage/oceanstor/attacher/dorado_v6_attacher.go index cd0bd835..de82a730 100644 --- a/storage/oceanstor/attacher/dorado_v6_attacher.go +++ b/storage/oceanstorage/oceanstor/attacher/dorado_v6_attacher.go @@ -1,5 +1,5 @@ /* - * Copyright (c) Huawei Technologies Co., Ltd. 2020-2023. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2020-2024. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,8 +20,9 @@ import ( "context" "errors" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/base/attacher" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // DoradoV6Attacher implements interface VolumeAttacherPlugin @@ -34,14 +35,20 @@ const ( AccessModeBalanced = "0" ) -func newDoradoV6Attacher(config VolumeAttacherConfig) VolumeAttacherPlugin { +func newDoradoV6OrV7Attacher(config VolumeAttacherConfig) VolumeAttacherPlugin { + baseAttacherConfig := attacher.AttachmentManagerConfig{ + Cli: config.Cli, + Protocol: config.Protocol, + Invoker: config.Invoker, + Portals: config.Portals, + Alua: config.Alua, + } + baseAttacher := attacher.NewAttachmentManager(baseAttacherConfig) + return &DoradoV6Attacher{ VolumeAttacher: VolumeAttacher{ - cli: config.Cli, - protocol: config.Protocol, - invoker: config.Invoker, - portals: config.Portals, - alua: config.Alua, + AttachmentManager: baseAttacher, + Cli: config.Cli, }, } } @@ -70,7 +77,7 @@ func (p *DoradoV6Attacher) needUpdateHost(host map[string]interface{}, hostAlua func (p *DoradoV6Attacher) ControllerAttach(ctx context.Context, lunName string, parameters map[string]interface{}) (map[string]interface{}, error) { - host, err := p.getHost(ctx, parameters, true) + host, err := p.GetHost(ctx, parameters, true) if err != nil { log.AddContext(ctx).Errorf("Get host ID error: %v", err) return nil, err @@ -80,26 +87,26 @@ func (p *DoradoV6Attacher) ControllerAttach(ctx context.Context, if !ok { return nil, errors.New("convert host[\"ID\"] to string failed") } - hostAlua := utils.GetAlua(ctx, p.alua, host["NAME"].(string)) + hostAlua := utils.GetAlua(ctx, p.Alua, host["NAME"].(string)) if hostAlua != nil && p.needUpdateHost(host, hostAlua) { - err := p.cli.UpdateHost(ctx, hostID, hostAlua) + err := p.Cli.UpdateHost(ctx, hostID, hostAlua) if err != nil { log.AddContext(ctx).Errorf("Update host %s error: %v", hostID, err) return nil, err } } - if p.protocol == "iscsi" { - _, err = p.VolumeAttacher.attachISCSI(ctx, hostID, parameters) - } else if p.protocol == "fc" || p.protocol == "fc-nvme" { - _, err = p.VolumeAttacher.attachFC(ctx, hostID, parameters) - } else if p.protocol == "roce" { - _, err = p.VolumeAttacher.attachRoCE(ctx, hostID, parameters) + if p.Protocol == "iscsi" { + _, err = p.VolumeAttacher.AttachISCSI(ctx, hostID, parameters) + } else if p.Protocol == "fc" || p.Protocol == "fc-nvme" { + _, err = p.VolumeAttacher.AttachFC(ctx, hostID, parameters) + } else if p.Protocol == "roce" { + _, err = p.VolumeAttacher.AttachRoCE(ctx, hostID, parameters) } if err != nil { - log.AddContext(ctx).Errorf("Attach %s connection error: %v", p.protocol, err) + log.AddContext(ctx).Errorf("Attach %s connection error: %v", p.Protocol, err) return nil, err } @@ -109,5 +116,5 @@ func (p *DoradoV6Attacher) ControllerAttach(ctx context.Context, return nil, err } - return p.getMappingProperties(ctx, wwn, hostLunId, parameters) + return p.GetMappingProperties(ctx, wwn, hostLunId, parameters) } diff --git a/storage/oceanstor/attacher/metroAttacher.go b/storage/oceanstorage/oceanstor/attacher/metroAttacher.go similarity index 94% rename from storage/oceanstor/attacher/metroAttacher.go rename to storage/oceanstorage/oceanstor/attacher/metroAttacher.go index a5c61e8c..af3f372a 100644 --- a/storage/oceanstor/attacher/metroAttacher.go +++ b/storage/oceanstorage/oceanstor/attacher/metroAttacher.go @@ -20,7 +20,7 @@ import ( "context" "errors" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // MetroAttacher implements interface VolumeAttacherPlugin @@ -119,15 +119,16 @@ func (p *MetroAttacher) mergeLunWWN(ctx context.Context, locLunWWN, rmtLunWWN st return locLunWWN, nil } -func (p *MetroAttacher) getTargetRoCEPortals(ctx context.Context) ([]string, error) { +// GetTargetRoCEPortals gets target roce portals +func (p *MetroAttacher) GetTargetRoCEPortals(ctx context.Context) ([]string, error) { var availablePortals []string - localPortals, err := p.localAttacher.getTargetRoCEPortals(ctx) + localPortals, err := p.localAttacher.GetTargetRoCEPortals(ctx) if err != nil { log.AddContext(ctx).Warningf("Get local roce portals error: %v", err) } availablePortals = append(availablePortals, localPortals...) - remotePortals, err := p.remoteAttacher.getTargetRoCEPortals(ctx) + remotePortals, err := p.remoteAttacher.GetTargetRoCEPortals(ctx) if err != nil { log.AddContext(ctx).Warningf("Get remote roce portals error: %v", err) } diff --git a/storage/oceanstor/attacher/oceanstor_attacher.go b/storage/oceanstorage/oceanstor/attacher/oceanstor_attacher.go similarity index 73% rename from storage/oceanstor/attacher/oceanstor_attacher.go rename to storage/oceanstorage/oceanstor/attacher/oceanstor_attacher.go index 6d3b1a4c..ffcf2ab9 100644 --- a/storage/oceanstor/attacher/oceanstor_attacher.go +++ b/storage/oceanstorage/oceanstor/attacher/oceanstor_attacher.go @@ -1,5 +1,5 @@ /* - * Copyright (c) Huawei Technologies Co., Ltd. 2020-2023. All rights reserved. + * Copyright (c) Huawei Technologies Co., Ltd. 2020-2024. All rights reserved. * * Licensed under the Apache License, Version 2.0 (the "License"); * you may not use this file except in compliance with the License. @@ -20,8 +20,9 @@ import ( "context" "errors" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/base/attacher" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // OceanStorAttacher implements interface VolumeAttacherPlugin @@ -35,13 +36,19 @@ const ( ) func newOceanStorAttacher(config VolumeAttacherConfig) VolumeAttacherPlugin { + baseAttacherConfig := attacher.AttachmentManagerConfig{ + Cli: config.Cli, + Protocol: config.Protocol, + Invoker: config.Invoker, + Portals: config.Portals, + Alua: config.Alua, + } + baseAttacher := attacher.NewAttachmentManager(baseAttacherConfig) + return &OceanStorAttacher{ VolumeAttacher: VolumeAttacher{ - cli: config.Cli, - protocol: config.Protocol, - invoker: config.Invoker, - portals: config.Portals, - alua: config.Alua, + AttachmentManager: baseAttacher, + Cli: config.Cli, }, } } @@ -79,14 +86,14 @@ func (p *OceanStorAttacher) needUpdateInitiatorAlua(initiator map[string]interfa func (p *OceanStorAttacher) attachISCSI(ctx context.Context, hostID, hostName string, parameters map[string]interface{}) error { - iscsiInitiator, err := p.VolumeAttacher.attachISCSI(ctx, hostID, parameters) + iscsiInitiator, err := p.VolumeAttacher.AttachISCSI(ctx, hostID, parameters) if err != nil { return err } - hostAlua := utils.GetAlua(ctx, p.alua, hostName) + hostAlua := utils.GetAlua(ctx, p.Alua, hostName) if hostAlua != nil && p.needUpdateInitiatorAlua(iscsiInitiator, hostAlua) { - err = p.cli.UpdateIscsiInitiator(ctx, iscsiInitiator["ID"].(string), hostAlua) + err = p.Cli.UpdateIscsiInitiator(ctx, iscsiInitiator["ID"].(string), hostAlua) } return err @@ -94,19 +101,19 @@ func (p *OceanStorAttacher) attachISCSI(ctx context.Context, hostID, hostName st func (p *OceanStorAttacher) attachFC(ctx context.Context, hostID, hostName string, parameters map[string]interface{}) error { - fcInitiators, err := p.VolumeAttacher.attachFC(ctx, hostID, parameters) + fcInitiators, err := p.VolumeAttacher.AttachFC(ctx, hostID, parameters) if err != nil { return err } - hostAlua := utils.GetAlua(ctx, p.alua, hostName) + hostAlua := utils.GetAlua(ctx, p.Alua, hostName) if hostAlua != nil { for _, i := range fcInitiators { if !p.needUpdateInitiatorAlua(i, hostAlua) { continue } - err := p.cli.UpdateFCInitiator(ctx, i["ID"].(string), hostAlua) + err := p.Cli.UpdateFCInitiator(ctx, i["ID"].(string), hostAlua) if err != nil { return err } @@ -117,7 +124,7 @@ func (p *OceanStorAttacher) attachFC(ctx context.Context, hostID, hostName strin } func (p *OceanStorAttacher) attachRoCE(ctx context.Context, hostID string, parameters map[string]interface{}) error { - _, err := p.VolumeAttacher.attachRoCE(ctx, hostID, parameters) + _, err := p.VolumeAttacher.AttachRoCE(ctx, hostID, parameters) return err } @@ -126,7 +133,7 @@ func (p *OceanStorAttacher) ControllerAttach(ctx context.Context, lunName string, parameters map[string]interface{}) ( map[string]interface{}, error) { - host, err := p.getHost(ctx, parameters, true) + host, err := p.GetHost(ctx, parameters, true) if err != nil { log.AddContext(ctx).Errorf("Get host ID error: %v", err) return nil, err @@ -141,16 +148,16 @@ func (p *OceanStorAttacher) ControllerAttach(ctx context.Context, return nil, errors.New("convert host[\"NAME\"] to string failed") } - if p.protocol == "iscsi" { + if p.Protocol == "iscsi" { err = p.attachISCSI(ctx, hostID, hostName, parameters) - } else if p.protocol == "fc" || p.protocol == "fc-nvme" { + } else if p.Protocol == "fc" || p.Protocol == "fc-nvme" { err = p.attachFC(ctx, hostID, hostName, parameters) - } else if p.protocol == "roce" { + } else if p.Protocol == "roce" { err = p.attachRoCE(ctx, hostID, parameters) } if err != nil { - log.AddContext(ctx).Errorf("Attach %s connection error: %v", p.protocol, err) + log.AddContext(ctx).Errorf("Attach %s connection error: %v", p.Protocol, err) return nil, err } @@ -160,5 +167,5 @@ func (p *OceanStorAttacher) ControllerAttach(ctx context.Context, return nil, err } - return p.getMappingProperties(ctx, wwn, hostLunId, parameters) + return p.GetMappingProperties(ctx, wwn, hostLunId, parameters) } diff --git a/storage/oceanstorage/oceanstor/client/client.go b/storage/oceanstorage/oceanstor/client/client.go new file mode 100644 index 00000000..1eb6b10a --- /dev/null +++ b/storage/oceanstorage/oceanstor/client/client.go @@ -0,0 +1,452 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2020-2024. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package client provides oceanstor storage client +package client + +import ( + "context" + "encoding/json" + "errors" + "fmt" + "io" + "net/http" + "regexp" + "sync/atomic" + + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/base" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" +) + +const ( + // DefaultParallelCount defines default parallel count + DefaultParallelCount int = 30 + + // MaxParallelCount defines max parallel count + MaxParallelCount int = 30 + + // MinParallelCount defines min parallel count + MinParallelCount int = 1 + + // GetInfoWaitInternal defines wait internal of getting info + GetInfoWaitInternal = 10 + + // UrlNotFound defines error msg of url not found + UrlNotFound = "404_NotFound" +) + +const ( + description string = "Created from huawei-csi for Kubernetes" + defaultVStore string = "System_vStore" + defaultVStoreID string = "0" +) + +// OceanstorClientInterface defines interfaces for base client operations +type OceanstorClientInterface interface { + base.RestClientInterface + base.ApplicationType + base.FC + base.Host + base.Iscsi + base.Mapping + base.Qos + base.RoCE + base.System + + Clone + Filesystem + FSSnapshot + HyperMetro + Lun + LunCopy + LunSnapshot + Replication + VStore + DTree + OceanStorQuota + LIF + + SafeCall(ctx context.Context, method string, url string, data map[string]interface{}) (base.Response, error) + SafeBaseCall(ctx context.Context, method string, url string, data map[string]interface{}) (base.Response, error) + SafeDelete(ctx context.Context, url string, data map[string]interface{}) (base.Response, error) + DuplicateClient() *OceanstorClient + + GetBackendID() string + GetDeviceSN() string + GetStorageVersion() string + GetCurrentSiteWwn() string + SetSystemInfo(ctx context.Context) error +} + +var ( + filterLog = map[string]map[string]bool{ + "POST": { + "/xx/sessions": true, + }, + } + + filterLogRegex = map[string][]string{ + "GET": { + `/vstore_pair\?filter=ID`, + `/FsHyperMetroDomain\?RUNNINGSTATUS=0`, + `/remote_device`, + }, + } + + debugLog = map[string]map[string]bool{ + "GET": { + "/license/feature": true, + "/nfsservice": true, + "/storagepool": true, + `/vstore_pair\?RETYPE=1`: true, + `/vstore?filter=NAME`: true, + `/container_pv`: true, + `/system`: true, + }, + } + + debugLogRegex = map[string][]string{ + "GET": { + `/vstore_pair\?RETYPE=1`, + `/vstore\?filter=NAME`, + `/system`, + }, + } +) + +func isFilterLog(method, url string) bool { + if filter, exist := filterLog[method]; exist && filter[url] { + return true + } + + if filter, exist := filterLogRegex[method]; exist { + for _, k := range filter { + match, err := regexp.MatchString(k, url) + if err == nil && match { + return true + } + } + } + + return false +} + +// OceanstorClient implements OceanstorClientInterface +type OceanstorClient struct { + *base.ApplicationTypeClient + *base.FCClient + *base.HostClient + *base.IscsiClient + *base.MappingClient + *base.QosClient + *base.RoCEClient + *base.SystemClient + + *RestClient +} + +// NewClientConfig stores the information needed to create a new oceanstor client +type NewClientConfig struct { + Urls []string + User string + SecretName string + SecretNamespace string + VstoreName string + ParallelNum string + BackendID string + UseCert bool + CertSecretMeta string + Storage string + Name string +} + +// NewClient inits a new oceanstor client +func NewClient(ctx context.Context, param *NewClientConfig) (*OceanstorClient, error) { + restClient, err := NewRestClient(ctx, param) + if err != nil { + return nil, err + } + + return &OceanstorClient{ + ApplicationTypeClient: &base.ApplicationTypeClient{RestClientInterface: restClient}, + FCClient: &base.FCClient{RestClientInterface: restClient}, + HostClient: &base.HostClient{RestClientInterface: restClient}, + IscsiClient: &base.IscsiClient{RestClientInterface: restClient}, + MappingClient: &base.MappingClient{RestClientInterface: restClient}, + QosClient: &base.QosClient{RestClientInterface: restClient}, + RoCEClient: &base.RoCEClient{RestClientInterface: restClient}, + SystemClient: &base.SystemClient{RestClientInterface: restClient}, + RestClient: restClient, + }, nil +} + +// SafeCall provides call for restful request +func (cli *OceanstorClient) SafeCall(ctx context.Context, + method string, url string, + data map[string]interface{}) (base.Response, error) { + var r base.Response + var err error + + r, err = cli.SafeBaseCall(ctx, method, url, data) + if !base.NeedReLogin(r, err) { + return r, err + } + + // Current connection fails, try to relogin to other Urls if exist, + // if relogin success, resend the request again. + log.AddContext(ctx).Infof("Try to re-login and resend request method: %s, Url: %s", method, url) + err = cli.ReLogin(ctx) + if err != nil { + return r, err + } + + // If the logical port changes from storage A to storage B, the system information must be updated. + if err = cli.SetSystemInfo(ctx); err != nil { + log.AddContext(ctx).Errorf("after re-login, can't get system info, error: %v", err) + return r, err + } + + return cli.SafeBaseCall(ctx, method, url, data) +} + +// SafeBaseCall provides base call for request +func (cli *OceanstorClient) SafeBaseCall(ctx context.Context, + method string, + url string, + data map[string]interface{}) (base.Response, error) { + var req *http.Request + var err error + + if cli.Client == nil { + return base.Response{}, fmt.Errorf("failed to send request method: %s, url: %s,"+ + " cause by client not init", method, url) + } + + reqUrl := cli.Url + reqUrl += url + + if url != "/xx/sessions" && url != "/sessions" { + cli.ReLoginMutex.Lock() + req, err = cli.GetRequest(ctx, method, url, data) + cli.ReLoginMutex.Unlock() + } else { + req, err = cli.GetRequest(ctx, method, url, data) + } + + if err != nil || req == nil { + return base.Response{}, fmt.Errorf("get request failed, error: %w", err) + } + + log.FilteredLog(ctx, isFilterLog(method, url), utils.IsDebugLog(method, url, debugLog, debugLogRegex), + fmt.Sprintf("Request method: %s, Url: %s, body: %v", method, req.URL, data)) + + if cli.RequestSemaphore != nil { + cli.RequestSemaphore.Acquire() + defer cli.RequestSemaphore.Release() + } + + if base.RequestSemaphoreMap[cli.GetDeviceSN()] != nil { + base.RequestSemaphoreMap[cli.GetDeviceSN()].Acquire() + defer base.RequestSemaphoreMap[cli.GetDeviceSN()].Release() + } + + return cli.safeDoCall(ctx, method, url, req) +} + +func (cli *OceanstorClient) safeDoCall(ctx context.Context, + method string, url string, req *http.Request) (base.Response, error) { + // check whether the logical port is changed from A to B before invoking. + // The possible cause is that other invoking operations are performed for re-login. + isNotSessionUrl := url != "/xx/sessions" && url != "/sessions" + if isNotSessionUrl && cli.CurrentLifWwn != "" { + if cli.systemInfoRefreshing() { + return base.Response{}, fmt.Errorf("querying lif and system information... Please wait") + } + + if cli.CurrentLifWwn != cli.CurrentSiteWwn { + currentPort := cli.GetCurrentLif(ctx) + log.AddContext(ctx).Errorf("current logical port [%s] is not running on own site, "+ + "currentLifWwn: %s, currentSiteWwn: %s", currentPort, cli.CurrentLifWwn, cli.CurrentSiteWwn) + return base.Response{}, fmt.Errorf("current logical port [%s] is not running on own site", currentPort) + } + } + + resp, err := cli.Client.Do(req) + if err != nil { + log.AddContext(ctx).Errorf("Send request method: %s, Url: %s, error: %v", method, req.URL, err) + return base.Response{}, errors.New(base.Unconnected) + } + + defer func() { + if err := resp.Body.Close(); err != nil { + log.AddContext(ctx).Infof("read close resp failed, error %v", err) + } + }() + + body, err := io.ReadAll(resp.Body) + if err != nil { + return base.Response{}, fmt.Errorf("read response data error: %w", err) + } + + log.FilteredLog(ctx, isFilterLog(method, url), utils.IsDebugLog(method, url, debugLog, debugLogRegex), + fmt.Sprintf("base.Response method: %s, Url: %s, body: %s", method, req.URL, body)) + + var r base.Response + err = json.Unmarshal(body, &r) + if err != nil { + return base.Response{}, fmt.Errorf("json.Unmarshal data %s error: %w", body, err) + } + + return r, nil +} + +// SafeDelete provides http request of DELETE method +func (cli *OceanstorClient) SafeDelete(ctx context.Context, + url string, data map[string]interface{}) (base.Response, error) { + return cli.SafeCall(ctx, "DELETE", url, data) +} + +// DuplicateClient clone a base client from origin client +func (cli *OceanstorClient) DuplicateClient() *OceanstorClient { + dup := *cli + + dup.Urls = make([]string, len(cli.Urls)) + copy(dup.Urls, cli.Urls) + + dup.Client = nil + + return &dup +} + +// ValidateLogin validates the login info +func (cli *OceanstorClient) ValidateLogin(ctx context.Context) error { + var resp base.Response + var err error + + password, err := utils.GetPasswordFromSecret(ctx, cli.SecretName, cli.SecretNamespace) + if err != nil { + return err + } + + data := map[string]interface{}{ + "username": cli.User, + "password": password, + "scope": "0", + } + + if len(cli.VStoreName) > 0 && cli.VStoreName != defaultVStore { + data["vstorename"] = cli.VStoreName + } + + cli.DeviceId = "" + cli.Token = "" + for i, url := range cli.Urls { + cli.Url = url + "/deviceManager/rest" + + log.AddContext(ctx).Infof("Try to login %s", cli.Url) + resp, err = cli.BaseCall(ctx, "POST", "/xx/sessions", data) + if err == nil { + /* Sort the login Url to the last slot of san addresses, so that + if this connection error, next time will try other Url first. */ + cli.Urls[i], cli.Urls[len(cli.Urls)-1] = cli.Urls[len(cli.Urls)-1], cli.Urls[i] + break + } else if err.Error() != base.Unconnected { + log.AddContext(ctx).Errorf("Login %s error", cli.Url) + break + } + + log.AddContext(ctx).Warningf("Login %s error due to connection failure, gonna try another Url", + cli.Url) + } + + if err != nil { + return err + } + + code := int64(resp.Error["code"].(float64)) + if code != 0 { + return fmt.Errorf("validate login %s error: %+v", cli.Url, resp) + } + + cli.setDeviceIdFromRespData(ctx, resp) + + log.AddContext(ctx).Infof("Validate login %s success", cli.Url) + return nil +} + +func (cli *OceanstorClient) setDeviceIdFromRespData(ctx context.Context, resp base.Response) { + respData, ok := resp.Data.(map[string]interface{}) + if !ok { + log.AddContext(ctx).Warningf("convert response data to map[string]interface{} failed, data type: [%T]", + resp.Data) + } + + cli.DeviceId, ok = respData["deviceid"].(string) + if !ok { + log.AddContext(ctx).Warningf("not found deviceId, response data is: [%v]", respData["deviceid"]) + } + + if _, exists := respData["iBaseToken"]; !exists { + log.AddContext(ctx).Warningf("not found iBaseToken, response data is: [%v]", resp.Data) + } + cli.Token, ok = respData["iBaseToken"].(string) + if !ok { + log.AddContext(ctx).Warningf("convert iBaseToken to string error, data type: [%T]", + respData["iBaseToken"]) + } +} + +func (cli *OceanstorClient) getResponseDataMap(ctx context.Context, data interface{}) (map[string]interface{}, error) { + respData, ok := data.(map[string]interface{}) + if !ok { + return nil, utils.Errorf(ctx, "the response data is not a map[string]interface{}") + } + return respData, nil +} + +func (cli *OceanstorClient) getResponseDataList(ctx context.Context, data interface{}) ([]interface{}, error) { + respData, ok := data.([]interface{}) + if !ok { + return nil, utils.Errorf(ctx, "the response data is not a []interface{}") + } + return respData, nil +} + +func (cli *OceanstorClient) getObjByvStoreName(objList []interface{}) map[string]interface{} { + for _, data := range objList { + obj, ok := data.(map[string]interface{}) + if !ok || obj == nil { + continue + } + + vStoreName, ok := obj["vstoreName"].(string) + if !ok { + vStoreName = defaultVStore + } + + if vStoreName == cli.GetvStoreName() { + return obj + } + continue + + } + return nil +} + +func (cli *OceanstorClient) systemInfoRefreshing() bool { + return atomic.LoadUint32(&cli.SystemInfoRefreshing) == 1 +} diff --git a/storage/oceanstor/client/client_clone.go b/storage/oceanstorage/oceanstor/client/client_clone.go similarity index 89% rename from storage/oceanstor/client/client_clone.go rename to storage/oceanstorage/oceanstor/client/client_clone.go index c4c74849..1e17dcd5 100644 --- a/storage/oceanstor/client/client_clone.go +++ b/storage/oceanstorage/oceanstor/client/client_clone.go @@ -21,7 +21,7 @@ import ( "errors" "fmt" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( @@ -48,7 +48,7 @@ type Clone interface { } // DeleteClonePair used for delete clone pair -func (cli *BaseClient) DeleteClonePair(ctx context.Context, clonePairID string) error { +func (cli *OceanstorClient) DeleteClonePair(ctx context.Context, clonePairID string) error { data := map[string]interface{}{ "ID": clonePairID, "isDeleteDstLun": false, @@ -72,7 +72,7 @@ func (cli *BaseClient) DeleteClonePair(ctx context.Context, clonePairID string) } // GetClonePairInfo used for get clone pair info -func (cli *BaseClient) GetClonePairInfo(ctx context.Context, clonePairID string) (map[string]interface{}, error) { +func (cli *OceanstorClient) GetClonePairInfo(ctx context.Context, clonePairID string) (map[string]interface{}, error) { url := fmt.Sprintf("/clonepair?filter=ID::%s", clonePairID) resp, err := cli.Get(ctx, url, nil) @@ -107,7 +107,7 @@ func (cli *BaseClient) GetClonePairInfo(ctx context.Context, clonePairID string) } // CreateClonePair used for create clone pair -func (cli *BaseClient) CreateClonePair(ctx context.Context, +func (cli *OceanstorClient) CreateClonePair(ctx context.Context, srcLunID, dstLunID string, cloneSpeed int) (map[string]interface{}, error) { data := map[string]interface{}{ @@ -135,7 +135,7 @@ func (cli *BaseClient) CreateClonePair(ctx context.Context, } // SyncClonePair used for synchronize clone pair -func (cli *BaseClient) SyncClonePair(ctx context.Context, clonePairID string) error { +func (cli *OceanstorClient) SyncClonePair(ctx context.Context, clonePairID string) error { data := map[string]interface{}{ "ID": clonePairID, "copyAction": 0, @@ -155,7 +155,7 @@ func (cli *BaseClient) SyncClonePair(ctx context.Context, clonePairID string) er } // StopCloneFSSplit used for stop clone split -func (cli *BaseClient) StopCloneFSSplit(ctx context.Context, fsID string) error { +func (cli *OceanstorClient) StopCloneFSSplit(ctx context.Context, fsID string) error { data := map[string]interface{}{ "ID": fsID, "SPLITENABLE": false, @@ -175,7 +175,7 @@ func (cli *BaseClient) StopCloneFSSplit(ctx context.Context, fsID string) error } // SplitCloneFS used to split clone -func (cli *BaseClient) SplitCloneFS(ctx context.Context, +func (cli *OceanstorClient) SplitCloneFS(ctx context.Context, fsID, vStoreId string, splitSpeed int, isDeleteParentSnapshot bool) error { @@ -200,7 +200,7 @@ func (cli *BaseClient) SplitCloneFS(ctx context.Context, } // CloneFileSystem used for clone file system -func (cli *BaseClient) CloneFileSystem(ctx context.Context, name string, allocType int, parentID, +func (cli *OceanstorClient) CloneFileSystem(ctx context.Context, name string, allocType int, parentID, parentSnapshotID string) (map[string]interface{}, error) { data := map[string]interface{}{ diff --git a/storage/oceanstor/client/client_dtree.go b/storage/oceanstorage/oceanstor/client/client_dtree.go similarity index 87% rename from storage/oceanstor/client/client_dtree.go rename to storage/oceanstorage/oceanstor/client/client_dtree.go index b82a54c9..d91e1d2b 100644 --- a/storage/oceanstor/client/client_dtree.go +++ b/storage/oceanstorage/oceanstor/client/client_dtree.go @@ -21,7 +21,7 @@ import ( "errors" "fmt" - "huawei-csi-driver/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" ) const ( @@ -48,7 +48,8 @@ type DTree interface { } // CreateDTree use for create a dTree -func (cli *BaseClient) CreateDTree(ctx context.Context, params map[string]interface{}) (map[string]interface{}, error) { +func (cli *OceanstorClient) CreateDTree(ctx context.Context, + params map[string]interface{}) (map[string]interface{}, error) { resp, err := cli.Post(ctx, "/QUOTATREE", params) if err != nil { return nil, err @@ -62,7 +63,7 @@ func (cli *BaseClient) CreateDTree(ctx context.Context, params map[string]interf } // GetDTreeByName use for get dTree information -func (cli *BaseClient) GetDTreeByName(ctx context.Context, +func (cli *OceanstorClient) GetDTreeByName(ctx context.Context, parentID, parentName, vStoreID, name string) (map[string]interface{}, error) { url := fmt.Sprintf("/QUOTATREE?PARENTNAME=%s&NAME=%s&vstoreId=%s", parentName, name, vStoreID) @@ -82,7 +83,7 @@ func (cli *BaseClient) GetDTreeByName(ctx context.Context, } // DeleteDTreeByID use for delete a dTree -func (cli *BaseClient) DeleteDTreeByID(ctx context.Context, vStoreID, dTreeID string) error { +func (cli *OceanstorClient) DeleteDTreeByID(ctx context.Context, vStoreID, dTreeID string) error { url := fmt.Sprintf("/QUOTATREE") resp, err := cli.Delete(ctx, url, map[string]interface{}{ "ID": dTreeID, @@ -100,7 +101,7 @@ func (cli *BaseClient) DeleteDTreeByID(ctx context.Context, vStoreID, dTreeID st } // DeleteDTreeByName use for delete a dTree -func (cli *BaseClient) DeleteDTreeByName(ctx context.Context, parentName, dTreeName, vStoreID string) error { +func (cli *OceanstorClient) DeleteDTreeByName(ctx context.Context, parentName, dTreeName, vStoreID string) error { url := fmt.Sprintf("/QUOTATREE") resp, err := cli.Delete(ctx, url, map[string]interface{}{ "PARENTNAME": parentName, diff --git a/storage/oceanstor/client/client_filesystem.go b/storage/oceanstorage/oceanstor/client/client_filesystem.go similarity index 89% rename from storage/oceanstor/client/client_filesystem.go rename to storage/oceanstorage/oceanstor/client/client_filesystem.go index 4c2d0786..9c85d068 100644 --- a/storage/oceanstor/client/client_filesystem.go +++ b/storage/oceanstorage/oceanstor/client/client_filesystem.go @@ -23,9 +23,9 @@ import ( "strconv" "time" - "huawei-csi-driver/pkg/constants" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( @@ -87,7 +87,7 @@ type Filesystem interface { } // DeleteFileSystem used for delete file system -func (cli *BaseClient) DeleteFileSystem(ctx context.Context, params map[string]interface{}) error { +func (cli *OceanstorClient) DeleteFileSystem(ctx context.Context, params map[string]interface{}) error { resp, err := cli.Delete(ctx, "/filesystem", params) if err != nil { return err @@ -107,7 +107,7 @@ func (cli *BaseClient) DeleteFileSystem(ctx context.Context, params map[string]i } // SafeDeleteFileSystem used for delete file system -func (cli *BaseClient) SafeDeleteFileSystem(ctx context.Context, params map[string]interface{}) error { +func (cli *OceanstorClient) SafeDeleteFileSystem(ctx context.Context, params map[string]interface{}) error { resp, err := cli.SafeDelete(ctx, "/filesystem", params) if err != nil { return err @@ -127,7 +127,7 @@ func (cli *BaseClient) SafeDeleteFileSystem(ctx context.Context, params map[stri } // GetFileSystemByName used for get file system by name -func (cli *BaseClient) GetFileSystemByName(ctx context.Context, name string) (map[string]interface{}, error) { +func (cli *OceanstorClient) GetFileSystemByName(ctx context.Context, name string) (map[string]interface{}, error) { url := fmt.Sprintf("/filesystem?filter=NAME::%s&range=[0-100]", name) resp, err := cli.Get(ctx, url, nil) if err != nil { @@ -153,7 +153,7 @@ func (cli *BaseClient) GetFileSystemByName(ctx context.Context, name string) (ma } // GetFileSystemByID used for get file system by id -func (cli *BaseClient) GetFileSystemByID(ctx context.Context, id string) (map[string]interface{}, error) { +func (cli *OceanstorClient) GetFileSystemByID(ctx context.Context, id string) (map[string]interface{}, error) { url := fmt.Sprintf("/filesystem/%s", id) resp, err := cli.Get(ctx, url, nil) if err != nil { @@ -174,7 +174,8 @@ func (cli *BaseClient) GetFileSystemByID(ctx context.Context, id string) (map[st } // GetNfsShareByPath used for get nfs share by path -func (cli *BaseClient) GetNfsShareByPath(ctx context.Context, path, vStoreID string) (map[string]interface{}, error) { +func (cli *OceanstorClient) GetNfsShareByPath(ctx context.Context, + path, vStoreID string) (map[string]interface{}, error) { url := fmt.Sprintf("/NFSHARE?filter=SHAREPATH::%s&range=[0-100]", path) var data = make(map[string]interface{}) if vStoreID != "" { @@ -217,7 +218,7 @@ func (cli *BaseClient) GetNfsShareByPath(ctx context.Context, path, vStoreID str } // GetNfsShareAccess used for get nfs share access -func (cli *BaseClient) GetNfsShareAccess(ctx context.Context, +func (cli *OceanstorClient) GetNfsShareAccess(ctx context.Context, parentID, name, vStoreID string) (map[string]interface{}, error) { count, err := cli.GetNfsShareAccessCount(ctx, parentID, vStoreID) if err != nil { @@ -251,7 +252,7 @@ func (cli *BaseClient) GetNfsShareAccess(ctx context.Context, } // GetNfsShareAccessCount used for get nfs share access count by id -func (cli *BaseClient) GetNfsShareAccessCount(ctx context.Context, parentID, vStoreID string) (int64, error) { +func (cli *OceanstorClient) GetNfsShareAccessCount(ctx context.Context, parentID, vStoreID string) (int64, error) { url := fmt.Sprintf("/NFS_SHARE_AUTH_CLIENT/count?filter=PARENTID::%s", parentID) var data = make(map[string]interface{}) if vStoreID != "" { @@ -280,7 +281,7 @@ func (cli *BaseClient) GetNfsShareAccessCount(ctx context.Context, parentID, vSt } // GetNfsShareAccessRange used for get nfs share access -func (cli *BaseClient) GetNfsShareAccessRange(ctx context.Context, parentID, vStoreID string, startRange, +func (cli *OceanstorClient) GetNfsShareAccessRange(ctx context.Context, parentID, vStoreID string, startRange, endRange int64) ([]interface{}, error) { url := fmt.Sprintf("/NFS_SHARE_AUTH_CLIENT?filter=PARENTID::%s&range=[%d-%d]", parentID, startRange, endRange) @@ -310,7 +311,7 @@ func (cli *BaseClient) GetNfsShareAccessRange(ctx context.Context, parentID, vSt } // UpdateFileSystem used for update file system -func (cli *BaseClient) UpdateFileSystem(ctx context.Context, fsID string, params map[string]interface{}) error { +func (cli *OceanstorClient) UpdateFileSystem(ctx context.Context, fsID string, params map[string]interface{}) error { url := fmt.Sprintf("/filesystem/%s", fsID) resp, err := cli.Put(ctx, url, params) if err != nil { @@ -326,7 +327,7 @@ func (cli *BaseClient) UpdateFileSystem(ctx context.Context, fsID string, params } // ExtendFileSystem used for extend file system by new capacity -func (cli *BaseClient) ExtendFileSystem(ctx context.Context, fsID string, newCapacity int64) error { +func (cli *OceanstorClient) ExtendFileSystem(ctx context.Context, fsID string, newCapacity int64) error { url := fmt.Sprintf("/filesystem/%s", fsID) data := map[string]interface{}{ "CAPACITY": newCapacity, @@ -360,7 +361,7 @@ type AllowNfsShareAccessRequest struct { } // AllowNfsShareAccess used for allow nfs share access -func (cli *BaseClient) AllowNfsShareAccess(ctx context.Context, req *AllowNfsShareAccessRequest) error { +func (cli *OceanstorClient) AllowNfsShareAccess(ctx context.Context, req *AllowNfsShareAccessRequest) error { data := map[string]interface{}{ "NAME": req.Name, "PARENTID": req.ParentID, @@ -396,7 +397,7 @@ func (cli *BaseClient) AllowNfsShareAccess(ctx context.Context, req *AllowNfsSha } // CreateNfsShare used for create nfs share -func (cli *BaseClient) CreateNfsShare(ctx context.Context, +func (cli *OceanstorClient) CreateNfsShare(ctx context.Context, params map[string]interface{}) (map[string]interface{}, error) { data := map[string]interface{}{ "SHAREPATH": params["sharepath"].(string), @@ -451,7 +452,7 @@ func (cli *BaseClient) CreateNfsShare(ctx context.Context, } // DeleteNfsShareAccess used for delete nfs share access -func (cli *BaseClient) DeleteNfsShareAccess(ctx context.Context, accessID, vStoreID string) error { +func (cli *OceanstorClient) DeleteNfsShareAccess(ctx context.Context, accessID, vStoreID string) error { url := fmt.Sprintf("/NFS_SHARE_AUTH_CLIENT/%s", accessID) var data = make(map[string]interface{}) if vStoreID != "" { @@ -471,7 +472,7 @@ func (cli *BaseClient) DeleteNfsShareAccess(ctx context.Context, accessID, vStor } // DeleteNfsShare used for delete nfs share by id -func (cli *BaseClient) DeleteNfsShare(ctx context.Context, id, vStoreID string) error { +func (cli *OceanstorClient) DeleteNfsShare(ctx context.Context, id, vStoreID string) error { url := fmt.Sprintf("/NFSHARE/%s", id) var data = make(map[string]interface{}) if vStoreID != "" { @@ -497,7 +498,7 @@ func (cli *BaseClient) DeleteNfsShare(ctx context.Context, id, vStoreID string) } // SafeDeleteNfsShare used for delete nfs share by id -func (cli *BaseClient) SafeDeleteNfsShare(ctx context.Context, id, vStoreID string) error { +func (cli *OceanstorClient) SafeDeleteNfsShare(ctx context.Context, id, vStoreID string) error { url := fmt.Sprintf("/NFSHARE/%s", id) var data = make(map[string]interface{}) if vStoreID != "" { @@ -523,7 +524,7 @@ func (cli *BaseClient) SafeDeleteNfsShare(ctx context.Context, id, vStoreID stri } // GetNFSServiceSetting used for get nfs service setting -func (cli *BaseClient) GetNFSServiceSetting(ctx context.Context) (map[string]bool, error) { +func (cli *OceanstorClient) GetNFSServiceSetting(ctx context.Context) (map[string]bool, error) { resp, err := cli.Get(ctx, "/nfsservice", nil) if err != nil { // All enterprise storage support this interface. @@ -545,6 +546,7 @@ func (cli *BaseClient) GetNFSServiceSetting(ctx context.Context) (map[string]boo "SupportNFS3": true, "SupportNFS4": false, "SupportNFS41": false, + "SupportNFS42": false, } respData, ok := resp.Data.(map[string]interface{}) if !ok { @@ -558,6 +560,8 @@ func (cli *BaseClient) GetNFSServiceSetting(ctx context.Context) (map[string]boo setting["SupportNFS4"], err = strconv.ParseBool(v.(string)) } else if k == "SUPPORTV41" { setting["SupportNFS41"], err = strconv.ParseBool(v.(string)) + } else if k == "SUPPORTV42" { + setting["SupportNFS42"], err = strconv.ParseBool(v.(string)) } if err != nil { @@ -570,7 +574,7 @@ func (cli *BaseClient) GetNFSServiceSetting(ctx context.Context) (map[string]boo } // CreateFileSystem used for create file system -func (cli *BaseClient) CreateFileSystem(ctx context.Context, params map[string]interface{}) ( +func (cli *OceanstorClient) CreateFileSystem(ctx context.Context, params map[string]interface{}) ( map[string]interface{}, error) { resp, err := cli.Post(ctx, "/filesystem", params) if err != nil { diff --git a/storage/oceanstor/client/client_filesystem_response.go b/storage/oceanstorage/oceanstor/client/client_filesystem_response.go similarity index 100% rename from storage/oceanstor/client/client_filesystem_response.go rename to storage/oceanstorage/oceanstor/client/client_filesystem_response.go diff --git a/storage/oceanstor/client/client_filesystem_test.go b/storage/oceanstorage/oceanstor/client/client_filesystem_test.go similarity index 76% rename from storage/oceanstor/client/client_filesystem_test.go rename to storage/oceanstorage/oceanstor/client/client_filesystem_test.go index 86062782..bb576817 100644 --- a/storage/oceanstor/client/client_filesystem_test.go +++ b/storage/oceanstorage/oceanstor/client/client_filesystem_test.go @@ -23,11 +23,13 @@ import ( "github.com/agiledragon/gomonkey/v2" "github.com/stretchr/testify/require" + + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/base" ) func TestAllowNfsShareAccess(t *testing.T) { t.Run("Normal", func(t *testing.T) { - p := gomonkey.ApplyMethodReturn(testClient, "Post", Response{ + p := gomonkey.ApplyMethodReturn(testClient.RestClient, "Post", base.Response{ Data: map[string]interface{}{ "ID": "5", }, @@ -51,18 +53,16 @@ func TestAllowNfsShareAccess(t *testing.T) { }) t.Run("Error code is not zero", func(t *testing.T) { - p := gomonkey.ApplyMethod(testClient, "Post", - func(_ *BaseClient, _ context.Context, _ string, _ map[string]interface{}) (Response, error) { - return Response{ - Data: map[string]interface{}{ - "ID": "5", - }, - Error: map[string]interface{}{ - "code": float64(100), - "description": "0", - }, - }, nil - }) + p := gomonkey.ApplyMethodReturn(testClient.RestClient, "Post", + base.Response{ + Data: map[string]interface{}{ + "ID": "5", + }, + Error: map[string]interface{}{ + "code": float64(100), + "description": "0", + }, + }, nil) defer p.Reset() err := testClient.AllowNfsShareAccess(context.TODO(), &AllowNfsShareAccessRequest{ @@ -78,10 +78,8 @@ func TestAllowNfsShareAccess(t *testing.T) { }) t.Run("Post quest return error", func(t *testing.T) { - p := gomonkey.ApplyMethod(testClient, "Post", - func(_ *BaseClient, _ context.Context, _ string, _ map[string]interface{}) (Response, error) { - return Response{}, errors.New("mock err") - }) + p := gomonkey.ApplyMethodReturn(testClient.RestClient, "Post", + base.Response{}, errors.New("mock err")) defer p.Reset() err := testClient.AllowNfsShareAccess(context.TODO(), &AllowNfsShareAccessRequest{ diff --git a/storage/oceanstor/client/client_fssnapshot.go b/storage/oceanstorage/oceanstor/client/client_fssnapshot.go similarity index 89% rename from storage/oceanstor/client/client_fssnapshot.go rename to storage/oceanstorage/oceanstor/client/client_fssnapshot.go index d3ca9679..4efc01f0 100644 --- a/storage/oceanstor/client/client_fssnapshot.go +++ b/storage/oceanstorage/oceanstor/client/client_fssnapshot.go @@ -21,8 +21,8 @@ import ( "errors" "fmt" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( @@ -45,7 +45,7 @@ type FSSnapshot interface { } // DeleteFSSnapshot used for delete file system snapshot by id -func (cli *BaseClient) DeleteFSSnapshot(ctx context.Context, snapshotID string) error { +func (cli *OceanstorClient) DeleteFSSnapshot(ctx context.Context, snapshotID string) error { url := fmt.Sprintf("/FSSNAPSHOT/%s", snapshotID) resp, err := cli.Delete(ctx, url, nil) if err != nil { @@ -65,8 +65,8 @@ func (cli *BaseClient) DeleteFSSnapshot(ctx context.Context, snapshotID string) } // GetFSSnapshotByName used for get file system snapshot by snapshot name -func (cli *BaseClient) GetFSSnapshotByName(ctx context.Context, parentID, snapshotName string) (map[string]interface{}, - error) { +func (cli *OceanstorClient) GetFSSnapshotByName(ctx context.Context, + parentID, snapshotName string) (map[string]interface{}, error) { url := fmt.Sprintf("/FSSNAPSHOT?PARENTID=%s&filter=NAME::%s", parentID, snapshotName) resp, err := cli.Get(ctx, url, nil) @@ -106,7 +106,7 @@ func (cli *BaseClient) GetFSSnapshotByName(ctx context.Context, parentID, snapsh } // GetFSSnapshotCountByParentId used for get file system snapshot count by parent id -func (cli *BaseClient) GetFSSnapshotCountByParentId(ctx context.Context, ParentId string) (int, error) { +func (cli *OceanstorClient) GetFSSnapshotCountByParentId(ctx context.Context, ParentId string) (int, error) { url := fmt.Sprintf("/FSSNAPSHOT/count?PARENTID=%s", ParentId) resp, err := cli.Get(ctx, url, nil) if err != nil { @@ -138,7 +138,8 @@ func (cli *BaseClient) GetFSSnapshotCountByParentId(ctx context.Context, ParentI } // CreateFSSnapshot used for create file system snapshot -func (cli *BaseClient) CreateFSSnapshot(ctx context.Context, name, parentID string) (map[string]interface{}, error) { +func (cli *OceanstorClient) CreateFSSnapshot(ctx context.Context, + name, parentID string) (map[string]interface{}, error) { data := map[string]interface{}{ "NAME": name, "DESCRIPTION": description, diff --git a/storage/oceanstor/client/client_hypermetro.go b/storage/oceanstorage/oceanstor/client/client_hypermetro.go similarity index 88% rename from storage/oceanstor/client/client_hypermetro.go rename to storage/oceanstorage/oceanstor/client/client_hypermetro.go index bea428d6..34b17245 100644 --- a/storage/oceanstor/client/client_hypermetro.go +++ b/storage/oceanstorage/oceanstor/client/client_hypermetro.go @@ -20,8 +20,8 @@ import ( "context" "fmt" - pkgUtils "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/utils/log" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( @@ -62,7 +62,8 @@ type HyperMetro interface { } // GetHyperMetroDomainByName used for get hyper metro domain by name -func (cli *BaseClient) GetHyperMetroDomainByName(ctx context.Context, name string) (map[string]interface{}, error) { +func (cli *OceanstorClient) GetHyperMetroDomainByName(ctx context.Context, + name string) (map[string]interface{}, error) { resp, err := cli.Get(ctx, "/HyperMetroDomain?range=[0-100]", nil) if err != nil { return nil, err @@ -96,7 +97,7 @@ func (cli *BaseClient) GetHyperMetroDomainByName(ctx context.Context, name strin } // GetHyperMetroDomain used for get hyper metro domain by domain id -func (cli *BaseClient) GetHyperMetroDomain(ctx context.Context, domainID string) (map[string]interface{}, error) { +func (cli *OceanstorClient) GetHyperMetroDomain(ctx context.Context, domainID string) (map[string]interface{}, error) { url := fmt.Sprintf("/HyperMetroDomain/%s", domainID) resp, err := cli.Get(ctx, url, nil) if err != nil { @@ -116,7 +117,8 @@ func (cli *BaseClient) GetHyperMetroDomain(ctx context.Context, domainID string) } // GetFSHyperMetroDomain used for get file system hyper metro domain by domain name -func (cli *BaseClient) GetFSHyperMetroDomain(ctx context.Context, domainName string) (map[string]interface{}, error) { +func (cli *OceanstorClient) GetFSHyperMetroDomain(ctx context.Context, + domainName string) (map[string]interface{}, error) { url := "/FsHyperMetroDomain?RUNNINGSTATUS=0" resp, err := cli.Get(ctx, url, nil) if err != nil { @@ -152,7 +154,7 @@ func (cli *BaseClient) GetFSHyperMetroDomain(ctx context.Context, domainName str } // GetHyperMetroPair used for get hyper metro pair by pair id -func (cli *BaseClient) GetHyperMetroPair(ctx context.Context, pairID string) (map[string]interface{}, error) { +func (cli *OceanstorClient) GetHyperMetroPair(ctx context.Context, pairID string) (map[string]interface{}, error) { url := fmt.Sprintf("/HyperMetroPair?filter=ID::%s", pairID) resp, err := cli.Get(ctx, url, nil) @@ -187,7 +189,7 @@ func (cli *BaseClient) GetHyperMetroPair(ctx context.Context, pairID string) (ma } // GetHyperMetroPairByLocalObjID used for get hyper metro pair by local object id -func (cli *BaseClient) GetHyperMetroPairByLocalObjID(ctx context.Context, objID string) (map[string]any, error) { +func (cli *OceanstorClient) GetHyperMetroPairByLocalObjID(ctx context.Context, objID string) (map[string]any, error) { url := fmt.Sprintf("/HyperMetroPair?filter=LOCALOBJID::%s", objID) resp, err := cli.Get(ctx, url, nil) @@ -225,7 +227,7 @@ func (cli *BaseClient) GetHyperMetroPairByLocalObjID(ctx context.Context, objID } // CreateHyperMetroPair used for create hyper metro pair -func (cli *BaseClient) CreateHyperMetroPair(ctx context.Context, data map[string]interface{}) ( +func (cli *OceanstorClient) CreateHyperMetroPair(ctx context.Context, data map[string]interface{}) ( map[string]interface{}, error) { resp, err := cli.Post(ctx, "/HyperMetroPair", data) @@ -246,7 +248,7 @@ func (cli *BaseClient) CreateHyperMetroPair(ctx context.Context, data map[string } // SyncHyperMetroPair used for synchronize hyper metro pair -func (cli *BaseClient) SyncHyperMetroPair(ctx context.Context, pairID string) error { +func (cli *OceanstorClient) SyncHyperMetroPair(ctx context.Context, pairID string) error { data := map[string]interface{}{ "ID": pairID, } @@ -265,7 +267,7 @@ func (cli *BaseClient) SyncHyperMetroPair(ctx context.Context, pairID string) er } // StopHyperMetroPair used for stop hyper metro pair -func (cli *BaseClient) StopHyperMetroPair(ctx context.Context, pairID string) error { +func (cli *OceanstorClient) StopHyperMetroPair(ctx context.Context, pairID string) error { data := map[string]interface{}{ "ID": pairID, } @@ -284,7 +286,7 @@ func (cli *BaseClient) StopHyperMetroPair(ctx context.Context, pairID string) er } // DeleteHyperMetroPair used for delete hyper metro pair by pair id -func (cli *BaseClient) DeleteHyperMetroPair(ctx context.Context, pairID string, onlineDelete bool) error { +func (cli *OceanstorClient) DeleteHyperMetroPair(ctx context.Context, pairID string, onlineDelete bool) error { url := fmt.Sprintf("/HyperMetroPair/%s", pairID) if !onlineDelete { url += "?isOnlineDeleting=0" diff --git a/storage/oceanstor/client/client_lif.go b/storage/oceanstorage/oceanstor/client/client_lif.go similarity index 75% rename from storage/oceanstor/client/client_lif.go rename to storage/oceanstorage/oceanstor/client/client_lif.go index 73db91b7..955640c2 100644 --- a/storage/oceanstor/client/client_lif.go +++ b/storage/oceanstorage/oceanstor/client/client_lif.go @@ -19,8 +19,9 @@ package client import ( "context" "fmt" + netUrl "net/url" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // LIF defines interfaces for lif operations @@ -34,7 +35,7 @@ type LIF interface { } // GetLogicPort gets logic port information by port address -func (cli *BaseClient) GetLogicPort(ctx context.Context, addr string) (*Lif, error) { +func (cli *OceanstorClient) GetLogicPort(ctx context.Context, addr string) (*Lif, error) { url := fmt.Sprintf("/lif?filter=IPV4ADDR:%s&range=[0-100]", addr) resp, err := cli.Get(ctx, url, nil) if err != nil { @@ -57,3 +58,18 @@ func (cli *BaseClient) GetLogicPort(ctx context.Context, addr string) (*Lif, err return lifs[0], nil } + +// GetCurrentLifWwn used for get current lif wwn +func (cli *OceanstorClient) GetCurrentLifWwn() string { + return cli.CurrentLifWwn +} + +// GetCurrentLif used for get current lif wwn +func (cli *OceanstorClient) GetCurrentLif(ctx context.Context) string { + u, err := netUrl.Parse(cli.Url) + if err != nil { + log.AddContext(ctx).Errorf("parse url failed, error: %v", err) + return "" + } + return u.Hostname() +} diff --git a/storage/oceanstor/client/client_lif_test.go b/storage/oceanstorage/oceanstor/client/client_lif_test.go similarity index 82% rename from storage/oceanstor/client/client_lif_test.go rename to storage/oceanstorage/oceanstor/client/client_lif_test.go index 460892fe..2076fbe6 100644 --- a/storage/oceanstor/client/client_lif_test.go +++ b/storage/oceanstorage/oceanstor/client/client_lif_test.go @@ -27,11 +27,12 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "huawei-csi-driver/storage/oceanstor/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/base" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceanstor/client" ) -func mockCli() *client.BaseClient { - return &client.BaseClient{ +func mockCli() *client.OceanstorClient { + restClient := &client.RestClient{ Client: http.DefaultClient, Url: "https://localhost:8088", Urls: []string{"localhost"}, @@ -39,6 +40,9 @@ func mockCli() *client.BaseClient { VStoreName: "testVStore", BackendID: "backend-test", } + return &client.OceanstorClient{ + RestClient: restClient, + } } var ( @@ -80,8 +84,8 @@ var ( }` ) -func responseOfLif(rawResp string) client.Response { - var resp client.Response +func responseOfLif(rawResp string) base.Response { + var resp base.Response if err := json.Unmarshal([]byte(rawResp), &resp); err != nil { fmt.Println(err) } @@ -97,7 +101,7 @@ func TestBaseClient_GetLogicPort(t *testing.T) { resp := responseOfLif(rawSuccessResp) // mock - patches := gomonkey.ApplyMethodReturn(cli, "Get", resp, nil) + patches := gomonkey.ApplyMethodReturn(cli.RestClient, "Get", resp, nil) defer patches.Reset() // act @@ -126,7 +130,7 @@ func TestBaseClient_GetLogicPort(t *testing.T) { cli := mockCli() // mock - patches := gomonkey.ApplyMethodReturn(cli, "Get", nil, assert.AnError) + patches := gomonkey.ApplyMethodReturn(cli.RestClient, "Get", nil, assert.AnError) defer patches.Reset() // act @@ -143,7 +147,7 @@ func TestBaseClient_GetLogicPort(t *testing.T) { resp := responseOfLif(errorCodeResp) // mock - patches := gomonkey.ApplyMethodReturn(cli, "Get", resp, nil) + patches := gomonkey.ApplyMethodReturn(cli.RestClient, "Get", resp, nil) defer patches.Reset() // act @@ -160,7 +164,7 @@ func TestBaseClient_GetLogicPort(t *testing.T) { resp := responseOfLif(wrongDataResp) // mock - patches := gomonkey.ApplyMethodReturn(cli, "Get", resp, nil) + patches := gomonkey.ApplyMethodReturn(cli.RestClient, "Get", resp, nil) defer patches.Reset() // act @@ -177,7 +181,7 @@ func TestBaseClient_GetLogicPort(t *testing.T) { resp := responseOfLif(notFoundResp) // mock - patches := gomonkey.ApplyMethodReturn(cli, "Get", resp, nil) + patches := gomonkey.ApplyMethodReturn(cli.RestClient, "Get", resp, nil) defer patches.Reset() // act diff --git a/storage/oceanstor/client/client_lif_types.go b/storage/oceanstorage/oceanstor/client/client_lif_types.go similarity index 100% rename from storage/oceanstor/client/client_lif_types.go rename to storage/oceanstorage/oceanstor/client/client_lif_types.go diff --git a/storage/oceanstor/client/client_lun.go b/storage/oceanstorage/oceanstor/client/client_lun.go similarity index 86% rename from storage/oceanstor/client/client_lun.go rename to storage/oceanstorage/oceanstor/client/client_lun.go index dab00df0..1f3e9fd9 100644 --- a/storage/oceanstor/client/client_lun.go +++ b/storage/oceanstorage/oceanstor/client/client_lun.go @@ -23,18 +23,19 @@ import ( "fmt" "strconv" - "huawei-csi-driver/pkg/constants" - pkgUtils "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( - objectNotExist int64 = 1077948996 - objectIdNotUnique int64 = 1077948997 - lunAlreadyInGroup int64 = 1077936862 - lunNotExist int64 = 1077936859 - parameterIncorrect int64 = 50331651 + objectNotExist int64 = 1077948996 + objectIdNotUnique int64 = 1077948997 + lunAlreadyInGroup int64 = 1077936862 + lunNotExist int64 = 1077936859 + parameterIncorrect int64 = 50331651 + objectNameAlreadyExist int64 = 1077948993 maxLunNameLength = 31 ) @@ -76,7 +77,8 @@ type Lun interface { } // QueryAssociateLunGroup used for query associate lun group by object type and object id -func (cli *BaseClient) QueryAssociateLunGroup(ctx context.Context, objType int, objID string) ([]interface{}, error) { +func (cli *OceanstorClient) QueryAssociateLunGroup(ctx context.Context, + objType int, objID string) ([]interface{}, error) { url := fmt.Sprintf("/lungroup/associate?ASSOCIATEOBJTYPE=%d&ASSOCIATEOBJID=%s", objType, objID) resp, err := cli.Get(ctx, url, nil) if err != nil { @@ -101,7 +103,7 @@ func (cli *BaseClient) QueryAssociateLunGroup(ctx context.Context, objType int, } // GetLunByName used for get lun by name -func (cli *BaseClient) GetLunByName(ctx context.Context, name string) (map[string]interface{}, error) { +func (cli *OceanstorClient) GetLunByName(ctx context.Context, name string) (map[string]interface{}, error) { url := fmt.Sprintf("/lun?filter=NAME::%s&range=[0-100]", name) resp, err := cli.Get(ctx, url, nil) if err != nil { @@ -132,7 +134,7 @@ func (cli *BaseClient) GetLunByName(ctx context.Context, name string) (map[strin } // MakeLunName v3/v5 storage support 1 to 31 characters -func (cli *BaseClient) MakeLunName(name string) string { +func (cli *OceanstorClient) MakeLunName(name string) string { if len(name) <= maxLunNameLength { return name } @@ -140,7 +142,7 @@ func (cli *BaseClient) MakeLunName(name string) string { } // GetLunByID used for get lun by id -func (cli *BaseClient) GetLunByID(ctx context.Context, id string) (map[string]interface{}, error) { +func (cli *OceanstorClient) GetLunByID(ctx context.Context, id string) (map[string]interface{}, error) { url := fmt.Sprintf("/lun/%s", id) resp, err := cli.Get(ctx, url, nil) if err != nil { @@ -162,7 +164,7 @@ func (cli *BaseClient) GetLunByID(ctx context.Context, id string) (map[string]in } // AddLunToGroup used for add lun to group -func (cli *BaseClient) AddLunToGroup(ctx context.Context, lunID string, groupID string) error { +func (cli *OceanstorClient) AddLunToGroup(ctx context.Context, lunID string, groupID string) error { data := map[string]interface{}{ "ID": groupID, "ASSOCIATEOBJTYPE": "11", @@ -188,7 +190,7 @@ func (cli *BaseClient) AddLunToGroup(ctx context.Context, lunID string, groupID } // RemoveLunFromGroup used for remove lun from group -func (cli *BaseClient) RemoveLunFromGroup(ctx context.Context, lunID, groupID string) error { +func (cli *OceanstorClient) RemoveLunFromGroup(ctx context.Context, lunID, groupID string) error { data := map[string]interface{}{ "ID": groupID, "ASSOCIATEOBJTYPE": "11", @@ -214,7 +216,7 @@ func (cli *BaseClient) RemoveLunFromGroup(ctx context.Context, lunID, groupID st } // GetLunGroupByName used for get lun group by name -func (cli *BaseClient) GetLunGroupByName(ctx context.Context, name string) (map[string]interface{}, error) { +func (cli *OceanstorClient) GetLunGroupByName(ctx context.Context, name string) (map[string]interface{}, error) { url := fmt.Sprintf("/lungroup?filter=NAME::%s", name) resp, err := cli.Get(ctx, url, nil) if err != nil { @@ -250,7 +252,7 @@ func (cli *BaseClient) GetLunGroupByName(ctx context.Context, name string) (map[ } // CreateLunGroup used for create lun group -func (cli *BaseClient) CreateLunGroup(ctx context.Context, name string) (map[string]interface{}, error) { +func (cli *OceanstorClient) CreateLunGroup(ctx context.Context, name string) (map[string]interface{}, error) { data := map[string]interface{}{ "NAME": name, "APPTYPE": 0, @@ -278,7 +280,7 @@ func (cli *BaseClient) CreateLunGroup(ctx context.Context, name string) (map[str } // DeleteLunGroup used for delete lun group by lun group id -func (cli *BaseClient) DeleteLunGroup(ctx context.Context, id string) error { +func (cli *OceanstorClient) DeleteLunGroup(ctx context.Context, id string) error { url := fmt.Sprintf("/lungroup/%s", id) resp, err := cli.Delete(ctx, url, nil) if err != nil { @@ -299,7 +301,8 @@ func (cli *BaseClient) DeleteLunGroup(ctx context.Context, id string) error { } // CreateLun used for create lun -func (cli *BaseClient) CreateLun(ctx context.Context, params map[string]interface{}) (map[string]interface{}, error) { +func (cli *OceanstorClient) CreateLun(ctx context.Context, + params map[string]interface{}) (map[string]interface{}, error) { data := map[string]interface{}{ "NAME": params["name"].(string), "PARENTID": params["parentid"].(string), @@ -334,7 +337,7 @@ func (cli *BaseClient) CreateLun(ctx context.Context, params map[string]interfac } // DeleteLun used for delete lun by lun id -func (cli *BaseClient) DeleteLun(ctx context.Context, id string) error { +func (cli *OceanstorClient) DeleteLun(ctx context.Context, id string) error { url := fmt.Sprintf("/lun/%s", id) resp, err := cli.Delete(ctx, url, nil) if err != nil { @@ -355,7 +358,7 @@ func (cli *BaseClient) DeleteLun(ctx context.Context, id string) error { } // ExtendLun used for extend lun -func (cli *BaseClient) ExtendLun(ctx context.Context, lunID string, newCapacity int64) error { +func (cli *OceanstorClient) ExtendLun(ctx context.Context, lunID string, newCapacity int64) error { data := map[string]interface{}{ "CAPACITY": newCapacity, "ID": lunID, @@ -375,7 +378,7 @@ func (cli *BaseClient) ExtendLun(ctx context.Context, lunID string, newCapacity } // GetLunCountOfMapping used for get lun count of mapping by mapping id -func (cli *BaseClient) GetLunCountOfMapping(ctx context.Context, mappingID string) (int64, error) { +func (cli *OceanstorClient) GetLunCountOfMapping(ctx context.Context, mappingID string) (int64, error) { url := fmt.Sprintf("/lun/count?ASSOCIATEOBJTYPE=245&ASSOCIATEOBJID=%s", mappingID) resp, err := cli.Get(ctx, url, nil) if err != nil { @@ -402,7 +405,7 @@ func (cli *BaseClient) GetLunCountOfMapping(ctx context.Context, mappingID strin } // GetLunCountOfHost used for get lun count of host -func (cli *BaseClient) GetLunCountOfHost(ctx context.Context, hostID string) (int64, error) { +func (cli *OceanstorClient) GetLunCountOfHost(ctx context.Context, hostID string) (int64, error) { url := fmt.Sprintf("/lun/count?ASSOCIATEOBJTYPE=21&ASSOCIATEOBJID=%s", hostID) resp, err := cli.Get(ctx, url, nil) if err != nil { @@ -429,7 +432,7 @@ func (cli *BaseClient) GetLunCountOfHost(ctx context.Context, hostID string) (in } // GetHostLunId used for get host lun id -func (cli *BaseClient) GetHostLunId(ctx context.Context, hostID, lunID string) (string, error) { +func (cli *OceanstorClient) GetHostLunId(ctx context.Context, hostID, lunID string) (string, error) { hostLunId := "1" url := fmt.Sprintf("/lun/associate?TYPE=11&ASSOCIATEOBJTYPE=21&ASSOCIATEOBJID=%s", hostID) resp, err := cli.Get(ctx, url, nil) @@ -473,7 +476,7 @@ func (cli *BaseClient) GetHostLunId(ctx context.Context, hostID, lunID string) ( } // UpdateLun used for update lun -func (cli *BaseClient) UpdateLun(ctx context.Context, lunID string, params map[string]interface{}) error { +func (cli *OceanstorClient) UpdateLun(ctx context.Context, lunID string, params map[string]interface{}) error { url := fmt.Sprintf("/lun/%s", lunID) resp, err := cli.Put(ctx, url, params) if err != nil { diff --git a/storage/oceanstor/client/client_luncopy.go b/storage/oceanstorage/oceanstor/client/client_luncopy.go similarity index 86% rename from storage/oceanstor/client/client_luncopy.go rename to storage/oceanstorage/oceanstor/client/client_luncopy.go index 7e31ba87..f95c224f 100644 --- a/storage/oceanstor/client/client_luncopy.go +++ b/storage/oceanstorage/oceanstor/client/client_luncopy.go @@ -20,8 +20,8 @@ import ( "context" "fmt" - pkgUtils "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/utils/log" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( @@ -45,7 +45,7 @@ type LunCopy interface { } // CreateLunCopy used for create lun copy -func (cli *BaseClient) CreateLunCopy(ctx context.Context, name, srcLunID, dstLunID string, copySpeed int) ( +func (cli *OceanstorClient) CreateLunCopy(ctx context.Context, name, srcLunID, dstLunID string, copySpeed int) ( map[string]interface{}, error) { data := map[string]interface{}{ @@ -73,7 +73,7 @@ func (cli *BaseClient) CreateLunCopy(ctx context.Context, name, srcLunID, dstLun } // GetLunCopyByID used for get lun copy by id -func (cli *BaseClient) GetLunCopyByID(ctx context.Context, lunCopyID string) (map[string]interface{}, error) { +func (cli *OceanstorClient) GetLunCopyByID(ctx context.Context, lunCopyID string) (map[string]interface{}, error) { url := fmt.Sprintf("/LUNCOPY/%s", lunCopyID) resp, err := cli.Get(ctx, url, nil) @@ -94,7 +94,7 @@ func (cli *BaseClient) GetLunCopyByID(ctx context.Context, lunCopyID string) (ma } // GetLunCopyByName used for get lun copy by name -func (cli *BaseClient) GetLunCopyByName(ctx context.Context, name string) (map[string]interface{}, error) { +func (cli *OceanstorClient) GetLunCopyByName(ctx context.Context, name string) (map[string]interface{}, error) { url := fmt.Sprintf("/LUNCOPY?filter=NAME::%s", name) resp, err := cli.Get(ctx, url, nil) @@ -129,7 +129,7 @@ func (cli *BaseClient) GetLunCopyByName(ctx context.Context, name string) (map[s } // StartLunCopy used for start lun copy -func (cli *BaseClient) StartLunCopy(ctx context.Context, lunCopyID string) error { +func (cli *OceanstorClient) StartLunCopy(ctx context.Context, lunCopyID string) error { data := map[string]interface{}{ "ID": lunCopyID, } @@ -148,7 +148,7 @@ func (cli *BaseClient) StartLunCopy(ctx context.Context, lunCopyID string) error } // StopLunCopy used for stop lun copy -func (cli *BaseClient) StopLunCopy(ctx context.Context, lunCopyID string) error { +func (cli *OceanstorClient) StopLunCopy(ctx context.Context, lunCopyID string) error { data := map[string]interface{}{ "ID": lunCopyID, } @@ -167,7 +167,7 @@ func (cli *BaseClient) StopLunCopy(ctx context.Context, lunCopyID string) error } // DeleteLunCopy used for delete lun copy by id -func (cli *BaseClient) DeleteLunCopy(ctx context.Context, lunCopyID string) error { +func (cli *OceanstorClient) DeleteLunCopy(ctx context.Context, lunCopyID string) error { url := fmt.Sprintf("/LUNCOPY/%s", lunCopyID) resp, err := cli.Delete(ctx, url, nil) diff --git a/storage/oceanstor/client/client_lunsnapshot.go b/storage/oceanstorage/oceanstor/client/client_lunsnapshot.go similarity index 86% rename from storage/oceanstor/client/client_lunsnapshot.go rename to storage/oceanstorage/oceanstor/client/client_lunsnapshot.go index ba77f8a0..0b38bac3 100644 --- a/storage/oceanstor/client/client_lunsnapshot.go +++ b/storage/oceanstorage/oceanstor/client/client_lunsnapshot.go @@ -20,8 +20,8 @@ import ( "context" "fmt" - pkgUtils "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/utils/log" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( @@ -44,7 +44,7 @@ type LunSnapshot interface { } // CreateLunSnapshot used for create lun snapshot -func (cli *BaseClient) CreateLunSnapshot(ctx context.Context, name, lunID string) (map[string]interface{}, error) { +func (cli *OceanstorClient) CreateLunSnapshot(ctx context.Context, name, lunID string) (map[string]interface{}, error) { data := map[string]interface{}{ "NAME": name, "DESCRIPTION": description, @@ -69,7 +69,7 @@ func (cli *BaseClient) CreateLunSnapshot(ctx context.Context, name, lunID string } // GetLunSnapshotByName used for get lun snapshot by name -func (cli *BaseClient) GetLunSnapshotByName(ctx context.Context, name string) (map[string]interface{}, error) { +func (cli *OceanstorClient) GetLunSnapshotByName(ctx context.Context, name string) (map[string]interface{}, error) { url := fmt.Sprintf("/snapshot?filter=NAME::%s&range=[0-100]", name) resp, err := cli.Get(ctx, url, nil) @@ -103,7 +103,7 @@ func (cli *BaseClient) GetLunSnapshotByName(ctx context.Context, name string) (m } // DeleteLunSnapshot used for delete lun snapshot -func (cli *BaseClient) DeleteLunSnapshot(ctx context.Context, snapshotID string) error { +func (cli *OceanstorClient) DeleteLunSnapshot(ctx context.Context, snapshotID string) error { url := fmt.Sprintf("/snapshot/%s", snapshotID) resp, err := cli.Delete(ctx, url, nil) if err != nil { @@ -123,7 +123,7 @@ func (cli *BaseClient) DeleteLunSnapshot(ctx context.Context, snapshotID string) } // ActivateLunSnapshot used for activate lun snapshot -func (cli *BaseClient) ActivateLunSnapshot(ctx context.Context, snapshotID string) error { +func (cli *OceanstorClient) ActivateLunSnapshot(ctx context.Context, snapshotID string) error { data := map[string]interface{}{ "SNAPSHOTLIST": []string{snapshotID}, } @@ -142,7 +142,7 @@ func (cli *BaseClient) ActivateLunSnapshot(ctx context.Context, snapshotID strin } // DeactivateLunSnapshot used for stop lun snapshot -func (cli *BaseClient) DeactivateLunSnapshot(ctx context.Context, snapshotID string) error { +func (cli *OceanstorClient) DeactivateLunSnapshot(ctx context.Context, snapshotID string) error { data := map[string]interface{}{ "ID": snapshotID, } diff --git a/storage/oceanstor/client/client_mock.go b/storage/oceanstorage/oceanstor/client/client_mock.go similarity index 100% rename from storage/oceanstor/client/client_mock.go rename to storage/oceanstorage/oceanstor/client/client_mock.go diff --git a/storage/oceanstor/client/client_quota.go b/storage/oceanstorage/oceanstor/client/client_quota.go similarity index 87% rename from storage/oceanstor/client/client_quota.go rename to storage/oceanstorage/oceanstor/client/client_quota.go index 1714f901..66b10666 100644 --- a/storage/oceanstor/client/client_quota.go +++ b/storage/oceanstorage/oceanstor/client/client_quota.go @@ -20,7 +20,7 @@ import ( "context" "fmt" - "huawei-csi-driver/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" ) const ( @@ -58,7 +58,8 @@ type OceanStorQuota interface { } // CreateQuota creates quota by params -func (cli *BaseClient) CreateQuota(ctx context.Context, params map[string]interface{}) (map[string]interface{}, error) { +func (cli *OceanstorClient) CreateQuota(ctx context.Context, + params map[string]interface{}) (map[string]interface{}, error) { resp, err := cli.Post(ctx, "/FS_QUOTA", params) if err != nil { return nil, err @@ -72,7 +73,7 @@ func (cli *BaseClient) CreateQuota(ctx context.Context, params map[string]interf } // UpdateQuota updates quota by id -func (cli *BaseClient) UpdateQuota(ctx context.Context, quotaID string, params map[string]interface{}) error { +func (cli *OceanstorClient) UpdateQuota(ctx context.Context, quotaID string, params map[string]interface{}) error { resp, err := cli.Put(ctx, fmt.Sprintf("/FS_QUOTA/%v", quotaID), params) if err != nil { return err @@ -86,7 +87,7 @@ func (cli *BaseClient) UpdateQuota(ctx context.Context, quotaID string, params m } // GetQuota gets quota info by id -func (cli *BaseClient) GetQuota(ctx context.Context, +func (cli *OceanstorClient) GetQuota(ctx context.Context, quotaID, vStoreID string, spaceUnitType uint32) (map[string]interface{}, error) { resp, err := cli.Get(ctx, fmt.Sprintf("/FS_QUOTA/%v", quotaID), map[string]interface{}{ "SPACEUNITTYPE": spaceUnitType, @@ -104,7 +105,7 @@ func (cli *BaseClient) GetQuota(ctx context.Context, } // BatchGetQuota gets quotas filtered by params -func (cli *BaseClient) BatchGetQuota(ctx context.Context, params map[string]interface{}) ([]interface{}, error) { +func (cli *OceanstorClient) BatchGetQuota(ctx context.Context, params map[string]interface{}) ([]interface{}, error) { url := fmt.Sprintf("/FS_QUOTA?PARENTTYPE=%v&PARENTID=%v&range=%v&vstoreId=%v&QUERYTYPE=%v&SPACEUNITTYPE=%v", params["PARENTTYPE"], params["PARENTID"], params["range"], params["vstoreId"], params["QUERYTYPE"], params["SPACEUNITTYPE"]) @@ -121,7 +122,7 @@ func (cli *BaseClient) BatchGetQuota(ctx context.Context, params map[string]inte } // DeleteQuota deletes quota by id -func (cli *BaseClient) DeleteQuota(ctx context.Context, quotaID, vStoreID string, forceFlag bool) error { +func (cli *OceanstorClient) DeleteQuota(ctx context.Context, quotaID, vStoreID string, forceFlag bool) error { resp, err := cli.Delete(ctx, fmt.Sprintf("/FS_QUOTA/%v", quotaID), map[string]interface{}{ "forceFlag": forceFlag, "vstoreId": vStoreID, diff --git a/storage/oceanstor/client/client_replication.go b/storage/oceanstorage/oceanstor/client/client_replication.go similarity index 88% rename from storage/oceanstor/client/client_replication.go rename to storage/oceanstorage/oceanstor/client/client_replication.go index d4c92fcd..538d058c 100644 --- a/storage/oceanstor/client/client_replication.go +++ b/storage/oceanstorage/oceanstor/client/client_replication.go @@ -20,8 +20,8 @@ import ( "context" "fmt" - pkgUtils "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/utils/log" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( @@ -47,7 +47,7 @@ type Replication interface { } // CreateReplicationPair used for create replication pair -func (cli *BaseClient) CreateReplicationPair(ctx context.Context, data map[string]interface{}) ( +func (cli *OceanstorClient) CreateReplicationPair(ctx context.Context, data map[string]interface{}) ( map[string]interface{}, error) { resp, err := cli.Post(ctx, "/REPLICATIONPAIR", data) @@ -68,7 +68,7 @@ func (cli *BaseClient) CreateReplicationPair(ctx context.Context, data map[strin } // SplitReplicationPair used for split replication pair by pair id -func (cli *BaseClient) SplitReplicationPair(ctx context.Context, pairID string) error { +func (cli *OceanstorClient) SplitReplicationPair(ctx context.Context, pairID string) error { data := map[string]interface{}{ "ID": pairID, } @@ -87,7 +87,7 @@ func (cli *BaseClient) SplitReplicationPair(ctx context.Context, pairID string) } // SyncReplicationPair used for synchronize replication pair -func (cli *BaseClient) SyncReplicationPair(ctx context.Context, pairID string) error { +func (cli *OceanstorClient) SyncReplicationPair(ctx context.Context, pairID string) error { data := map[string]interface{}{ "ID": pairID, } @@ -106,7 +106,7 @@ func (cli *BaseClient) SyncReplicationPair(ctx context.Context, pairID string) e } // DeleteReplicationPair used for delete replication pair by pair id -func (cli *BaseClient) DeleteReplicationPair(ctx context.Context, pairID string) error { +func (cli *OceanstorClient) DeleteReplicationPair(ctx context.Context, pairID string) error { url := fmt.Sprintf("/REPLICATIONPAIR/%s", pairID) resp, err := cli.Delete(ctx, url, nil) if err != nil { @@ -126,7 +126,7 @@ func (cli *BaseClient) DeleteReplicationPair(ctx context.Context, pairID string) } // GetReplicationPairByResID used for get replication -func (cli *BaseClient) GetReplicationPairByResID(ctx context.Context, resID string, resType int) ( +func (cli *OceanstorClient) GetReplicationPairByResID(ctx context.Context, resID string, resType int) ( []map[string]interface{}, error) { url := fmt.Sprintf("/REPLICATIONPAIR/associate?ASSOCIATEOBJTYPE=%d&ASSOCIATEOBJID=%s", resType, resID) @@ -159,7 +159,7 @@ func (cli *BaseClient) GetReplicationPairByResID(ctx context.Context, resID stri } // GetReplicationPairByID used for get replication pair by pair id -func (cli *BaseClient) GetReplicationPairByID(ctx context.Context, pairID string) (map[string]interface{}, error) { +func (cli *OceanstorClient) GetReplicationPairByID(ctx context.Context, pairID string) (map[string]interface{}, error) { url := fmt.Sprintf("/REPLICATIONPAIR/%s", pairID) resp, err := cli.Get(ctx, url, nil) if err != nil { @@ -179,7 +179,7 @@ func (cli *BaseClient) GetReplicationPairByID(ctx context.Context, pairID string } // GetReplicationvStorePairByvStore used for get replication vstore pair by vstore id -func (cli *BaseClient) GetReplicationvStorePairByvStore(ctx context.Context, +func (cli *OceanstorClient) GetReplicationvStorePairByvStore(ctx context.Context, vStoreID string) (map[string]interface{}, error) { url := fmt.Sprintf("/replication_vstorepair/associate?ASSOCIATEOBJTYPE=16442&ASSOCIATEOBJID=%s", vStoreID) resp, err := cli.Get(ctx, url, nil) diff --git a/storage/oceanstorage/oceanstor/client/client_restful.go b/storage/oceanstorage/oceanstor/client/client_restful.go new file mode 100644 index 00000000..b70e831b --- /dev/null +++ b/storage/oceanstorage/oceanstor/client/client_restful.go @@ -0,0 +1,569 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package client provides oceanstor storage client +package client + +import ( + "bytes" + "context" + "encoding/json" + "errors" + "fmt" + "io" + "io/ioutil" + "net/http" + netUrl "net/url" + "strconv" + "sync" + "sync/atomic" + + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/base" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" +) + +// RestClient defines client implements the rest interface +type RestClient struct { + Client base.HTTP + Url string + Urls []string + + User string + SecretNamespace string + SecretName string + VStoreName string + VStoreID string + StorageVersion string + BackendID string + Storage string + CurrentSiteWwn string + CurrentLifWwn string + LastLif string + Product constants.OceanstorVersion + DeviceId string + Token string + + SystemInfoRefreshing uint32 + ReLoginMutex sync.Mutex + RequestSemaphore *utils.Semaphore +} + +// NewRestClient inits a new rest client +func NewRestClient(ctx context.Context, param *NewClientConfig) (*RestClient, error) { + var err error + var parallelCount int + + parallelCount, err = strconv.Atoi(param.ParallelNum) + if err != nil || parallelCount > MaxParallelCount || parallelCount < MinParallelCount { + log.Infof("the config parallelNum %d is invalid, set it to the default value %d", + parallelCount, DefaultParallelCount) + parallelCount = DefaultParallelCount + } + + log.AddContext(ctx).Infof("Init parallel count is %d", parallelCount) + httpClient, err := base.NewHTTPClientByCertMeta(ctx, param.UseCert, param.CertSecretMeta) + if err != nil { + log.AddContext(ctx).Errorf("new http client by cert meta failed, err is %v", err) + return nil, err + } + + return &RestClient{ + Urls: param.Urls, + User: param.User, + Storage: param.Storage, + SecretName: param.SecretName, + SecretNamespace: param.SecretNamespace, + VStoreName: param.VstoreName, + Client: httpClient, + BackendID: param.BackendID, + RequestSemaphore: utils.NewSemaphore(parallelCount), + }, nil +} + +// Call provides call for restful request +func (cli *RestClient) Call(ctx context.Context, + method string, url string, + data map[string]interface{}) (base.Response, error) { + var r base.Response + var err error + + r, err = cli.BaseCall(ctx, method, url, data) + if !base.NeedReLogin(r, err) { + return r, err + } + + // Current connection fails, try to relogin to other Urls if exist, + // if relogin success, resend the request again. + log.AddContext(ctx).Infof("Try to relogin and resend request method: %s, Url: %s", method, url) + err = cli.ReLogin(ctx) + if err != nil { + return r, err + } + + // If the logical port changes from storage A to storage B, the system information must be updated. + if err = cli.SetSystemInfo(ctx); err != nil { + log.AddContext(ctx).Errorf("after relogin, can't get system info, error: %v", err) + return r, err + } + + return cli.BaseCall(ctx, method, url, data) +} + +// BaseCall provides base call for request +func (cli *RestClient) BaseCall(ctx context.Context, method string, url string, + data map[string]interface{}) (base.Response, error) { + var r base.Response + var req *http.Request + var err error + + if cli.Client == nil { + errMsg := "http client is nil" + log.AddContext(ctx).Errorf("Failed to send request method: %s, url: %s, error: %s", method, url, errMsg) + return base.Response{}, errors.New(errMsg) + } + + if url != "/xx/sessions" && url != "/sessions" { + cli.ReLoginMutex.Lock() + req, err = cli.GetRequest(ctx, method, url, data) + cli.ReLoginMutex.Unlock() + } else { + req, err = cli.GetRequest(ctx, method, url, data) + } + + if err != nil { + return base.Response{}, err + } + + log.FilteredLog(ctx, isFilterLog(method, url), utils.IsDebugLog(method, url, debugLog, debugLogRegex), + fmt.Sprintf("Request method: %s, Url: %s, body: %v", method, req.URL, data)) + + if cli.RequestSemaphore == nil { + return base.Response{}, errors.New("request semaphore is nil") + } + + cli.RequestSemaphore.Acquire() + defer cli.RequestSemaphore.Release() + + if base.RequestSemaphoreMap[cli.GetDeviceSN()] != nil { + base.RequestSemaphoreMap[cli.GetDeviceSN()].Acquire() + defer base.RequestSemaphoreMap[cli.GetDeviceSN()].Release() + } else { + base.RequestSemaphoreMap[base.UninitializedStorage].Acquire() + defer base.RequestSemaphoreMap[base.UninitializedStorage].Release() + } + + resp, err := cli.Client.Do(req) + if err != nil { + log.AddContext(ctx).Errorf("Send request method: %s, Url: %s, error: %v", method, req.URL, err) + return base.Response{}, errors.New(base.Unconnected) + } + defer resp.Body.Close() + + body, err := ioutil.ReadAll(resp.Body) + if err != nil { + log.AddContext(ctx).Errorf("Read response data error: %v", err) + return base.Response{}, err + } + + log.FilteredLog(ctx, isFilterLog(method, url), utils.IsDebugLog(method, url, debugLog, debugLogRegex), + fmt.Sprintf("Response method: %s, Url: %s, body: %s", method, req.URL, body)) + + err = json.Unmarshal(body, &r) + if err != nil { + log.AddContext(ctx).Errorf("json.Unmarshal data %s error: %v", body, err) + return base.Response{}, err + } + + return r, nil +} + +// Get provides http request of GET method +func (cli *RestClient) Get(ctx context.Context, url string, data map[string]interface{}) (base.Response, error) { + return cli.Call(ctx, "GET", url, data) +} + +// Post provides http request of POST method +func (cli *RestClient) Post(ctx context.Context, url string, data map[string]interface{}) (base.Response, error) { + return cli.Call(ctx, "POST", url, data) +} + +// Put provides http request of PUT method +func (cli *RestClient) Put(ctx context.Context, url string, data map[string]interface{}) (base.Response, error) { + return cli.Call(ctx, "PUT", url, data) +} + +// Delete provides http request of DELETE method +func (cli *RestClient) Delete(ctx context.Context, url string, data map[string]interface{}) (base.Response, error) { + return cli.Call(ctx, "DELETE", url, data) +} + +// GetRequest return the request info +func (cli *RestClient) GetRequest(ctx context.Context, + method string, url string, + data map[string]interface{}) (*http.Request, error) { + var req *http.Request + var err error + + reqUrl := cli.Url + if cli.DeviceId != "" { + reqUrl += "/" + cli.DeviceId + } + reqUrl += url + + var reqBody io.Reader + + if data != nil { + reqBytes, err := json.Marshal(data) + if err != nil { + log.AddContext(ctx).Errorf("json.Marshal data %v error: %v", base.MaskRequestData(data), err) + return req, err + } + reqBody = bytes.NewReader(reqBytes) + } + + req, err = http.NewRequest(method, reqUrl, reqBody) + if err != nil { + log.AddContext(ctx).Errorf("Construct http request error: %s", err.Error()) + return req, err + } + + req.Header.Set("Connection", "keep-alive") + req.Header.Set("Content-Type", "application/json") + + if cli.Token != "" { + req.Header.Set("iBaseToken", cli.Token) + } + + return req, nil +} + +// Login login and set data from response +func (cli *RestClient) Login(ctx context.Context) error { + var resp base.Response + var err error + + cli.Client, err = base.NewHTTPClientByBackendID(ctx, cli.BackendID) + if err != nil { + log.AddContext(ctx).Errorf("new http client by backend %s failed, err is %v", cli.BackendID, err) + return err + } + + data, err := cli.getRequestParams(ctx, cli.BackendID) + if err != nil { + return err + } + + cli.DeviceId = "" + cli.Token = "" + for i, url := range cli.Urls { + cli.Url = url + "/deviceManager/rest" + + log.AddContext(ctx).Infof("Try to login %s", cli.Url) + resp, err = cli.BaseCall(ctx, "POST", "/xx/sessions", data) + if err == nil { + /* Sort the login Url to the last slot of san addresses, so that + if this connection error, next time will try other Url first. */ + cli.Urls[i], cli.Urls[len(cli.Urls)-1] = cli.Urls[len(cli.Urls)-1], cli.Urls[i] + break + } else if err.Error() != base.Unconnected { + log.AddContext(ctx).Errorf("Login %s error", cli.Url) + break + } + + log.AddContext(ctx).Warningf("Login %s error due to connection failure, gonna try another Url", cli.Url) + } + + if err != nil { + return err + } + + errCode, _ := resp.Error["code"].(float64) + if code := int64(errCode); code != 0 { + msg := fmt.Sprintf("Login %s error: %+v", cli.Url, resp) + if utils.Contains(base.WrongPasswordErrorCodes, code) || utils.Contains(base.AccountBeenLocked, code) || + code == base.IPLockErrorCode { + if err := pkgUtils.SetStorageBackendContentOnlineStatus(ctx, cli.BackendID, false); err != nil { + msg = msg + fmt.Sprintf("\nSetStorageBackendContentOffline [%s] failed. error: %v", cli.BackendID, err) + } + } + return errors.New(msg) + } + + if err = cli.setDataFromRespData(ctx, resp); err != nil { + cli.Logout(ctx) + setErr := pkgUtils.SetStorageBackendContentOnlineStatus(ctx, cli.BackendID, false) + if setErr != nil { + log.AddContext(ctx).Errorf("SetStorageBackendContentOffline [%s] failed. error: %v", cli.BackendID, setErr) + } + return err + } + return nil +} + +func (cli *RestClient) setDataFromRespData(ctx context.Context, resp base.Response) error { + respData, ok := resp.Data.(map[string]interface{}) + if !ok { + return pkgUtils.Errorln(ctx, fmt.Sprintf("convert resp.Data to map[string]interface{} failed,"+ + " data type: [%T]", resp.Data)) + } + cli.DeviceId, ok = respData["deviceid"].(string) + if !ok { + return pkgUtils.Errorln(ctx, fmt.Sprintf("convert respData[\"deviceid\"]: [%v] to string failed", + respData["deviceid"])) + } + + if base.RequestSemaphoreMap[cli.DeviceId] == nil { + base.RequestSemaphoreMap[cli.DeviceId] = utils.NewSemaphore(base.MaxStorageThreads) + } + + cli.Token, ok = respData["iBaseToken"].(string) + if !ok { + return pkgUtils.Errorln(ctx, fmt.Sprintf("convert respData[\"iBaseToken\"]: [%T] to string failed", + respData["iBaseToken"])) + } + + vStoreName, exist := respData["vstoreName"].(string) + vStoreID, idExist := respData["vstoreId"].(string) + if !exist && !idExist { + log.AddContext(ctx).Infof("storage client login response vstoreName is empty, set it to default %s", + defaultVStore) + cli.VStoreName = defaultVStore + } else if exist { + cli.VStoreName = vStoreName + } + + if !idExist { + log.AddContext(ctx).Infof("storage client login response vstoreID is empty, set it to default %s", + defaultVStoreID) + cli.VStoreID = defaultVStoreID + } else { + cli.VStoreID = vStoreID + } + + log.AddContext(ctx).Infof("Login %s success", cli.Url) + return nil +} + +// Logout logout +func (cli *RestClient) Logout(ctx context.Context) { + resp, err := cli.BaseCall(ctx, "DELETE", "/sessions", nil) + if err != nil { + log.AddContext(ctx).Warningf("Logout %s error: %v", cli.Url, err) + return + } + + code := int64(resp.Error["code"].(float64)) + if code != 0 { + log.AddContext(ctx).Warningf("Logout %s error: %d", cli.Url, code) + return + } + + log.AddContext(ctx).Infof("Logout %s success", cli.Url) +} + +// ReLogin logout and login again +func (cli *RestClient) ReLogin(ctx context.Context) error { + oldToken := cli.Token + + cli.ReLoginMutex.Lock() + defer cli.ReLoginMutex.Unlock() + + if cli.Token != "" && oldToken != cli.Token { + // Coming here indicates other thread had already done relogin, so no need to relogin again + return nil + } else if cli.Token != "" { + cli.Logout(ctx) + } + + err := cli.Login(ctx) + if err != nil { + log.AddContext(ctx).Errorf("Try to relogin error: %v", err) + return err + } + + return nil +} + +func (cli *RestClient) getRequestParams(ctx context.Context, backendID string) (map[string]interface{}, error) { + password, err := pkgUtils.GetPasswordFromBackendID(ctx, backendID) + if err != nil { + return nil, err + } + + data := map[string]interface{}{ + "username": cli.User, + "password": password, + "scope": "0", + } + + if len(cli.VStoreName) > 0 && cli.VStoreName != defaultVStore { + data["vstorename"] = cli.VStoreName + } + + return data, err +} + +// SetSystemInfo set system info +// the mutex lock is required for re-login. Therefore, the internal query of the login interface cannot be performed. +func (cli *RestClient) SetSystemInfo(ctx context.Context) error { + log.AddContext(ctx).Infof("set backend [%s] system info is refreshing", cli.BackendID) + atomic.StoreUint32(&cli.SystemInfoRefreshing, 1) + defer func() { + log.AddContext(ctx).Infof("set backend [%s] system info are refreshed", cli.BackendID) + atomic.StoreUint32(&cli.SystemInfoRefreshing, 0) + }() + + err := cli.setBaseInfo(ctx) + if err != nil { + return err + } + cli.setLifInfo(ctx) + + log.AddContext(ctx).Infof("backend type [%s], backend [%s], storage product [%s], storage version [%s], "+ + "current site wwn [%s], current lif [%s], current lif wwn [%s]", + cli.Storage, cli.BackendID, cli.Product, cli.GetStorageVersion(), + cli.CurrentSiteWwn, cli.GetCurrentLif(ctx), cli.CurrentLifWwn) + return nil +} + +func (cli *RestClient) setBaseInfo(ctx context.Context) error { + system, err := cli.GetSystem(ctx) + if err != nil { + log.AddContext(ctx).Errorf("get system info failed, error: %v", err) + return err + } + + cli.Product, err = utils.GetProductVersion(system) + if err != nil { + log.AddContext(ctx).Errorf("get product version failed, error: %v", err) + return err + } + + storagePointVersion, ok := system["pointRelease"].(string) + if ok { + cli.StorageVersion = storagePointVersion + } + + currentSiteWwn, ok := system["wwn"].(string) + if ok { + cli.CurrentSiteWwn = currentSiteWwn + } + + return nil +} + +// GetSystem used for get system info +func (cli *RestClient) GetSystem(ctx context.Context) (map[string]interface{}, error) { + resp, err := cli.Get(ctx, "/system/", nil) + if err != nil { + return nil, err + } + + code := int64(resp.Error["code"].(float64)) + if code != 0 { + msg := fmt.Sprintf("get system info error: %d", code) + return nil, errors.New(msg) + } + + respData, ok := resp.Data.(map[string]interface{}) + if !ok { + return nil, pkgUtils.Errorf(ctx, "convert respData to map failed, data: %v", resp.Data) + } + + return respData, nil +} + +func (cli *RestClient) setLifInfo(ctx context.Context) { + if !cli.Product.IsDoradoV6OrV7() || cli.Storage != constants.OceanStorNas { + log.AddContext(ctx).Infof("backend type [%s], name [%s], version [%s], not need query lif", cli.Storage, + cli.BackendID, cli.GetStorageVersion()) + return + } + + currentLif := cli.GetCurrentLif(ctx) + if cli.LastLif == currentLif { + log.AddContext(ctx).Infof("backend [%s] last lif [%s], current lif [%s], not change", + cli.BackendID, cli.LastLif, currentLif) + return + } + + port, err := cli.GetLogicPort(ctx, currentLif) + if err != nil { + log.AddContext(ctx).Errorf("get logic port failed, error: %v", err) + return + } + + cli.LastLif = currentLif + cli.CurrentLifWwn = port.HomeSiteWwn +} + +// GetBackendID get backend id of client +func (cli *RestClient) GetBackendID() string { + return cli.BackendID +} + +// GetDeviceSN used for get device sn +func (cli *RestClient) GetDeviceSN() string { + return cli.DeviceId +} + +// GetStorageVersion used for get storage version +func (cli *RestClient) GetStorageVersion() string { + return cli.StorageVersion +} + +// GetCurrentSiteWwn used for get current site wwn +func (cli *RestClient) GetCurrentSiteWwn() string { + return cli.CurrentSiteWwn +} + +// GetCurrentLif used for get current lif wwn +func (cli *RestClient) GetCurrentLif(ctx context.Context) string { + u, err := netUrl.Parse(cli.Url) + if err != nil { + log.AddContext(ctx).Errorf("parse url failed, error: %v", err) + return "" + } + return u.Hostname() +} + +// GetLogicPort gets logic port information by port address +func (cli *RestClient) GetLogicPort(ctx context.Context, addr string) (*Lif, error) { + url := fmt.Sprintf("/lif?filter=IPV4ADDR:%s&range=[0-100]", addr) + resp, err := cli.Get(ctx, url, nil) + if err != nil { + return nil, err + } + if err := resp.AssertErrorCode(); err != nil { + return nil, err + } + + var lifs []*Lif + if err := resp.GetData(&lifs); err != nil { + return nil, fmt.Errorf("get logic port error: %w", err) + } + + if len(lifs) == 0 { + // because manage lif is not exist lif list + log.AddContext(ctx).Infof("return lis list not contains [%s], it is considered as the management LIF", addr) + return &Lif{}, nil + } + + return lifs[0], nil +} diff --git a/storage/oceanstor/client/client_test.go b/storage/oceanstorage/oceanstor/client/client_test.go similarity index 99% rename from storage/oceanstor/client/client_test.go rename to storage/oceanstorage/oceanstor/client/client_test.go index 1c87baef..78d82976 100644 --- a/storage/oceanstor/client/client_test.go +++ b/storage/oceanstorage/oceanstor/client/client_test.go @@ -31,14 +31,14 @@ import ( "github.com/prashantv/gostub" "github.com/stretchr/testify/assert" - "huawei-csi-driver/csi/app" - cfg "huawei-csi-driver/csi/app/config" - pkgUtils "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + cfg "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app/config" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) var ( - testClient *BaseClient + testClient *OceanstorClient ) type responseCode int diff --git a/storage/oceanstor/client/client_vstore.go b/storage/oceanstorage/oceanstor/client/client_vstore.go similarity index 82% rename from storage/oceanstor/client/client_vstore.go rename to storage/oceanstorage/oceanstor/client/client_vstore.go index a917b2f9..fdd82a63 100644 --- a/storage/oceanstor/client/client_vstore.go +++ b/storage/oceanstorage/oceanstor/client/client_vstore.go @@ -20,15 +20,15 @@ import ( "context" "fmt" - pkgUtils "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/utils/log" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // VStore defines interfaces for vstore operations type VStore interface { - // GetvStoreName used for get vstore name in *BaseClient + // GetvStoreName used for get vstore name in oceanstor client GetvStoreName() string - // GetvStoreID used for get vstore ID in *BaseClient + // GetvStoreID used for get vstore ID in oceanstor client GetvStoreID() string // GetvStoreByName used for get vstore info by vstore name GetvStoreByName(ctx context.Context, name string) (map[string]interface{}, error) @@ -38,18 +38,18 @@ type VStore interface { GetVStorePairs(ctx context.Context) ([]interface{}, error) } -// GetvStoreName used for get vstore name in *BaseClient -func (cli *BaseClient) GetvStoreName() string { +// GetvStoreName used for get vstore name in oceanstor client +func (cli *OceanstorClient) GetvStoreName() string { return cli.VStoreName } -// GetvStoreID used for get vstore ID in *BaseClient -func (cli *BaseClient) GetvStoreID() string { +// GetvStoreID used for get vstore ID in oceanstor client +func (cli *OceanstorClient) GetvStoreID() string { return cli.VStoreID } // GetvStoreByName used for get vstore info by vstore name -func (cli *BaseClient) GetvStoreByName(ctx context.Context, name string) (map[string]interface{}, error) { +func (cli *OceanstorClient) GetvStoreByName(ctx context.Context, name string) (map[string]interface{}, error) { url := fmt.Sprintf("/vstore?filter=NAME::%s", name) resp, err := cli.Get(ctx, url, nil) if err != nil { @@ -82,7 +82,7 @@ func (cli *BaseClient) GetvStoreByName(ctx context.Context, name string) (map[st } // GetVStorePairs used for get vStore pairs -func (cli *BaseClient) GetVStorePairs(ctx context.Context) ([]interface{}, error) { +func (cli *OceanstorClient) GetVStorePairs(ctx context.Context) ([]interface{}, error) { resp, err := cli.Get(ctx, "/vstore_pair?RETYPE=1", nil) if err != nil { return nil, err @@ -106,7 +106,7 @@ func (cli *BaseClient) GetVStorePairs(ctx context.Context) ([]interface{}, error } // GetvStorePairByID used for get vstore pair by pair id -func (cli *BaseClient) GetvStorePairByID(ctx context.Context, pairID string) (map[string]interface{}, error) { +func (cli *OceanstorClient) GetvStorePairByID(ctx context.Context, pairID string) (map[string]interface{}, error) { url := fmt.Sprintf("/vstore_pair?filter=ID::%s", pairID) resp, err := cli.Get(ctx, url, nil) if err != nil { diff --git a/storage/oceanstor/client/ulrs.go b/storage/oceanstorage/oceanstor/client/ulrs.go similarity index 100% rename from storage/oceanstor/client/ulrs.go rename to storage/oceanstorage/oceanstor/client/ulrs.go diff --git a/storage/oceanstor/clientv6/clientv6.go b/storage/oceanstorage/oceanstor/clientv6/clientv6.go similarity index 69% rename from storage/oceanstor/clientv6/clientv6.go rename to storage/oceanstorage/oceanstor/clientv6/clientv6.go index e650857a..6376b916 100644 --- a/storage/oceanstor/clientv6/clientv6.go +++ b/storage/oceanstorage/oceanstor/clientv6/clientv6.go @@ -19,43 +19,23 @@ package clientv6 import ( "context" "fmt" - "strconv" - "huawei-csi-driver/storage/oceanstor/client" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceanstor/client" ) // V6Client provides base client of clientv6 type V6Client struct { - client.BaseClient + client.OceanstorClient } // NewClientV6 inits a new client of clientv6 func NewClientV6(ctx context.Context, param *client.NewClientConfig) (*V6Client, error) { - var err error - var parallelCount int - - if len(param.ParallelNum) > 0 { - parallelCount, err = strconv.Atoi(param.ParallelNum) - if err != nil || parallelCount > client.MaxParallelCount || parallelCount < client.MinParallelCount { - log.Warningf("The config parallelNum %d is invalid, set it to the default value %d", - parallelCount, client.DefaultParallelCount) - parallelCount = client.DefaultParallelCount - } - } else { - parallelCount = client.DefaultParallelCount - } - - log.AddContext(ctx).Infof("Init parallel count is %d", parallelCount) - client.RequestSemaphore = utils.NewSemaphore(parallelCount) - cli, err := client.NewClient(ctx, param) if err != nil { return nil, err } - return &V6Client{BaseClient: *cli}, nil + return &V6Client{OceanstorClient: *cli}, nil } // SplitCloneFS used to split clone for dorado or oceantor v6 diff --git a/storage/oceanstor/clientv6/clientv6_test.go b/storage/oceanstorage/oceanstor/clientv6/clientv6_test.go similarity index 88% rename from storage/oceanstor/clientv6/clientv6_test.go rename to storage/oceanstorage/oceanstor/clientv6/clientv6_test.go index d18961a4..b246ed76 100644 --- a/storage/oceanstor/clientv6/clientv6_test.go +++ b/storage/oceanstorage/oceanstor/clientv6/clientv6_test.go @@ -26,8 +26,8 @@ import ( "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" - "huawei-csi-driver/storage/oceanstor/client" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceanstor/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) var testClient *V6Client @@ -61,9 +61,9 @@ func TestSplitCloneFS(t *testing.T) { mockBaseClient := client.NewMockHTTPClient(ctrl) - temp := testClient.BaseClient.Client - defer func() { testClient.BaseClient.Client = temp }() - testClient.BaseClient.Client = mockBaseClient + temp := testClient.OceanstorClient.Client + defer func() { testClient.OceanstorClient.Client = temp }() + testClient.OceanstorClient.Client = mockBaseClient for _, c := range cases { r := ioutil.NopCloser(bytes.NewReader([]byte(c.responseBody))) diff --git a/storage/oceanstor/clientv6/urls.go b/storage/oceanstorage/oceanstor/clientv6/urls.go similarity index 100% rename from storage/oceanstor/clientv6/urls.go rename to storage/oceanstorage/oceanstor/clientv6/urls.go diff --git a/storage/oceanstor/smartx/smartx.go b/storage/oceanstorage/oceanstor/smartx/smartx.go similarity index 88% rename from storage/oceanstor/smartx/smartx.go rename to storage/oceanstorage/oceanstor/smartx/smartx.go index 256e23d8..79235118 100644 --- a/storage/oceanstor/smartx/smartx.go +++ b/storage/oceanstorage/oceanstor/smartx/smartx.go @@ -26,10 +26,11 @@ import ( "strings" "time" - "huawei-csi-driver/pkg/constants" - "huawei-csi-driver/storage/oceanstor/client" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/base" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceanstor/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( @@ -47,8 +48,9 @@ type qosParameterValidators map[string]func(int) bool type qosParameterList map[string]struct{} var ( - oceanStorQosValidators = map[string]qosParameterValidators{ - constants.OceanStorDoradoV6: doradoV6ParameterValidators, + oceanStorQosValidators = map[constants.OceanstorVersion]qosParameterValidators{ + constants.OceanStorDoradoV7: doradoV6V7ParameterValidators, + constants.OceanStorDoradoV6: doradoV6V7ParameterValidators, constants.OceanStorDoradoV3: doradoParameterValidators, constants.OceanStorV3: oceanStorV3V5ParameterValidators, constants.OceanStorV5: oceanStorV3V5ParameterValidators, @@ -87,7 +89,7 @@ var ( }, } - doradoV6ParameterValidators = map[string]func(int) bool{ + doradoV6V7ParameterValidators = map[string]func(int) bool{ "IOTYPE": func(value int) bool { return value == ioType }, @@ -123,7 +125,8 @@ var ( } // one of parameter is mandatory for respective products - oceanStorQoSMandatoryParameters = map[string]qosParameterList{ + oceanStorQoSMandatoryParameters = map[constants.OceanstorVersion]qosParameterList{ + constants.OceanStorDoradoV7: oceanStorCommonParameters, constants.OceanStorDoradoV6: oceanStorCommonParameters, constants.OceanStorDoradoV3: { "MAXBANDWIDTH": struct{}{}, @@ -135,7 +138,7 @@ var ( ) // CheckQoSParameterSupport verify QoS supported parameters and value validation -func CheckQoSParameterSupport(ctx context.Context, product, qosConfig string) error { +func CheckQoSParameterSupport(ctx context.Context, product constants.OceanstorVersion, qosConfig string) error { qosParam, err := ExtractQoSParameters(ctx, product, qosConfig) if err != nil { return err @@ -149,7 +152,8 @@ func CheckQoSParameterSupport(ctx context.Context, product, qosConfig string) er return nil } -func validateQoSParametersSupport(ctx context.Context, product string, qosParam map[string]float64) error { +func validateQoSParametersSupport(ctx context.Context, + product constants.OceanstorVersion, qosParam map[string]float64) error { var lowerLimit, upperLimit bool // decide validators based on product @@ -176,7 +180,7 @@ func validateQoSParametersSupport(ctx context.Context, product string, qosParam } } - if product != constants.OceanStorDoradoV6 && lowerLimit && upperLimit { + if !product.IsDoradoV6OrV7() && lowerLimit && upperLimit { return utils.Errorf(ctx, "Cannot specify both lower and upper limits in qos for OceanStor %s", product) } @@ -184,7 +188,8 @@ func validateQoSParametersSupport(ctx context.Context, product string, qosParam } // ExtractQoSParameters unmarshal QoS configuration parameters -func ExtractQoSParameters(ctx context.Context, product string, qosConfig string) (map[string]float64, error) { +func ExtractQoSParameters(ctx context.Context, + product constants.OceanstorVersion, qosConfig string) (map[string]float64, error) { var unmarshalParams map[string]interface{} params := make(map[string]float64) @@ -203,7 +208,7 @@ func ExtractQoSParameters(ctx context.Context, product string, qosConfig string) key, val, product) } - if product == constants.OceanStorDoradoV6 && key == "LATENCY" { + if product.IsDoradoV6OrV7() && key == "LATENCY" { // convert OceanStoreDoradoV6 Latency from millisecond to microsecond params[key] = value * kilo continue @@ -216,7 +221,7 @@ func ExtractQoSParameters(ctx context.Context, product string, qosConfig string) } // ValidateQoSParameters check QoS parameters -func ValidateQoSParameters(product string, qosParam map[string]float64) (map[string]int, error) { +func ValidateQoSParameters(product constants.OceanstorVersion, qosParam map[string]float64) (map[string]int, error) { // ensure at least one parameter params := oceanStorQoSMandatoryParameters[product] paramExist := false @@ -250,11 +255,11 @@ func ValidateQoSParameters(product string, qosParam map[string]float64) (map[str // Client provides smartx client type Client struct { - cli client.BaseClientInterface + cli client.OceanstorClientInterface } // NewSmartX inits a new smartx client -func NewSmartX(cli client.BaseClientInterface) *Client { +func NewSmartX(cli client.OceanstorClientInterface) *Client { return &Client{ cli: cli, } @@ -453,8 +458,8 @@ func (p *Client) DeleteFSSnapshot(ctx context.Context, snapshotID string) error return nil } -func (p *Client) getCreateQosArgs(name, objID, objType, vStoreID string, params map[string]int) client.CreateQoSArgs { - return client.CreateQoSArgs{ +func (p *Client) getCreateQosArgs(name, objID, objType, vStoreID string, params map[string]int) base.CreateQoSArgs { + return base.CreateQoSArgs{ Name: name, ObjID: objID, ObjType: objType, diff --git a/storage/oceanstor/volume/base.go b/storage/oceanstorage/oceanstor/volume/base.go similarity index 86% rename from storage/oceanstor/volume/base.go rename to storage/oceanstorage/oceanstor/volume/base.go index f380be51..7bde3ec9 100644 --- a/storage/oceanstor/volume/base.go +++ b/storage/oceanstorage/oceanstor/volume/base.go @@ -22,20 +22,20 @@ import ( "fmt" "strconv" - "huawei-csi-driver/pkg/constants" - pkgUtils "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/storage/oceanstor/client" - "huawei-csi-driver/storage/oceanstor/smartx" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceanstor/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceanstor/smartx" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // Base defines the base storage client type Base struct { - cli client.BaseClientInterface - metroRemoteCli client.BaseClientInterface - replicaRemoteCli client.BaseClientInterface - product string + cli client.OceanstorClientInterface + metroRemoteCli client.OceanstorClientInterface + replicaRemoteCli client.OceanstorClientInterface + product constants.OceanstorVersion } func (p *Base) commonPreModify(ctx context.Context, params map[string]interface{}) error { @@ -161,7 +161,7 @@ func (p *Base) getQoS(ctx context.Context, params map[string]interface{}) error } func (p *Base) getRemotePoolID(ctx context.Context, - params map[string]interface{}, remoteCli client.BaseClientInterface) (string, error) { + params map[string]interface{}, remoteCli client.OceanstorClientInterface) (string, error) { remotePool, exist := params["remotestoragepool"].(string) if !exist || len(remotePool) == 0 { msg := "no remote pool is specified" @@ -233,7 +233,7 @@ func (p *Base) getRemoteDeviceID(ctx context.Context, deviceSN string) (string, } func (p *Base) getWorkLoadIDByName(ctx context.Context, - cli client.BaseClientInterface, + cli client.OceanstorClientInterface, workloadTypeName string) (string, error) { workloadTypeID, err := cli.GetApplicationTypeByName(ctx, workloadTypeName) if err != nil { @@ -248,7 +248,8 @@ func (p *Base) getWorkLoadIDByName(ctx context.Context, return workloadTypeID, nil } -func (p *Base) setWorkLoadID(ctx context.Context, cli client.BaseClientInterface, params map[string]interface{}) error { +func (p *Base) setWorkLoadID(ctx context.Context, + cli client.OceanstorClientInterface, params map[string]interface{}) error { if val, ok := params["applicationtype"].(string); ok { workloadTypeID, err := p.getWorkLoadIDByName(ctx, cli, val) if err != nil { @@ -259,19 +260,22 @@ func (p *Base) setWorkLoadID(ctx context.Context, cli client.BaseClientInterface return nil } -func (p *Base) prepareVolObj(ctx context.Context, params, res map[string]interface{}) utils.Volume { - volName, isStr := params["name"].(string) - if !isStr { - // Not expecting this error to happen - log.AddContext(ctx).Warningf("Expecting string for volume name, received type %T", params["name"]) +func (p *Base) prepareVolObj(ctx context.Context, params, res map[string]interface{}) (utils.Volume, error) { + volName, ok := params["name"].(string) + if !ok { + return nil, utils.Errorf(ctx, "expecting string for volume name, received type %T", params["name"]) } + volObj := utils.NewVolume(volName) if res != nil { if lunWWN, ok := res["lunWWN"].(string); ok { volObj.SetLunWWN(lunWWN) } } - return volObj + + capacity := utils.GetValueOrFallback(params, "capacity", int64(0)) + volObj.SetSize(utils.TransK8SCapacity(capacity, constants.AllocationUnitBytes)) + return volObj, nil } func (p *Base) getMetroPairSyncSpeed(_ context.Context, params map[string]interface{}) error { diff --git a/storage/oceanstor/volume/codes.go b/storage/oceanstorage/oceanstor/volume/codes.go similarity index 100% rename from storage/oceanstor/volume/codes.go rename to storage/oceanstorage/oceanstor/volume/codes.go diff --git a/storage/oceanstor/volume/creator/base.go b/storage/oceanstorage/oceanstor/volume/creator/base.go similarity index 91% rename from storage/oceanstor/volume/creator/base.go rename to storage/oceanstorage/oceanstor/volume/creator/base.go index 34320048..08f22449 100644 --- a/storage/oceanstor/volume/creator/base.go +++ b/storage/oceanstorage/oceanstor/volume/creator/base.go @@ -14,6 +14,7 @@ * limitations under the License. */ +// Package creator provides creator of volume package creator import ( @@ -23,11 +24,11 @@ import ( "strings" "time" - "huawei-csi-driver/storage/oceanstor/client" - "huawei-csi-driver/storage/oceanstor/smartx" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/flow" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceanstor/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceanstor/smartx" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/flow" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( @@ -37,7 +38,7 @@ const ( // BaseCreator provides some common methods for volume creation. type BaseCreator struct { - cli client.BaseClientInterface + cli client.OceanstorClientInterface transaction *flow.Transaction // common fields of the filesystem @@ -103,11 +104,11 @@ func (c *BaseCreator) Init(params *Parameter) { c.snapshotReservePer = &val } - if params.Product().IsV6() && params.IsHyperMetro() { + if params.Product().IsDoradoV6OrV7() && params.IsHyperMetro() { c.vStoreId = c.cli.GetvStoreID() } - if params.Product().IsV6() { + if params.Product().IsDoradoV6OrV7() { c.isSyncPair = false c.isPairOnlineDelete = false } else { @@ -126,11 +127,11 @@ func (c *BaseCreator) GetPoolID(ctx context.Context, storagePoolName string) (st if err != nil { return "", fmt.Errorf("get storage pool %s info error: %w", storagePoolName, err) } - if pool == nil || getValueOrFallback(pool, "ID", "") == "" { + if pool == nil || utils.GetValueOrFallback(pool, "ID", "") == "" { return "", fmt.Errorf("storage pool %s doesn't exist", storagePoolName) } - return getValueOrFallback(pool, "ID", ""), nil + return utils.GetValueOrFallback(pool, "ID", ""), nil } // CreateNfsShare creates nfs share for the filesystem. @@ -146,7 +147,7 @@ func (c *BaseCreator) CreateNfsShare(ctx context.Context, fsName, fsId, desc, vS } if share != nil { - return getValueOrFallback(share, "ID", ""), nil + return utils.GetValueOrFallback(share, "ID", ""), nil } req := map[string]any{ @@ -161,7 +162,7 @@ func (c *BaseCreator) CreateNfsShare(ctx context.Context, fsName, fsId, desc, vS return "", fmt.Errorf("create nfs share %v error: %w", req, err) } - return getValueOrFallback(share, "ID", ""), nil + return utils.GetValueOrFallback(share, "ID", ""), nil } // RollbackShare rollbacks nfs share resource. @@ -197,7 +198,7 @@ func (c *BaseCreator) AllowShareAccess(ctx context.Context, params ShareAccessPa } removeShareAccess := func(authClient map[string]any) { - accessId := getValueOrFallback(authClient, "ID", "") + accessId := utils.GetValueOrFallback(authClient, "ID", "") if accessId == "" { return } @@ -222,12 +223,12 @@ func (c *BaseCreator) RollbackShareAccess(ctx context.Context, shareId, vStoreId removeShareAccessInAuthClient := func(authClientMap map[string]any) { authClients := strings.Split(authClient, ";") - authClientName := getValueOrFallback(authClientMap, "NAME", "") + authClientName := utils.GetValueOrFallback(authClientMap, "NAME", "") if !utils.Contains(authClients, authClientName) { return } - accessId := getValueOrFallback(authClientMap, "ID", "") + accessId := utils.GetValueOrFallback(authClientMap, "ID", "") if accessId == "" { return } @@ -418,7 +419,7 @@ func (c *BaseCreator) createHyperMetroPair(ctx context.Context, // There is no need to synchronize when use NAS Dorado V6 or OceanStor V6 HyperMetro Volume if c.isSyncPair { - pairId := getValueOrFallback(pair, "ID", "") + pairId := utils.GetValueOrFallback(pair, "ID", "") err = c.cli.SyncHyperMetroPair(ctx, pairId) if err != nil { log.AddContext(ctx).Errorf("Sync nas hypermetro pair %s error: %v", pairId, err) @@ -430,7 +431,7 @@ func (c *BaseCreator) createHyperMetroPair(ctx context.Context, } } - return getValueOrFallback(pair, "ID", ""), nil + return utils.GetValueOrFallback(pair, "ID", ""), nil } func (c *BaseCreator) rollbackHyperMetroPair(ctx context.Context, pairId string) error { @@ -441,7 +442,7 @@ func (c *BaseCreator) rollbackHyperMetroPair(ctx context.Context, pairId string) if pair == nil { return nil } - pairStatus := getValueOrFallback(pair, "RUNNINGSTATUS", "") + pairStatus := utils.GetValueOrFallback(pair, "RUNNINGSTATUS", "") if utils.Contains(pairRationalStatus, pairStatus) { if err := c.cli.StopHyperMetroPair(ctx, pairId); err != nil { return err diff --git a/storage/oceanstor/volume/creator/clone_fs.go b/storage/oceanstorage/oceanstor/volume/creator/clone_fs.go similarity index 89% rename from storage/oceanstor/volume/creator/clone_fs.go rename to storage/oceanstorage/oceanstor/volume/creator/clone_fs.go index c74a20e1..36236364 100644 --- a/storage/oceanstor/volume/creator/clone_fs.go +++ b/storage/oceanstorage/oceanstor/volume/creator/clone_fs.go @@ -22,10 +22,11 @@ import ( "strconv" "time" - pkgUtils "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/storage/oceanstor/client" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceanstor/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( @@ -57,6 +58,8 @@ type CloneFsCreator struct { cloneSpeed int parentSnapshotId string isDeleteParentSnapshot bool + + createdFilesystem map[string]any } // WithParentSnapshotId sets parentSnapshotId field of CloneFsCreator @@ -81,7 +84,7 @@ func WithCloneFrom(cloneFrom string) CloneFsOptionFunc { } // NewCloneFsCreatorByParams returns an instance of CloneFsCreator -func NewCloneFsCreatorByParams(cli client.BaseClientInterface, +func NewCloneFsCreatorByParams(cli client.OceanstorClientInterface, params *Parameter, opts ...CloneFsOptionFunc) *CloneFsCreator { base := &BaseCreator{cli: cli} base.Init(params) @@ -110,11 +113,11 @@ func (creator *CloneFsCreator) CreateVolume(ctx context.Context) (utils.Volume, } if fs != nil { - if err := creator.waitFsSplit(ctx, getValueOrFallback(fs, "ID", "")); err != nil { + if err := creator.waitFsSplit(ctx, utils.GetValueOrFallback(fs, "ID", "")); err != nil { return nil, err } - volume.SetID(getValueOrFallback(fs, "ID", "")) + volume.SetID(utils.GetValueOrFallback(fs, "ID", "")) return volume, nil } @@ -124,6 +127,8 @@ func (creator *CloneFsCreator) CreateVolume(ctx context.Context) (utils.Volume, } volume.SetID(fsId) + volume.SetSize(utils.TransK8SCapacity(creator.capacity, constants.AllocationUnitBytes)) + return volume, nil } @@ -131,6 +136,10 @@ func (creator *CloneFsCreator) rollback(ctx context.Context) { creator.transaction.Rollback() } +func (creator *CloneFsCreator) getCreatedFilesystem() map[string]any { + return creator.createdFilesystem +} + func (creator *CloneFsCreator) createResources(ctx context.Context) (string, error) { var fsId string var err error @@ -180,7 +189,7 @@ func (creator *CloneFsCreator) clone(ctx context.Context) (string, error) { } srcFsCapacity, err := strconv.ParseInt( - getValueOrFallback(cloneFromFS, "CAPACITY", ""), + utils.GetValueOrFallback(cloneFromFS, "CAPACITY", ""), capacityBase, capacityBitSize, ) @@ -195,7 +204,7 @@ func (creator *CloneFsCreator) clone(ctx context.Context) (string, error) { cloneFilesystemReq := &cloneFilesystemRequest{ fsName: creator.fsName, - parentID: getValueOrFallback(cloneFromFS, "ID", ""), + parentID: utils.GetValueOrFallback(cloneFromFS, "ID", ""), parentSnapshotID: creator.parentSnapshotId, allocType: creator.allocType, cloneSpeed: creator.cloneSpeed, @@ -226,7 +235,7 @@ func (creator *CloneFsCreator) cloneFilesystem(ctx context.Context, req *cloneFi req.parentID, err) return "", err } - cloneFSID := getValueOrFallback(cloneFS, "ID", "") + cloneFSID := utils.GetValueOrFallback(cloneFS, "ID", "") if req.cloneFsCapacity > req.srcCapacity { err := creator.cli.ExtendFileSystem(ctx, cloneFSID, req.cloneFsCapacity) if err != nil { @@ -236,13 +245,15 @@ func (creator *CloneFsCreator) cloneFilesystem(ctx context.Context, req *cloneFi return "", err } } - req.vStoreId = getValueOrFallback(cloneFS, "vstoreId", systemVStore) + req.vStoreId = utils.GetValueOrFallback(cloneFS, "vstoreId", systemVStore) err = creator.splitClone(ctx, cloneFSID, req) if err != nil { log.AddContext(ctx).Errorf("split clone failed. err: %v", err) } + creator.createdFilesystem = cloneFS + return cloneFSID, nil } diff --git a/storage/oceanstor/volume/creator/creator.go b/storage/oceanstorage/oceanstor/volume/creator/creator.go similarity index 74% rename from storage/oceanstor/volume/creator/creator.go rename to storage/oceanstorage/oceanstor/volume/creator/creator.go index 4c50220a..83d71202 100644 --- a/storage/oceanstor/volume/creator/creator.go +++ b/storage/oceanstorage/oceanstor/volume/creator/creator.go @@ -20,9 +20,9 @@ import ( "context" "errors" - "huawei-csi-driver/storage/oceanstor/client" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceanstor/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) var ( @@ -32,18 +32,30 @@ var ( ErrNotFoundCli = errors.New("not found client") ) +// StandbyVolumeCreator is the interface that defines methods for standby volume creator. +type StandbyVolumeCreator interface { + SingleVolumeCreator + setStandbyParameters(map[string]any) +} + +// SingleVolumeCreator is the interface that defines methods for single volume creator. +type SingleVolumeCreator interface { + VolumeCreator + rollback(context.Context) + getCreatedFilesystem() map[string]any +} + // VolumeCreator is the interface that wraps the CreateVolume method. type VolumeCreator interface { CreateVolume(context.Context) (utils.Volume, error) - rollback(context.Context) } // NewFromParameters generates a volume creator instance from parameters. func NewFromParameters( ctx context.Context, parameters map[string]any, - activeCli client.BaseClientInterface, - standbyCli client.BaseClientInterface, + activeCli client.OceanstorClientInterface, + standbyCli client.OceanstorClientInterface, ) (VolumeCreator, error) { params := NewParameter(parameters) @@ -63,8 +75,8 @@ func NewFromParameters( func validateParameters( ctx context.Context, params *Parameter, - activeCli client.BaseClientInterface, - standbyCli client.BaseClientInterface, + activeCli client.OceanstorClientInterface, + standbyCli client.OceanstorClientInterface, ) error { if activeCli == nil { log.AddContext(ctx).Errorln(ErrNotFoundCli) @@ -83,7 +95,7 @@ func validateParameters( return nil } -func newSingle(params *Parameter, cli client.BaseClientInterface) VolumeCreator { +func newSingle(params *Parameter, cli client.OceanstorClientInterface) SingleVolumeCreator { if params.IsClone() { return NewCloneFsCreatorByParams(cli, params) } else if params.IsSnapshot() { diff --git a/storage/oceanstor/volume/creator/filesystem.go b/storage/oceanstorage/oceanstor/volume/creator/filesystem.go similarity index 77% rename from storage/oceanstor/volume/creator/filesystem.go rename to storage/oceanstorage/oceanstor/volume/creator/filesystem.go index 9f182006..4bd63c43 100644 --- a/storage/oceanstor/volume/creator/filesystem.go +++ b/storage/oceanstorage/oceanstor/volume/creator/filesystem.go @@ -21,9 +21,10 @@ import ( "fmt" "strconv" - "huawei-csi-driver/storage/oceanstor/client" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceanstor/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // FsOptionFunc defines the function to change fields of FilesystemCreator @@ -38,19 +39,24 @@ type FilesystemCreator struct { fileSystemMode string unixPermissions string workloadTypeID string + + createdFilesystem map[string]any + standbyRequest map[string]any } // NewFsCreatorFromParams returns an instance of FilesystemCreator -func NewFsCreatorFromParams(cli client.BaseClientInterface, +func NewFsCreatorFromParams(cli client.OceanstorClientInterface, params *Parameter, opts ...FsOptionFunc) *FilesystemCreator { base := &BaseCreator{cli: cli} base.Init(params) creator := &FilesystemCreator{ - BaseCreator: base, - fileSystemMode: client.LocalFilesystemMode, - unixPermissions: params.FsPermission(), - workloadTypeID: params.WorkloadTypeID(), + BaseCreator: base, + fileSystemMode: client.LocalFilesystemMode, + unixPermissions: params.FsPermission(), + workloadTypeID: params.WorkloadTypeID(), + createdFilesystem: make(map[string]any), + standbyRequest: make(map[string]any), } if params.IsHyperMetro() { @@ -79,6 +85,7 @@ func (creator *FilesystemCreator) CreateVolume(ctx context.Context) (utils.Volum return nil, err } volume.SetID(fsId) + volume.SetSize(utils.TransK8SCapacity(creator.capacity, constants.AllocationUnitBytes)) return volume, nil } @@ -87,6 +94,20 @@ func (creator *FilesystemCreator) rollback(ctx context.Context) { creator.transaction.Rollback() } +func (creator *FilesystemCreator) getCreatedFilesystem() map[string]any { + return creator.createdFilesystem +} + +var standbySyncFields = []string{"ENABLEDEDUP", "ENABLECOMPRESSION"} + +func (creator *FilesystemCreator) setStandbyParameters(req map[string]any) { + for _, field := range standbySyncFields { + if value, ok := req[field]; ok { + creator.standbyRequest[field] = value + } + } +} + func (creator *FilesystemCreator) createResources(ctx context.Context) (string, error) { var fsId string var err error @@ -128,7 +149,7 @@ func (creator *FilesystemCreator) createFilesystem(ctx context.Context) (string, } if fs != nil { - return getValueOrFallback(fs, "ID", ""), nil + return utils.GetValueOrFallback(fs, "ID", ""), nil } req, err := creator.genCreateRequest(ctx, poolId) @@ -142,7 +163,9 @@ func (creator *FilesystemCreator) createFilesystem(ctx context.Context) (string, return "", fmt.Errorf("create filesystem %s error: %w", creator.fsName, err) } - return getValueOrFallback(fs, "ID", ""), nil + creator.createdFilesystem = fs + + return utils.GetValueOrFallback(fs, "ID", ""), nil } func (creator *FilesystemCreator) genCreateRequest(ctx context.Context, poolId string) (map[string]any, error) { @@ -155,6 +178,12 @@ func (creator *FilesystemCreator) genCreateRequest(ctx context.Context, poolId s "fileSystemMode": creator.fileSystemMode, } + if len(creator.standbyRequest) != 0 { + for k, v := range creator.standbyRequest { + req[k] = v + } + } + if creator.fileSystemMode == client.HyperMetroFilesystemMode { req["vstoreId"] = creator.vStoreId } diff --git a/storage/oceanstor/volume/creator/hypermetro_fs.go b/storage/oceanstorage/oceanstor/volume/creator/hypermetro_fs.go similarity index 88% rename from storage/oceanstor/volume/creator/hypermetro_fs.go rename to storage/oceanstorage/oceanstor/volume/creator/hypermetro_fs.go index 20510098..8e0664f9 100644 --- a/storage/oceanstor/volume/creator/hypermetro_fs.go +++ b/storage/oceanstorage/oceanstor/volume/creator/hypermetro_fs.go @@ -20,9 +20,10 @@ import ( "context" "fmt" - "huawei-csi-driver/storage/oceanstor/client" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceanstor/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( @@ -49,8 +50,8 @@ type HyperMetroFsOptionFunc func(creator *HyperMetroFsCreator) // HyperMetroFsRequiredOptions defines the required options of ModifyFsCreator type HyperMetroFsRequiredOptions struct { - activeCli client.BaseClientInterface - standbyCli client.BaseClientInterface + activeCli client.OceanstorClientInterface + standbyCli client.OceanstorClientInterface activeCreator VolumeCreator standbyCreator VolumeCreator @@ -71,14 +72,14 @@ type HyperMetroFsRequiredOptions struct { // HyperMetroFsCreator is the filesystem creator that implement VolumeCreator interface. type HyperMetroFsCreator struct { *BaseCreator - active VolumeCreator - standby VolumeCreator + active SingleVolumeCreator + standby StandbyVolumeCreator } // NewHyperMetroCreatorFromParams returns an instance of HyperMetroFsCreator func NewHyperMetroCreatorFromParams( - activeCli client.BaseClientInterface, - standbyCli client.BaseClientInterface, + activeCli client.OceanstorClientInterface, + standbyCli client.OceanstorClientInterface, params *Parameter, opts ...HyperMetroFsOptionFunc, ) *HyperMetroFsCreator { @@ -128,6 +129,7 @@ func (creator *HyperMetroFsCreator) CreateVolume(ctx context.Context) (utils.Vol }) creator.transaction.Then(func() error { var err error + creator.standby.setStandbyParameters(creator.active.getCreatedFilesystem()) standbyFs, err = creator.standby.CreateVolume(ctx) if err != nil { return err @@ -159,6 +161,7 @@ func (creator *HyperMetroFsCreator) CreateVolume(ctx context.Context) (utils.Vol } volume := utils.NewVolume(activeFs.GetVolumeName()) + volume.SetSize(utils.TransK8SCapacity(creator.capacity, constants.AllocationUnitBytes)) return volume, nil } diff --git a/storage/oceanstor/volume/creator/modify_fs.go b/storage/oceanstorage/oceanstor/volume/creator/modify_fs.go similarity index 63% rename from storage/oceanstor/volume/creator/modify_fs.go rename to storage/oceanstorage/oceanstor/volume/creator/modify_fs.go index 013920ba..46896379 100644 --- a/storage/oceanstor/volume/creator/modify_fs.go +++ b/storage/oceanstorage/oceanstor/volume/creator/modify_fs.go @@ -21,9 +21,9 @@ import ( "errors" "fmt" - "huawei-csi-driver/storage/oceanstor/client" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceanstor/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) var _ VolumeCreator = (*ModifyFsCreator)(nil) @@ -33,8 +33,8 @@ type ModifyFsOptionFunc func(creator *ModifyFsCreator) // NewModifyCreatorFromParams returns an instance of ModifyFsCreator func NewModifyCreatorFromParams( - activeCli client.BaseClientInterface, - standbyCli client.BaseClientInterface, + activeCli client.OceanstorClientInterface, + standbyCli client.OceanstorClientInterface, params *Parameter, opts ...ModifyFsOptionFunc, ) *ModifyFsCreator { @@ -58,8 +58,8 @@ func NewModifyCreatorFromParams( // ModifyFsCreator is the filesystem creator that implement VolumeCreator interface. type ModifyFsCreator struct { *BaseCreator - activeCli client.BaseClientInterface - standbyCli client.BaseClientInterface + activeCli client.OceanstorClientInterface + standbyCli client.OceanstorClientInterface params *Parameter hyperMetro bool @@ -77,44 +77,10 @@ func (creator *ModifyFsCreator) CreateVolume(ctx context.Context) (utils.Volume, return nil, fmt.Errorf("file system not exists on active site during modify volume: %s", creator.fsName) } - volume := utils.NewVolume(creator.fsName) - if creator.hyperMetro { - // modify local filesystem to hyper metro filesystem - pairId, err := creator.getPairIdByFsId(ctx, activeFs) - if err != nil { - return nil, fmt.Errorf("failed to get pair id of active hypermetro pair, error: %v", err) - } - if pairId != "" { - log.AddContext(ctx).Infof("hypermetro pair already exists, pair id: %s", pairId) - return volume, nil - } - - standbyCreator, err := creator.newStandbyCreatorForModify(ctx, activeFs, err) - if err != nil { + if err = creator.modifyHyperMetro(ctx, activeFs); err != nil { return nil, err } - - var standbyFsId string - creator.transaction.Then(func() error { - standbyFsId, err = standbyCreator.createFilesystem(ctx) - return err - }, func() { - req := map[string]any{"ID": standbyFsId} - if creator.vStoreId != "" { - req["vstoreId"] = creator.vStoreId - } - if err := standbyCreator.cli.SafeDeleteFileSystem(ctx, req); err != nil { - log.AddContext(ctx).Errorf("delete filesystem %s error: %v", creator.fsName, err) - } - }).Then(func() error { - pairId, err = creator.createHyperMetroPair(ctx, getValueOrFallback(activeFs, "ID", ""), standbyFsId) - return err - }, func() { - if err := creator.rollbackHyperMetroPair(ctx, pairId); err != nil { - log.AddContext(ctx).Errorf("failed to rollback hypermetro pair %s, error: %v", pairId, err) - } - }) } err = creator.transaction.Commit() @@ -123,16 +89,58 @@ func (creator *ModifyFsCreator) CreateVolume(ctx context.Context) (utils.Volume, return nil, err } + volume := utils.NewVolume(creator.fsName) return volume, nil } +func (creator *ModifyFsCreator) modifyHyperMetro(ctx context.Context, activeFs map[string]any) error { + // modify local filesystem to hyper metro filesystem + pairId, err := creator.getPairIdByFsId(ctx, activeFs) + if err != nil { + return fmt.Errorf("failed to get pair id of active hypermetro pair, error: %v", err) + } + if pairId != "" { + log.AddContext(ctx).Infof("hypermetro pair already exists, pair id: %s", pairId) + return nil + } + + standbyCreator, err := creator.newStandbyCreatorForModify(ctx, activeFs, err) + if err != nil { + return err + } + + var standbyFsId string + creator.transaction.Then(func() error { + standbyCreator.setStandbyParameters(activeFs) + standbyFsId, err = standbyCreator.createFilesystem(ctx) + return err + }, func() { + req := map[string]any{"ID": standbyFsId} + if creator.vStoreId != "" { + req["vstoreId"] = creator.vStoreId + } + if err := standbyCreator.cli.SafeDeleteFileSystem(ctx, req); err != nil { + log.AddContext(ctx).Errorf("delete filesystem %s error: %v", creator.fsName, err) + } + }).Then(func() error { + pairId, err = creator.createHyperMetroPair(ctx, utils.GetValueOrFallback(activeFs, "ID", ""), standbyFsId) + return err + }, func() { + if err := creator.rollbackHyperMetroPair(ctx, pairId); err != nil { + log.AddContext(ctx).Errorf("failed to rollback hypermetro pair %s, error: %v", pairId, err) + } + }) + + return nil +} + func (creator *ModifyFsCreator) rollback(ctx context.Context) { creator.transaction.Rollback() } func (creator *ModifyFsCreator) newStandbyCreatorForModify(ctx context.Context, activeFs map[string]interface{}, err error) (*FilesystemCreator, error) { - poolName := getValueOrFallback(activeFs, "PARENTNAME", "") + poolName := utils.GetValueOrFallback(activeFs, "PARENTNAME", "") if poolName == "" { return nil, errors.New("pool name cannot be empty") } @@ -148,7 +156,7 @@ func (creator *ModifyFsCreator) newStandbyCreatorForModify(ctx context.Context, standbyCreator := NewFsCreatorFromParams(creator.standbyCli, creator.params) standbyCreator.storagePoolName = creator.params.RemoteStoragePool() - standbyCreator.description = getValueOrFallback(activeFs, "DESCRIPTION", "") + standbyCreator.description = utils.GetValueOrFallback(activeFs, "DESCRIPTION", "") standbyCreator.capacity = int64(capacity) return standbyCreator, nil } diff --git a/storage/oceanstor/volume/creator/parameter.go b/storage/oceanstorage/oceanstor/volume/creator/parameter.go similarity index 68% rename from storage/oceanstor/volume/creator/parameter.go rename to storage/oceanstorage/oceanstor/volume/creator/parameter.go index 346b9f6e..5b58281c 100644 --- a/storage/oceanstor/volume/creator/parameter.go +++ b/storage/oceanstorage/oceanstor/volume/creator/parameter.go @@ -16,36 +16,48 @@ package creator +import ( + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" +) + const ( - AccessKrb5ReadOnly = "read_only" + // AccessKrb5ReadOnly defines ReadOnly AccessKrb5 access level + AccessKrb5ReadOnly = "read_only" + + // AccessKrb5ReadWrite defines ReadWrite AccessKrb5 access level AccessKrb5ReadWrite = "read_write" - AccessKrb5ReadNone = "none" - doradoV6 = StorageProduct("DoradoV6") -) + // AccessKrb5ReadNone defines None AccessKrb5 access level + AccessKrb5ReadNone = "none" -type StorageProduct string + // AccessKrb5ReadOnlyInt defines ReadOnly AccessKrb5 access level of int value + AccessKrb5ReadOnlyInt = 0 -func (s StorageProduct) IsV6() bool { - if s == doradoV6 { - return true - } + // AccessKrb5ReadWriteInt defines ReadWrite AccessKrb5 access level of int value + AccessKrb5ReadWriteInt = 1 - return false -} + // AccessKrb5ReadNoneInt defines None AccessKrb5 access level of int value + AccessKrb5ReadNoneInt = 5 + + // AccessKrb5UndefinedInt is undefined int value + AccessKrb5UndefinedInt = -1 +) +// AccessKrb is a type alias for a string that represents access levels to Kerberos. type AccessKrb string +// Int converts the access level to an integer. func (a AccessKrb) Int() int { switch a { case AccessKrb5ReadOnly: - return 0 + return AccessKrb5ReadOnlyInt case AccessKrb5ReadWrite: - return 1 + return AccessKrb5ReadWriteInt case AccessKrb5ReadNone: - return 5 + return AccessKrb5ReadNoneInt default: - return -1 + return AccessKrb5UndefinedInt } } @@ -149,161 +161,179 @@ type Parameter struct { func NewParameter(params map[string]any) *Parameter { return &Parameter{params: params} } // PvcName gets the PvcName value of the params map. -func (p *Parameter) PvcName() string { return getValueOrFallback(p.params, PvcNameKey, "") } +func (p *Parameter) PvcName() string { return utils.GetValueOrFallback(p.params, PvcNameKey, "") } // AllocType gets the AllocType value of the params map. func (p *Parameter) AllocType() int { - return getValueOrFallback(p.params, AllocTypeKey, DefaultAllocType) + return utils.GetValueOrFallback(p.params, AllocTypeKey, DefaultAllocType) } // AllSquash gets the AllSquash value of the params map. func (p *Parameter) AllSquash() int { - return getValueOrFallback(p.params, AllSquashKey, DefaultAllSquash) + return utils.GetValueOrFallback(p.params, AllSquashKey, DefaultAllSquash) } // AuthClient gets the AuthClient value of the params map. -func (p *Parameter) AuthClient() string { return getValueOrFallback(p.params, AuthClientKey, "") } +func (p *Parameter) AuthClient() string { return utils.GetValueOrFallback(p.params, AuthClientKey, "") } // Backend gets the Backend value of the params map. -func (p *Parameter) Backend() string { return getValueOrFallback(p.params, BackendKey, "") } +func (p *Parameter) Backend() string { return utils.GetValueOrFallback(p.params, BackendKey, "") } // Capacity gets the Capacity value of the params map. -func (p *Parameter) Capacity() int64 { return getValueOrFallback(p.params, CapacityKey, int64(0)) } +func (p *Parameter) Capacity() int64 { + return utils.GetValueOrFallback(p.params, CapacityKey, int64(0)) +} // Description gets the Description value of the params map. -func (p *Parameter) Description() string { return getValueOrFallback(p.params, DescriptionKey, "") } +func (p *Parameter) Description() string { + return utils.GetValueOrFallback(p.params, DescriptionKey, "") +} // MetroDomainID gets the MetroDomainID value of the params map. -func (p *Parameter) MetroDomainID() string { return getValueOrFallback(p.params, MetroDomainIDKey, "") } +func (p *Parameter) MetroDomainID() string { + return utils.GetValueOrFallback(p.params, MetroDomainIDKey, "") +} // PoolID gets the PoolID value of the params map. -func (p *Parameter) PoolID() string { return getValueOrFallback(p.params, PoolIDKey, "") } +func (p *Parameter) PoolID() string { return utils.GetValueOrFallback(p.params, PoolIDKey, "") } // RootSquash gets the RootSquash value of the params map. func (p *Parameter) RootSquash() int { - return getValueOrFallback(p.params, RootSquashKey, DefaultRootSquash) + return utils.GetValueOrFallback(p.params, RootSquashKey, DefaultRootSquash) } // StoragePool gets the StoragePool value of the params map. -func (p *Parameter) StoragePool() string { return getValueOrFallback(p.params, StoragePoolKey, "") } +func (p *Parameter) StoragePool() string { + return utils.GetValueOrFallback(p.params, StoragePoolKey, "") +} // ActiveVStoreID gets the ActiveVStoreID value of the params map. func (p *Parameter) ActiveVStoreID() string { - return getValueOrFallback(p.params, ActiveVStoreIDKey, "") + return utils.GetValueOrFallback(p.params, ActiveVStoreIDKey, "") } // StandByVStoreID gets the ActiveVStoreID value of the params map. func (p *Parameter) StandByVStoreID() string { - return getValueOrFallback(p.params, StandByVStoreIDKey, "") + return utils.GetValueOrFallback(p.params, StandByVStoreIDKey, "") } // CloneFrom gets the CloneFrom value of the params map. -func (p *Parameter) CloneFrom() string { return getValueOrFallback(p.params, CloneFromKey, "") } +func (p *Parameter) CloneFrom() string { return utils.GetValueOrFallback(p.params, CloneFromKey, "") } // CloneSpeed gets the CloneSpeed value of the params map. func (p *Parameter) CloneSpeed() int { - return getValueOrFallback(p.params, CloneSpeedKey, DefaultCloneSpeed) + return utils.GetValueOrFallback(p.params, CloneSpeedKey, DefaultCloneSpeed) } // SnapshotParentId gets the SnapshotParentId value of the params map. func (p *Parameter) SnapshotParentId() string { - return getValueOrFallback(p.params, SnapshotParentIdKey, "") + return utils.GetValueOrFallback(p.params, SnapshotParentIdKey, "") } // VStorePairId gets the VStorePairId value of the params map. -func (p *Parameter) VStorePairId() string { return getValueOrFallback(p.params, VStorePairIdKey, "") } +func (p *Parameter) VStorePairId() string { + return utils.GetValueOrFallback(p.params, VStorePairIdKey, "") +} // IsHyperMetro gets the HyperMetro value of the params map. -func (p *Parameter) IsHyperMetro() bool { return getValueOrFallback(p.params, HyperMetroKey, false) } +func (p *Parameter) IsHyperMetro() bool { + return utils.GetValueOrFallback(p.params, HyperMetroKey, false) +} // SyncMetroPairSpeed gets the SyncMetroPairSpeed value of the params map. func (p *Parameter) SyncMetroPairSpeed() int { - return getValueOrFallback(p.params, MetroPairSyncSpeedKey, 0) + return utils.GetValueOrFallback(p.params, MetroPairSyncSpeedKey, 0) } // IsReplication gets the Replication value of the params map. -func (p *Parameter) IsReplication() bool { return getValueOrFallback(p.params, ReplicationKey, false) } +func (p *Parameter) IsReplication() bool { + return utils.GetValueOrFallback(p.params, ReplicationKey, false) +} // FsPermission gets the FsPermission value of the params map. -func (p *Parameter) FsPermission() string { return getValueOrFallback(p.params, FsPermissionKey, "") } +func (p *Parameter) FsPermission() string { + return utils.GetValueOrFallback(p.params, FsPermissionKey, "") +} // SourceVolumeName gets the SourceVolumeName value of the params map. func (p *Parameter) SourceVolumeName() string { - return getValueOrFallback(p.params, SourceVolumeNameKey, "") + return utils.GetValueOrFallback(p.params, SourceVolumeNameKey, "") } // SourceSnapshotName gets the SourceSnapshotName value of the params map. func (p *Parameter) SourceSnapshotName() string { - return getValueOrFallback(p.params, SourceSnapshotNameKey, "") + return utils.GetValueOrFallback(p.params, SourceSnapshotNameKey, "") } // RemoteStoragePool gets the RemoteStoragePool value of the params map. func (p *Parameter) RemoteStoragePool() string { - return getValueOrFallback(p.params, RemoteStoragePoolKey, "") + return utils.GetValueOrFallback(p.params, RemoteStoragePoolKey, "") } // RemotePoolId gets the RemotePoolID value of the params map. func (p *Parameter) RemotePoolId() string { - return getValueOrFallback(p.params, RemotePoolIdKey, "") + return utils.GetValueOrFallback(p.params, RemotePoolIdKey, "") } // QoS gets the QoS value of the params map. func (p *Parameter) QoS() map[string]int { - return getValueOrFallback[map[string]int](p.params, QoSKey, nil) + return utils.GetValueOrFallback[map[string]int](p.params, QoSKey, nil) } // WorkloadTypeID gets the WorkloadTypeID value of the params map. func (p *Parameter) WorkloadTypeID() string { - return getValueOrFallback(p.params, WorkloadTypeIDKey, "") + return utils.GetValueOrFallback(p.params, WorkloadTypeIDKey, "") } // IsShowSnapDir gets the IsShowSnapDir value of the params map. -func (p *Parameter) IsShowSnapDir() (bool, bool) { return getValue[bool](p.params, IsShowSnapDirKey) } +func (p *Parameter) IsShowSnapDir() (bool, bool) { + return utils.GetValue[bool](p.params, IsShowSnapDirKey) +} // SnapshotReservePer gets the SnapshotReservePer value of the params map. func (p *Parameter) SnapshotReservePer() (int, bool) { - return getValue[int](p.params, SnapshotReservePerKey) + return utils.GetValue[int](p.params, SnapshotReservePerKey) } // AccessKrb5 gets the AccessKrb5 value of the params map. func (p *Parameter) AccessKrb5() int { - val := getValueOrFallback(p.params, AccessKrb5Key, AccessKrb("")) + val := utils.GetValueOrFallback(p.params, AccessKrb5Key, AccessKrb("")) return val.Int() } // AccessKrb5i gets the AccessKrb5i value of the params map. func (p *Parameter) AccessKrb5i() int { - val := getValueOrFallback(p.params, AccessKrb5iKey, AccessKrb("")) + val := utils.GetValueOrFallback(p.params, AccessKrb5iKey, AccessKrb("")) return val.Int() } // AccessKrb5p gets the AccessKrb5p value of the params map. func (p *Parameter) AccessKrb5p() int { - val := getValueOrFallback(p.params, AccessKrb5pKey, AccessKrb("")) + val := utils.GetValueOrFallback(p.params, AccessKrb5pKey, AccessKrb("")) return val.Int() } // FilesystemMode gets the FilesystemMode value of the params map. func (p *Parameter) FilesystemMode() string { - return getValueOrFallback(p.params, FilesystemModeKey, "") + return utils.GetValueOrFallback(p.params, FilesystemModeKey, "") } // Product gets the Product value of the params map. -func (p *Parameter) Product() StorageProduct { - return StorageProduct(getValueOrFallback(p.params, ProductKey, "")) +func (p *Parameter) Product() constants.OceanstorVersion { + return constants.OceanstorVersion(utils.GetValueOrFallback(p.params, ProductKey, "")) } // IsModifyVolume gets the ModifyVolume value of the params map. func (p *Parameter) IsModifyVolume() bool { - return getValueOrFallback(p.params, ModifyVolumeKey, false) + return utils.GetValueOrFallback(p.params, ModifyVolumeKey, false) } // SnapshotID gets the SnapshotID value of the params map. -func (p *Parameter) SnapshotID() string { return getValueOrFallback(p.params, SnapshotIDKey, "") } +func (p *Parameter) SnapshotID() string { return utils.GetValueOrFallback(p.params, SnapshotIDKey, "") } // SnapshotParentName gets the SnapshotParentName value of the params map. func (p *Parameter) SnapshotParentName() string { - return getValueOrFallback(p.params, SnapshotParentNameKey, "") + return utils.GetValueOrFallback(p.params, SnapshotParentNameKey, "") } // IsClone returns true if a clone filesystem needs to be created, or returns false. @@ -320,7 +350,7 @@ func (p *Parameter) IsSnapshot() bool { // IsSkipNfsShareAndQos returns true if the filesystem didn't need to create nfs share and QoS, or return false. func (p *Parameter) IsSkipNfsShareAndQos() bool { - return getValueOrFallback(p.params, IsSkipNfsShareAndQoS, false) + return utils.GetValueOrFallback(p.params, IsSkipNfsShareAndQoS, false) } // SetIsSkipNfsShare sets the value of isSkipNfsShare @@ -332,39 +362,3 @@ func (p *Parameter) SetIsSkipNfsShare(isSkip bool) { func (p *Parameter) SetQos(qos map[string]int) { p.params[QoSKey] = qos } - -// getValueOrFallback returns the value of the given key -// or the fallback value if the key is not present or type convert error. -func getValueOrFallback[T any](m map[string]any, k string, fallback T) T { - v, exists := m[k] - if !exists { - return fallback - } - - val, ok := v.(T) - if !ok { - return fallback - } - - return val -} - -func getValue[T any](m map[string]any, k string) (T, bool) { - v, exists := m[k] - if !exists { - return zeroValue[T](), false - } - - val, ok := v.(T) - if !ok { - return zeroValue[T](), false - } - - return val, true -} - -// zeroValue returns zero value of the given type. -func zeroValue[T any]() T { - var zero T - return zero -} diff --git a/storage/oceanstor/volume/creator/parameter_test.go b/storage/oceanstorage/oceanstor/volume/creator/parameter_test.go similarity index 97% rename from storage/oceanstor/volume/creator/parameter_test.go rename to storage/oceanstorage/oceanstor/volume/creator/parameter_test.go index 26e0d092..05660c04 100644 --- a/storage/oceanstor/volume/creator/parameter_test.go +++ b/storage/oceanstorage/oceanstor/volume/creator/parameter_test.go @@ -21,7 +21,8 @@ import ( "github.com/stretchr/testify/require" - "huawei-csi-driver/storage/oceanstor/volume/creator" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceanstor/volume/creator" ) func TestParameters_AllSquash(t *testing.T) { @@ -436,7 +437,7 @@ func TestParameters_VStorePairId(t *testing.T) { func TestParameters_Product(t *testing.T) { // arrange - want := creator.StorageProduct("DoradoV6") + want := constants.OceanstorVersion("DoradoV6") in := map[string]any{"product": "DoradoV6"} params := creator.NewParameter(in) @@ -453,7 +454,7 @@ func TestParameters_ProductIsV6(t *testing.T) { params := creator.NewParameter(in) // act - got := params.Product().IsV6() + got := params.Product().IsDoradoV6() // assert require.True(t, got) diff --git a/storage/oceanstorage/oceanstor/volume/creator/snapshot_fs.go b/storage/oceanstorage/oceanstor/volume/creator/snapshot_fs.go new file mode 100644 index 00000000..65f87248 --- /dev/null +++ b/storage/oceanstorage/oceanstor/volume/creator/snapshot_fs.go @@ -0,0 +1,46 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package creator + +import ( + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceanstor/client" +) + +// SnapshotFsOptionFunc defines the function to change fields of SnapshotFsCreator +type SnapshotFsOptionFunc func(*SnapshotFsCreator) + +var _ VolumeCreator = (*SnapshotFsCreator)(nil) + +// SnapshotFsCreator provides the ability to create a snapshot file system. +type SnapshotFsCreator struct { + *CloneFsCreator +} + +// NewSnapshotFsFromParams returns an instance of NewSnapshotFsFromParams +func NewSnapshotFsFromParams(cli client.OceanstorClientInterface, + params *Parameter, opts ...SnapshotFsOptionFunc) *SnapshotFsCreator { + + creator := &SnapshotFsCreator{ + CloneFsCreator: NewCloneFsCreatorByParams(cli, + params, + WithParentSnapshotId(params.SnapshotID()), + WithIsDeleteParentSnapshot(false), + WithCloneFrom(params.SnapshotParentName())), + } + + return creator +} diff --git a/storage/oceanstor/volume/dtree.go b/storage/oceanstorage/oceanstor/volume/dtree.go similarity index 98% rename from storage/oceanstor/volume/dtree.go rename to storage/oceanstorage/oceanstor/volume/dtree.go index b1866a47..5fba0e27 100644 --- a/storage/oceanstor/volume/dtree.go +++ b/storage/oceanstorage/oceanstor/volume/dtree.go @@ -22,10 +22,10 @@ import ( "fmt" "strings" - "huawei-csi-driver/storage/oceanstor/client" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/flow" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceanstor/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/flow" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( @@ -41,7 +41,7 @@ type DTree struct { } // NewDTree inits a new DTree client -func NewDTree(cli client.BaseClientInterface) *DTree { +func NewDTree(cli client.OceanstorClientInterface) *DTree { return &DTree{ Base: Base{ cli: cli, @@ -118,8 +118,7 @@ func (p *DTree) Create(ctx context.Context, params map[string]interface{}) (util return nil, err } - volObj := p.prepareVolObj(ctx, params, nil) - return volObj, nil + return p.prepareVolObj(ctx, params, nil) } // Delete deletes volume @@ -373,7 +372,7 @@ func (p *DTree) allowShareAccess(ctx context.Context, params, taskResult map[str } func (p *DTree) getCurrentShareAccess(ctx context.Context, shareID, vStoreID string, - cli client.BaseClientInterface) (map[string]interface{}, error) { + cli client.OceanstorClientInterface) (map[string]interface{}, error) { count, err := cli.GetNfsShareAccessCount(ctx, shareID, vStoreID) if err != nil { return nil, err diff --git a/storage/oceanstor/volume/dtree_test.go b/storage/oceanstorage/oceanstor/volume/dtree_test.go similarity index 90% rename from storage/oceanstor/volume/dtree_test.go rename to storage/oceanstorage/oceanstor/volume/dtree_test.go index 2cfa7514..fb9dff29 100644 --- a/storage/oceanstor/volume/dtree_test.go +++ b/storage/oceanstorage/oceanstor/volume/dtree_test.go @@ -19,11 +19,11 @@ package volume import ( "testing" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + cfg "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" "github.com/prashantv/gostub" "github.com/stretchr/testify/assert" - "huawei-csi-driver/csi/app" - cfg "huawei-csi-driver/csi/app/config" - "huawei-csi-driver/utils/log" ) const ( diff --git a/storage/oceanstor/volume/nas.go b/storage/oceanstorage/oceanstor/volume/nas.go similarity index 94% rename from storage/oceanstor/volume/nas.go rename to storage/oceanstorage/oceanstor/volume/nas.go index 1eb875c4..ec24092b 100644 --- a/storage/oceanstor/volume/nas.go +++ b/storage/oceanstorage/oceanstor/volume/nas.go @@ -25,14 +25,14 @@ import ( "strings" "time" - "huawei-csi-driver/pkg/constants" - pkgUtils "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/storage/oceanstor/client" - "huawei-csi-driver/storage/oceanstor/smartx" - "huawei-csi-driver/storage/oceanstor/volume/creator" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/flow" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceanstor/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceanstor/smartx" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceanstor/volume/creator" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/flow" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( @@ -70,7 +70,7 @@ type NAS struct { } // NewNAS inits a new nas client -func NewNAS(cli, metroRemoteCli client.BaseClientInterface, product string, +func NewNAS(cli, metroRemoteCli client.OceanstorClientInterface, product constants.OceanstorVersion, nasHyperMetro NASHyperMetro, isRunningOnOwnSite bool) *NAS { return &NAS{ @@ -496,12 +496,12 @@ func (p *NAS) Expand(ctx context.Context, fsName string, newSize int64) error { } expandTask.AddTask("Set-HyperMetro-ActiveClient", p.setActiveClient, nil) - if p.product == constants.OceanStorDoradoV6 && !p.FsHyperMetroActiveSite { + if p.product.IsDoradoV6OrV7() && !p.FsHyperMetroActiveSite { // If product is DoradoV6 and the filesystem is created by standby site, need to get remote filesystem id. expandTask.AddTask("Expand-Remote-PreCheck-Capacity", p.preExpandCheckRemoteCapacity, nil) } - if p.product != constants.OceanStorDoradoV6 { + if !p.product.IsDoradoV6OrV7() { // The NAS hyper metro feature of Dorado V6 can automatically synchronize capacity from the // active storage to the standby storage, so we don't need to expand the capacity of standby filesystem. expandTask.AddTask("Expand-Remote-PreCheck-Capacity", p.preExpandCheckRemoteCapacity, nil) @@ -541,10 +541,10 @@ func (p *NAS) createRemoteFS(ctx context.Context, params, taskResult map[string] if !ok { return nil, pkgUtils.Errorf(ctx, "convert fsName to string failed, data: %v", params["name"]) } - remoteCli, ok := taskResult["remoteCli"].(client.BaseClientInterface) + remoteCli, ok := taskResult["remoteCli"].(client.OceanstorClientInterface) if !ok { return nil, pkgUtils.Errorf(ctx, - "convert remoteCli to client.BaseClientInterface failed, data: %v", taskResult["remoteCli"]) + "convert remoteCli to client.OceanstorClientInterface failed, data: %v", taskResult["remoteCli"]) } fs, err := remoteCli.GetFileSystemByName(ctx, fsName) @@ -578,10 +578,10 @@ func (p *NAS) revertRemoteFS(ctx context.Context, taskResult map[string]interfac if !exist || fsID == "" { return nil } - remoteCli, ok := taskResult["remoteCli"].(client.BaseClientInterface) + remoteCli, ok := taskResult["remoteCli"].(client.OceanstorClientInterface) if !ok { return pkgUtils.Errorf(ctx, - "convert remoteCli to client.BaseClientInterface failed, data: %v", taskResult["remoteCli"]) + "convert remoteCli to client.OceanstorClientInterface failed, data: %v", taskResult["remoteCli"]) } deleteParams := map[string]interface{}{ "ID": fsID, @@ -594,11 +594,11 @@ func (p *NAS) revertRemoteFS(ctx context.Context, taskResult map[string]interfac func (p *NAS) setActiveClient(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { - if p.product != "DoradoV6" { + if !p.product.IsDoradoV6OrV7() { return nil, nil } - var activeClient client.BaseClientInterface + var activeClient client.OceanstorClientInterface if p.FsHyperMetroActiveSite { activeClient = p.cli } else { @@ -627,7 +627,7 @@ func (p *NAS) deleteHyperMetroShare(ctx context.Context, } // DeleteShare used to delete filesystem share -func (p *NAS) DeleteShare(ctx context.Context, name, vStoreID string, cli client.BaseClientInterface) error { +func (p *NAS) DeleteShare(ctx context.Context, name, vStoreID string, cli client.OceanstorClientInterface) error { sharePath := utils.GetOriginSharePath(name) share, err := cli.GetNfsShareByPath(ctx, sharePath, vStoreID) if err != nil { @@ -651,7 +651,7 @@ func (p *NAS) DeleteShare(ctx context.Context, name, vStoreID string, cli client } // SafeDeleteShare used to delete filesystem share -func (p *NAS) SafeDeleteShare(ctx context.Context, name, vStoreID string, cli client.BaseClientInterface) error { +func (p *NAS) SafeDeleteShare(ctx context.Context, name, vStoreID string, cli client.OceanstorClientInterface) error { sharePath := utils.GetOriginSharePath(name) share, err := cli.GetNfsShareByPath(ctx, sharePath, vStoreID) if err != nil { @@ -672,7 +672,7 @@ func (p *NAS) SafeDeleteShare(ctx context.Context, name, vStoreID string, cli cl } // DeleteFS used to delete filesystem by name -func (p *NAS) DeleteFS(ctx context.Context, fsName string, cli client.BaseClientInterface) error { +func (p *NAS) DeleteFS(ctx context.Context, fsName string, cli client.OceanstorClientInterface) error { fs, err := cli.GetFileSystemByName(ctx, fsName) if err != nil { log.AddContext(ctx).Errorf("Get filesystem %s error: %v", fsName, err) @@ -838,9 +838,9 @@ func (p *NAS) DeleteHyperMetro(ctx context.Context, } func (p *NAS) waitHyperMetroPairDeleted(ctx context.Context, - pairID string, activeClient client.BaseClientInterface) error { + pairID string, activeClient client.OceanstorClientInterface) error { var err error - if p.product == "DoradoV6" { + if p.product.IsDoradoV6OrV7() { err = activeClient.DeleteHyperMetroPair(ctx, pairID, false) } else { err = activeClient.DeleteHyperMetroPair(ctx, pairID, true) @@ -874,7 +874,7 @@ func (p *NAS) setLocalFSID(ctx context.Context, func (p *NAS) preExpandCheckRemoteCapacity(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { // define the client - var cli client.BaseClientInterface + var cli client.OceanstorClientInterface if p.replicaRemoteCli != nil { cli = p.replicaRemoteCli } else if p.metroRemoteCli != nil { @@ -923,7 +923,7 @@ func (p *NAS) preExpandCheckRemoteCapacity(ctx context.Context, }, nil } -func (p *NAS) expandFS(ctx context.Context, objID string, newSize int64, cli client.BaseClientInterface) error { +func (p *NAS) expandFS(ctx context.Context, objID string, newSize int64, cli client.OceanstorClientInterface) error { params := map[string]interface{}{ "CAPACITY": newSize, } @@ -937,7 +937,7 @@ func (p *NAS) expandFS(ctx context.Context, objID string, newSize int64, cli cli func (p *NAS) expandHyperMetroRemoteFS(ctx context.Context, params, taskResult map[string]interface{}) (map[string]interface{}, error) { - if p.product == "DoradoV6" { + if p.product.IsDoradoV6OrV7() { return nil, nil } @@ -1070,7 +1070,7 @@ func (p *NAS) DeleteSnapshot(ctx context.Context, snapshotParentId, snapshotName } func (p *NAS) tryGetSnapshotByName(ctx context.Context, - snapshotParentId, snapshotName string) (client.BaseClientInterface, map[string]any, error) { + snapshotParentId, snapshotName string) (client.OceanstorClientInterface, map[string]any, error) { snapshot, err := p.cli.GetFSSnapshotByName(ctx, snapshotParentId, snapshotName) if err != nil { return nil, nil, fmt.Errorf("get filesystem %s snapshot by name %s error: %v", @@ -1089,7 +1089,7 @@ func (p *NAS) tryGetSnapshotByName(ctx context.Context, } func (p *NAS) deleteFsSnapshot(ctx context.Context, - activeCli client.BaseClientInterface, snapshot map[string]any) error { + activeCli client.OceanstorClientInterface, snapshot map[string]any) error { snapshotId, ok := snapshot["ID"].(string) if !ok { return fmt.Errorf("convert snapshotId to string failed, data: [%v]", snapshot["ID"]) @@ -1102,7 +1102,7 @@ func (p *NAS) deleteFsSnapshot(ctx context.Context, } func (p *NAS) getActiveSnapshot(ctx context.Context, - activeCli client.BaseClientInterface, snapshotName string, snapshotParentName string) (map[string]any, error) { + activeCli client.OceanstorClientInterface, snapshotName string, snapshotParentName string) (map[string]any, error) { filesystem, err := p.getFilesystemByName(ctx, activeCli, snapshotParentName) if err != nil { return nil, fmt.Errorf("get filesystem name %s error: %w", snapshotParentName, err) @@ -1116,8 +1116,8 @@ func (p *NAS) getActiveSnapshot(ctx context.Context, return snapshot, nil } -func (p *NAS) getActiveClient(taskResult map[string]interface{}) client.BaseClientInterface { - activeClient, exist := taskResult["activeClient"].(client.BaseClientInterface) +func (p *NAS) getActiveClient(taskResult map[string]interface{}) client.OceanstorClientInterface { + activeClient, exist := taskResult["activeClient"].(client.OceanstorClientInterface) if !exist { activeClient = p.cli } @@ -1143,7 +1143,7 @@ func (p *NAS) getVStoreID(taskResult map[string]interface{}) string { } // GetActiveHyperMetroCli used to get active cli -func (p *NAS) GetActiveHyperMetroCli() client.BaseClientInterface { +func (p *NAS) GetActiveHyperMetroCli() client.OceanstorClientInterface { if p.metroRemoteCli == nil { return p.cli } @@ -1156,7 +1156,7 @@ func (p *NAS) GetActiveHyperMetroCli() client.BaseClientInterface { } // GetStandbyHyperMetroCli used to get standby cli -func (p *NAS) GetStandbyHyperMetroCli() client.BaseClientInterface { +func (p *NAS) GetStandbyHyperMetroCli() client.OceanstorClientInterface { if p.metroRemoteCli == nil { return nil } @@ -1169,7 +1169,7 @@ func (p *NAS) GetStandbyHyperMetroCli() client.BaseClientInterface { } func (p *NAS) getFilesystemByName(ctx context.Context, - cli client.BaseClientInterface, name string) (*client.FilesystemResponse, error) { + cli client.OceanstorClientInterface, name string) (*client.FilesystemResponse, error) { fsMap, err := cli.GetFileSystemByName(ctx, name) if err != nil { log.AddContext(ctx).Errorf("Get filesystem by name %s error: %v", name, err) @@ -1198,7 +1198,7 @@ func (p *NAS) getFilesystemByName(ctx context.Context, } func (p *NAS) getFilesystemByID(ctx context.Context, - cli client.BaseClientInterface, id string) (*client.FilesystemResponse, error) { + cli client.OceanstorClientInterface, id string) (*client.FilesystemResponse, error) { fsMap, err := cli.GetFileSystemByID(ctx, id) if err != nil { log.AddContext(ctx).Errorf("get filesystem by id %s error: %v", id, err) diff --git a/storage/oceanstor/volume/nas_test.go b/storage/oceanstorage/oceanstor/volume/nas_test.go similarity index 100% rename from storage/oceanstor/volume/nas_test.go rename to storage/oceanstorage/oceanstor/volume/nas_test.go diff --git a/storage/oceanstor/volume/san.go b/storage/oceanstorage/oceanstor/volume/san.go similarity index 97% rename from storage/oceanstor/volume/san.go rename to storage/oceanstorage/oceanstor/volume/san.go index 07811c89..87828453 100644 --- a/storage/oceanstor/volume/san.go +++ b/storage/oceanstorage/oceanstor/volume/san.go @@ -24,13 +24,13 @@ import ( "strconv" "time" - "huawei-csi-driver/pkg/constants" - pkgUtils "huawei-csi-driver/pkg/utils" - "huawei-csi-driver/storage/oceanstor/client" - "huawei-csi-driver/storage/oceanstor/smartx" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/flow" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" + pkgUtils "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceanstor/client" + "github.com/Huawei/eSDK_K8S_Plugin/v4/storage/oceanstorage/oceanstor/smartx" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/flow" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( @@ -44,7 +44,8 @@ type SAN struct { } // NewSAN inits a new san client -func NewSAN(cli, metroRemoteCli, replicaRemoteCli client.BaseClientInterface, product string) *SAN { +func NewSAN(cli, metroRemoteCli, replicaRemoteCli client.OceanstorClientInterface, + product constants.OceanstorVersion) *SAN { return &SAN{ Base: Base{ cli: cli, @@ -112,8 +113,7 @@ func (p *SAN) Create(ctx context.Context, params map[string]interface{}) (utils. return nil, err } - volObj := p.prepareVolObj(ctx, params, res) - return volObj, nil + return p.prepareVolObj(ctx, params, res) } // Query queries volume by name @@ -675,7 +675,7 @@ func (p *SAN) createLunCopy(ctx context.Context, func (p *SAN) clone(ctx context.Context, params map[string]interface{}, taskResult map[string]interface{}) (map[string]interface{}, error) { - if p.product == "DoradoV6" { + if p.product.IsDoradoV6OrV7() { return p.clonePair(ctx, params) } else { return p.lunCopy(ctx, params) @@ -684,7 +684,7 @@ func (p *SAN) clone(ctx context.Context, func (p *SAN) createFromSnapshot(ctx context.Context, params map[string]interface{}, taskResult map[string]interface{}) (map[string]interface{}, error) { - if p.product == "DoradoV6" { + if p.product.IsDoradoV6OrV7() { return p.fromSnapshotByClonePair(ctx, params) } else { return p.fromSnapshotByLunCopy(ctx, params) @@ -906,7 +906,7 @@ func (p *SAN) waitCloneFinish(ctx context.Context, if !ok { return pkgUtils.Errorf(ctx, "lunID convert to string failed, data: %v", lun["ID"]) } - if p.product == "DoradoV6" { + if p.product.IsDoradoV6OrV7() { // ID of clone pair is the same as destination LUN ID err := p.waitClonePairFinish(ctx, lunID) if err != nil { @@ -935,10 +935,10 @@ func (p *SAN) createRemoteLun(ctx context.Context, if !ok { return nil, pkgUtils.Errorf(ctx, "lunName convert to string failed, data: %v", params["name"]) } - remoteCli, ok := taskResult["remoteCli"].(client.BaseClientInterface) + remoteCli, ok := taskResult["remoteCli"].(client.OceanstorClientInterface) if !ok { return nil, pkgUtils.Errorf(ctx, - "remoteCli convert to client.BaseClientInterface failed, data: %v", taskResult["remoteCli"]) + "remoteCli convert to client.OceanstorClientInterface failed, data: %v", taskResult["remoteCli"]) } lun, err := remoteCli.GetLunByName(ctx, lunName) @@ -971,10 +971,10 @@ func (p *SAN) revertRemoteLun(ctx context.Context, taskResult map[string]interfa if !exist { return nil } - remoteCli, ok := taskResult["remoteCli"].(client.BaseClientInterface) + remoteCli, ok := taskResult["remoteCli"].(client.OceanstorClientInterface) if !ok { return pkgUtils.Errorf(ctx, - "remoteCli convert to client.BaseClientInterface failed, data: %v", taskResult["remoteCli"]) + "remoteCli convert to client.OceanstorClientInterface failed, data: %v", taskResult["remoteCli"]) } return remoteCli.DeleteLun(ctx, lunID) } @@ -991,10 +991,10 @@ func (p *SAN) createRemoteQoS(ctx context.Context, return nil, pkgUtils.Errorf(ctx, "lunID convert to string failed, data: %v", taskResult["remoteLunID"]) } - remoteCli, ok := taskResult["remoteCli"].(client.BaseClientInterface) + remoteCli, ok := taskResult["remoteCli"].(client.OceanstorClientInterface) if !ok { return nil, pkgUtils.Errorf(ctx, - "remoteCli convert to client.BaseClientInterface failed, data: %v", taskResult["remoteCli"]) + "remoteCli convert to client.OceanstorClientInterface failed, data: %v", taskResult["remoteCli"]) } lun, err := remoteCli.GetLunByID(ctx, lunID) if err != nil { @@ -1022,10 +1022,10 @@ func (p *SAN) revertRemoteQoS(ctx context.Context, taskResult map[string]interfa if !lunIDExist || !qosIDExist { return nil } - remoteCli, ok := taskResult["remoteCli"].(client.BaseClientInterface) + remoteCli, ok := taskResult["remoteCli"].(client.OceanstorClientInterface) if !ok { return pkgUtils.Errorf(ctx, - "remoteCli convert to client.BaseClientInterface failed, data: %v", taskResult["remoteCli"]) + "remoteCli convert to client.OceanstorClientInterface failed, data: %v", taskResult["remoteCli"]) } smartX := smartx.NewSmartX(remoteCli) return smartX.DeleteQos(ctx, qosID, lunID, "lun", "") @@ -1326,7 +1326,7 @@ func (p *SAN) deleteHyperMetro(ctx context.Context, } func (p *SAN) preExpandCheckRemoteCapacity(ctx context.Context, - params map[string]interface{}, cli client.BaseClientInterface) (string, error) { + params map[string]interface{}, cli client.OceanstorClientInterface) (string, error) { // check the remote pool name, ok := params["name"].(string) if !ok { @@ -1684,7 +1684,7 @@ func (p *SAN) deactivateSnapshot(ctx context.Context, return nil, nil } -func (p *SAN) deleteLun(ctx context.Context, name string, cli client.BaseClientInterface) error { +func (p *SAN) deleteLun(ctx context.Context, name string, cli client.OceanstorClientInterface) error { lun, err := cli.GetLunByName(ctx, name) if err != nil { log.AddContext(ctx).Errorf("Get lun by name %s error: %v", name, err) diff --git a/storage/oceanstor/volume/types.go b/storage/oceanstorage/oceanstor/volume/types.go similarity index 100% rename from storage/oceanstor/volume/types.go rename to storage/oceanstorage/oceanstor/volume/types.go diff --git a/utils/cert/cert.go b/utils/cert/cert.go new file mode 100644 index 00000000..0f379353 --- /dev/null +++ b/utils/cert/cert.go @@ -0,0 +1,202 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2022-2023. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package cert provides methods for TLS certification +package cert + +import ( + "bytes" + "context" + "crypto/ecdsa" + "crypto/elliptic" + "crypto/rand" + "crypto/tls" + "crypto/x509" + "crypto/x509/pkix" + "encoding/pem" + "fmt" + "math/big" + "time" + + "google.golang.org/grpc/credentials" + "k8s.io/api/core/v1" + apisErrors "k8s.io/apimachinery/pkg/api/errors" + metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" + + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" +) + +const ( + certUntilYears = 99 + serviceName = "huawei-csi-controller" + grpcSecretName = "huawei-csi-controller-grpc-secret" + grpcSecretKey = "key" + grpcSecretCert = "cert" + getGrpcSecretRetries = 3 + getGrpcSecretRetryPeriod = time.Second +) + +// GenerateCertificate Self Signed certificate using given CN, returns x509 cert +// and priv key in PEM format +func GenerateCertificate(ctx context.Context, cn string, dnsName string) ([]byte, []byte, error) { + var err error + pemCert := &bytes.Buffer{} + + // generate private key + priv, err := ecdsa.GenerateKey(elliptic.P521(), rand.Reader) + if err != nil { + log.AddContext(ctx).Errorf("error generating crypt keys: %v", err) + return nil, nil, err + } + + // create certificate + template := x509.Certificate{ + SerialNumber: big.NewInt(1), + Subject: pkix.Name{ + CommonName: cn, + }, + NotBefore: time.Now(), + NotAfter: time.Now().AddDate(certUntilYears, 0, 0), + KeyUsage: x509.KeyUsageKeyEncipherment | x509.KeyUsageDigitalSignature | x509.KeyUsageCertSign, + ExtKeyUsage: []x509.ExtKeyUsage{x509.ExtKeyUsageServerAuth}, + BasicConstraintsValid: true, + IsCA: true, + DNSNames: []string{dnsName}, + } + derBytes, err := x509.CreateCertificate(rand.Reader, &template, &template, &priv.PublicKey, priv) + if err != nil { + log.AddContext(ctx).Errorf("Failed to create x509 certificate: %s", err) + return nil, nil, err + } + + err = pem.Encode(pemCert, &pem.Block{Type: "CERTIFICATE", Bytes: derBytes}) + if err != nil { + log.AddContext(ctx).Errorf("Unable to encode x509 certificate to PEM format: %v", err) + return nil, nil, err + } + + b, err := x509.MarshalECPrivateKey(priv) + if err != nil { + log.AddContext(ctx).Errorf("Unable to marshal ECDSA private key: %v", err) + return nil, nil, err + } + privateBlock := pem.Block{ + Type: "RSA PRIVATE KEY", + Headers: nil, + Bytes: b, + } + + return pemCert.Bytes(), pem.EncodeToMemory(&privateBlock), nil +} + +// GetTLSCertificate from pub and priv key +func GetTLSCertificate(cert, priv []byte) (tls.Certificate, error) { + return tls.X509KeyPair(cert, priv) +} + +// GetGrpcCredential gets the gRPC credentials from Secret data in kubernetes cluster +// - If the secret data that carries the TLS certification information exists in the cluster, just use it. +// - If no secret is found, create and save it in cluster, and use it for gRPC. +func GetGrpcCredential(ctx context.Context) (credentials.TransportCredentials, error) { + var pair x509KeyPair + for { + // If CSI controller is running on AA mode, create secret may be conflict. + // When a conflict occurs, try again to get or create the secret. + var err error + pair, err = getOrCreateX509PairFromSecret(ctx, grpcSecretName, app.GetGlobalConfig().Namespace) + if err == nil { + break + } + + if !apisErrors.IsAlreadyExists(err) { + return nil, err + } + + time.Sleep(getGrpcSecretRetryPeriod) + } + + tlsCert, err := GetTLSCertificate(pair.certPEMBlock, pair.keyPEMBlock) + if err != nil { + return nil, fmt.Errorf("get TLS certificate failed, error: %v", err) + } + + return credentials.NewTLS(&tls.Config{ + MinVersion: tls.VersionTLS12, + Certificates: []tls.Certificate{tlsCert}, + }), nil +} + +type x509KeyPair struct { + certPEMBlock []byte + keyPEMBlock []byte +} + +func getOrCreateX509PairFromSecret(ctx context.Context, secretName string, namespace string) (x509KeyPair, error) { + pair := x509KeyPair{} + secret, err := app.GetGlobalConfig().K8sUtils.GetSecret(ctx, secretName, namespace) + if err == nil { + // there is already a secret, + if secret == nil { + return pair, fmt.Errorf("get nil secret %s in namespace %s", secretName, namespace) + } + + if secret.Data == nil { + return pair, fmt.Errorf("get nil data of secret %s in namespace %s", secretName, namespace) + } + + var exist bool + pair.certPEMBlock, exist = secret.Data[grpcSecretCert] + if !exist { + return pair, fmt.Errorf("invalid secret cert") + } + + pair.keyPEMBlock, exist = secret.Data[grpcSecretKey] + if !exist { + return pair, fmt.Errorf("invalid secret key") + } + + return pair, nil + } + + if !apisErrors.IsNotFound(err) { + // unexpected error, return it + return pair, fmt.Errorf("get secret %s in namespace %s failed, error: %v", secretName, namespace, err) + } + + // not found, create a new secret + dnsName := serviceName + "." + namespace + ".svc" + cn := fmt.Sprintf("%s CA", serviceName) + pair.certPEMBlock, pair.keyPEMBlock, err = GenerateCertificate(ctx, cn, dnsName) + if err != nil { + return pair, fmt.Errorf("generate certificate failed when get gRPC credential, error: %v", err) + } + + if err = createSecretForGrpcServer(ctx, pair, secretName, namespace); err != nil { + return pair, err + } + + return pair, nil +} + +func createSecretForGrpcServer(ctx context.Context, pair x509KeyPair, name, namespace string) error { + secretData := make(map[string][]byte) + secretData[grpcSecretKey] = pair.keyPEMBlock + secretData[grpcSecretCert] = pair.certPEMBlock + secret := &v1.Secret{ObjectMeta: metaV1.ObjectMeta{Name: name, Namespace: namespace}, Data: secretData} + _, err := app.GetGlobalConfig().K8sUtils.CreateSecret(ctx, secret) + return err +} diff --git a/utils/flow/taskflow.go b/utils/flow/taskflow.go index 35c987e6..60065bbe 100644 --- a/utils/flow/taskflow.go +++ b/utils/flow/taskflow.go @@ -20,8 +20,8 @@ package flow import ( "context" - "huawei-csi-driver/utils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // TaskRunFunc run task @@ -70,12 +70,12 @@ func (p *TaskFlow) AddTask(name string, run TaskRunFunc, revert TaskRevertFunc) // Run execute tasks in the task flow func (p *TaskFlow) Run(params map[string]interface{}) (map[string]interface{}, error) { - log.AddContext(p.ctx).Debugf("Start to run taskflow %s", p.name) + log.AddContext(p.ctx).Debugf("Start to run task flow %s", p.name) for _, task := range p.tasks { result, err := task.run(p.ctx, params, p.result) if err != nil { - log.AddContext(p.ctx).Errorf("Run task %s of taskflow %s error: %v", task.name, p.name, err) + log.AddContext(p.ctx).Errorf("Run task %s of task flow %s error: %v", task.name, p.name, err) return nil, err } @@ -86,7 +86,7 @@ func (p *TaskFlow) Run(params map[string]interface{}) (map[string]interface{}, e } } - log.AddContext(p.ctx).Debugf("Taskflow %s is finished", p.name) + log.AddContext(p.ctx).Debugf("Task flow %s is finished", p.name) return p.result, nil } diff --git a/utils/flow/taskflow_test.go b/utils/flow/taskflow_test.go index 6331522a..6c148d0d 100644 --- a/utils/flow/taskflow_test.go +++ b/utils/flow/taskflow_test.go @@ -22,7 +22,7 @@ import ( "reflect" "testing" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) var ( diff --git a/utils/flow/transaction_test.go b/utils/flow/transaction_test.go index 92f5c983..8e4e9bfc 100644 --- a/utils/flow/transaction_test.go +++ b/utils/flow/transaction_test.go @@ -22,7 +22,7 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" - "huawei-csi-driver/utils/flow" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/flow" ) func TestTransaction_NoError(t *testing.T) { diff --git a/utils/host.go b/utils/host.go index 0553f887..07ade8b2 100644 --- a/utils/host.go +++ b/utils/host.go @@ -22,7 +22,7 @@ import ( "regexp" "strconv" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) // ChmodFsPermission used for change target path permission diff --git a/utils/host_test.go b/utils/host_test.go index 223c493e..57cc6915 100644 --- a/utils/host_test.go +++ b/utils/host_test.go @@ -24,7 +24,7 @@ import ( "github.com/stretchr/testify/require" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const FilePermission0777 = os.FileMode(0777) diff --git a/utils/k8sutils/k8s_utils.go b/utils/k8sutils/k8s_utils.go index 66574283..09cc86c8 100644 --- a/utils/k8sutils/k8s_utils.go +++ b/utils/k8sutils/k8s_utils.go @@ -27,12 +27,12 @@ import ( corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes" "k8s.io/client-go/rest" - "k8s.io/client-go/tools/cache" "k8s.io/client-go/tools/clientcmd" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( @@ -65,6 +65,9 @@ type Interface interface { // GetVolumeAttributes returns volume attributes of PV GetVolumeAttributes(ctx context.Context, pvName string) (map[string]string, error) + // GetVolumeAttrByVolumeId returns volume attributes of PV from volume id + GetVolumeAttrByVolumeId(volumeId string) (map[string]string, error) + // Activate the k8s helpers when start the service Activate() // Deactivate the k8s helpers when stop the service @@ -79,11 +82,11 @@ type Interface interface { type KubeClient struct { clientSet kubernetes.Interface - // pvc resources cache - pvcIndexer cache.Indexer - pvcController cache.SharedIndexInformer - pvcControllerStopChan chan struct{} - pvcSource cache.ListerWatcher + // kubernetes resources cache + informersStopChan chan struct{} + informerFactory informers.SharedInformerFactory + pvcAccessor *ResourceAccessor[*corev1.PersistentVolumeClaim] + pvAccessor *ResourceAccessor[*corev1.PersistentVolume] volumeNamePrefix string volumeLabels map[string]string @@ -115,12 +118,19 @@ func NewK8SUtils(kubeConfig string, volumeNamePrefix string, volumeLabels map[st } helper := &KubeClient{ - clientSet: clientset, - pvcControllerStopChan: make(chan struct{}), - volumeNamePrefix: volumeNamePrefix, - volumeLabels: volumeLabels, + clientSet: clientset, + informersStopChan: make(chan struct{}), + volumeNamePrefix: volumeNamePrefix, + volumeLabels: volumeLabels, + informerFactory: informers.NewSharedInformerFactory(clientset, cacheSyncPeriod), + } + + if err = initPVCAccessor(helper); err != nil { + return nil, err + } + if err = initPVAccessor(helper); err != nil { + return nil, err } - initPVCWatcher(context.Background(), helper) return helper, nil } @@ -282,11 +292,11 @@ func (k *KubeClient) GetVolumeAttributes(ctx context.Context, pvName string) (ma // Activate activate k8s helpers func (k *KubeClient) Activate() { log.Infoln("Activate k8S helpers.") - go k.pvcController.Run(k.pvcControllerStopChan) + go k.informerFactory.Start(k.informersStopChan) } // Deactivate deactivate k8s helpers func (k *KubeClient) Deactivate() { log.Infoln("Deactivate k8S helpers.") - close(k.pvcControllerStopChan) + close(k.informersStopChan) } diff --git a/utils/k8sutils/pv_helper.go b/utils/k8sutils/pv_helper.go new file mode 100644 index 00000000..43ad9bce --- /dev/null +++ b/utils/k8sutils/pv_helper.go @@ -0,0 +1,80 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package k8sutils provides Kubernetes utilities +package k8sutils + +import ( + "errors" + "fmt" + + corev1 "k8s.io/api/core/v1" + "k8s.io/client-go/tools/cache" +) + +const volumeIdIndex = "volumeId" + +// GetVolumeAttrByVolumeId returns volume attributes of PV by volume id +func (k *KubeClient) GetVolumeAttrByVolumeId(volumeId string) (map[string]string, error) { + volume, err := k.pvAccessor.GetByIndex(volumeIdIndex, volumeId) + if err != nil { + return nil, fmt.Errorf("get pv %s by index failed: %v", volumeId, err) + } + + if volume.Spec.CSI == nil { + return nil, errors.New("CSI volume attribute missing from PV") + } + + return volume.Spec.CSI.VolumeAttributes, nil +} + +// volumeIdKeyFunc is a default index function that indexes based on volume id +func volumeIdKeyFunc(obj any) ([]string, error) { + volume, ok := obj.(*corev1.PersistentVolume) + if !ok { + return []string{}, fmt.Errorf("convert obj to v1.PersistentVolume failed") + } + if volume.Spec.CSI == nil { + return []string{}, nil + } + + return []string{volume.Spec.CSI.VolumeHandle}, nil +} + +func initPVAccessor(helper *KubeClient) error { + pvAccessor, err := NewResourceAccessor( + helper.informerFactory.Core().V1().PersistentVolumes().Informer(), + WithTransformer[*corev1.PersistentVolume](stripUnusedPvFields), + WithIndexers[*corev1.PersistentVolume](cache.Indexers{volumeIdIndex: volumeIdKeyFunc}), + ) + helper.pvAccessor = pvAccessor + + return err +} + +func stripUnusedPvFields(obj any) (any, error) { + pv, ok := obj.(*corev1.PersistentVolume) + if !ok { + return obj, nil + } + + res := &corev1.PersistentVolume{} + res.SetUID(pv.GetUID()) + res.SetName(pv.Name) + res.Spec.CSI = pv.Spec.CSI + + return res, nil +} diff --git a/utils/k8sutils/pvc_helper.go b/utils/k8sutils/pvc_helper.go index 658578ba..cda51fb3 100644 --- a/utils/k8sutils/pvc_helper.go +++ b/utils/k8sutils/pvc_helper.go @@ -24,14 +24,11 @@ import ( "strings" "time" - v1 "k8s.io/api/core/v1" + corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/meta" - metaV1 "k8s.io/apimachinery/pkg/apis/meta/v1" - "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/watch" "k8s.io/client-go/tools/cache" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( @@ -51,35 +48,47 @@ type persistentVolumeClaimOps interface { GetVolumeConfiguration(ctx context.Context, pvName string) (map[string]string, error) } -func initPVCWatcher(ctx context.Context, helper *KubeClient) { - // Set up a watch for PVCs - helper.pvcSource = &cache.ListWatch{ - ListFunc: func(options metaV1.ListOptions) (runtime.Object, error) { - return helper.clientSet.CoreV1().PersistentVolumeClaims(v1.NamespaceAll).List(ctx, options) - }, - WatchFunc: func(options metaV1.ListOptions) (watch.Interface, error) { - return helper.clientSet.CoreV1().PersistentVolumeClaims(v1.NamespaceAll).Watch(ctx, options) - }, +// GetVolumeConfiguration return pvc's annotations +func (k *KubeClient) GetVolumeConfiguration(ctx context.Context, pvName string) (map[string]string, error) { + log.AddContext(ctx).Infof("Start to get volume %s configuration.", pvName) + // Get the PVC corresponding to the new PV being provisioned + pvcUID := strings.TrimPrefix(pvName, fmt.Sprintf("%s-", k.volumeNamePrefix)) + pvc, err := k.pvcAccessor.GetByIndex(uidIndex, pvcUID) + if err != nil { + return nil, err } - // Set up the PVC indexing controller - helper.pvcController = cache.NewSharedIndexInformer( - helper.pvcSource, - &v1.PersistentVolumeClaim{}, - cacheSyncPeriod, - cache.Indexers{uidIndex: metaUIDKeyFunc}, - ) - - helper.pvcIndexer = helper.pvcController.GetIndexer() - _, err := helper.pvcController.AddEventHandler( - cache.ResourceEventHandlerFuncs{ + return pvc.Annotations, nil +} + +func initPVCAccessor(helper *KubeClient) error { + pvcAccessor, err := NewResourceAccessor( + helper.informerFactory.Core().V1().PersistentVolumeClaims().Informer(), + WithTransformer[*corev1.PersistentVolumeClaim](stripUnusedPvcFields), + WithIndexers[*corev1.PersistentVolumeClaim](cache.Indexers{uidIndex: metaUIDKeyFunc}), + WithHandler[*corev1.PersistentVolumeClaim](cache.ResourceEventHandlerFuncs{ AddFunc: helper.addPVC, UpdateFunc: helper.updatePVC, DeleteFunc: helper.deletePVC, - }) - if err != nil { - log.Errorf("Add event handler failed, error %v", err) + })) + helper.pvcAccessor = pvcAccessor + + return err +} + +func stripUnusedPvcFields(obj any) (any, error) { + pvc, ok := obj.(*corev1.PersistentVolumeClaim) + if !ok { + return obj, nil } + + res := &corev1.PersistentVolumeClaim{} + res.SetUID(pvc.GetUID()) + res.SetName(pvc.Name) + res.SetAnnotations(pvc.GetAnnotations()) + res.Spec.VolumeName = pvc.Spec.VolumeName + + return res, nil } // metaUIDKeyFunc is a default index function that indexes based on an object's uid @@ -101,7 +110,7 @@ func metaUIDKeyFunc(obj interface{}) ([]string, error) { func (k *KubeClient) processPVC(obj interface{}, eventType string) { switch pvc := obj.(type) { - case *v1.PersistentVolumeClaim: + case *corev1.PersistentVolumeClaim: k.processPVCEvent(pvc, eventType) default: log.Errorf("K8S helper expected PVC; got %v", obj) @@ -121,9 +130,9 @@ func (k *KubeClient) deletePVC(obj interface{}) { } // processPVC logs the add/update/delete PVC events. -func (k *KubeClient) processPVCEvent(pvc *v1.PersistentVolumeClaim, eventType string) { +func (k *KubeClient) processPVCEvent(pvc *corev1.PersistentVolumeClaim, eventType string) { // Validate the PVC - size, ok := pvc.Spec.Resources.Requests[v1.ResourceStorage] + size, ok := pvc.Spec.Resources.Requests[corev1.ResourceStorage] if !ok { log.Debugf("Rejecting PVC %s/%s, no size specified.", pvc.Namespace, pvc.Name) return @@ -142,52 +151,3 @@ func (k *KubeClient) processPVCEvent(pvc *v1.PersistentVolumeClaim, eventType st log.Warningf("UnSupport event type %s", eventType) } } - -// GetVolumeConfiguration return pvc's annotations -func (k *KubeClient) GetVolumeConfiguration(ctx context.Context, pvName string) (map[string]string, error) { - log.AddContext(ctx).Infof("Start to get volume %s configuration.", pvName) - // Get the PVC corresponding to the new PV being provisioned - pvc, err := k.getPVC(ctx, pvName) - if err != nil { - return nil, err - } - - return pvc.Annotations, nil -} - -func (k *KubeClient) getPVC(ctx context.Context, pvName string) (*v1.PersistentVolumeClaim, error) { - pvcUID := strings.TrimPrefix(pvName, fmt.Sprintf("%s-", k.volumeNamePrefix)) - pvc, err := k.getCachedPVCByUID(pvcUID) - if err != nil { - log.AddContext(ctx).Debugf("PVC %s not found in local cache: %v", pvName, err) - - // Not found immediately, so re-sync and try again - if err = k.pvcIndexer.Resync(); err != nil { - return nil, fmt.Errorf("could not refresh local PVC cache: %v", err) - } - - if pvc, err = k.getCachedPVCByUID(pvcUID); err != nil { - log.AddContext(ctx).Debugf("PVC %s not found in local cache after reSync: %v", pvName, err) - return nil, fmt.Errorf("PVCNotFound %s: %v", pvcUID, err) - } - } - - return pvc, nil -} - -func (k *KubeClient) getCachedPVCByUID(uid string) (*v1.PersistentVolumeClaim, error) { - items, err := k.pvcIndexer.ByIndex(uidIndex, uid) - if err != nil { - return nil, fmt.Errorf("could not search cache for PVC by UID %s", uid) - } else if len(items) == 0 { - return nil, fmt.Errorf("PVC object not found in cache by UID %s", uid) - } else if len(items) > 1 { - return nil, fmt.Errorf("multiple cached PVC objects found by UID %s", uid) - } - - if pvc, ok := items[0].(*v1.PersistentVolumeClaim); !ok { - return nil, fmt.Errorf("non-PVC cached object found by UID %s", uid) - } else { - return pvc, nil - } -} diff --git a/utils/k8sutils/pvc_helper_test.go b/utils/k8sutils/pvc_helper_test.go index 305f86df..079726c5 100644 --- a/utils/k8sutils/pvc_helper_test.go +++ b/utils/k8sutils/pvc_helper_test.go @@ -21,13 +21,14 @@ import ( "context" "testing" + "github.com/stretchr/testify/assert" v1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/api/resource" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/informers" "k8s.io/client-go/kubernetes/fake" - "k8s.io/client-go/tools/cache" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) var logName = "pvc_helper_test.log" @@ -40,43 +41,55 @@ func TestMain(m *testing.M) { } func initClient() *KubeClient { + clientSet := fake.NewSimpleClientset() helper := KubeClient{ - clientSet: fake.NewSimpleClientset(), - pvcControllerStopChan: make(chan struct{}), - volumeNamePrefix: "pvc", + clientSet: clientSet, + informersStopChan: make(chan struct{}), + volumeNamePrefix: "pvc", + informerFactory: informers.NewSharedInformerFactory(clientSet, 0), } - helper.pvcController = cache.NewSharedIndexInformer( - helper.pvcSource, - &v1.PersistentVolumeClaim{}, - cacheSyncPeriod, - cache.Indexers{uidIndex: metaUIDKeyFunc}, - ) return &helper } -func TestInitPVCWatcher(t *testing.T) { +func genFakePvc(name string) *v1.PersistentVolumeClaim { + return &v1.PersistentVolumeClaim{ + TypeMeta: metav1.TypeMeta{Kind: "PersistentVolumeClaim"}, + ObjectMeta: metav1.ObjectMeta{Name: name}, + Spec: v1.PersistentVolumeClaimSpec{ + Resources: v1.VolumeResourceRequirements{ + Requests: map[v1.ResourceName]resource.Quantity{ + v1.ResourceStorage: {}, + }}, + }, + } +} + +func TestInitPVCAccessor(t *testing.T) { + // arrange helper := initClient() - initPVCWatcher(context.TODO(), helper) + + // action + err := initPVCAccessor(helper) + + // assert + assert.NoError(t, err) } func TestActivate(t *testing.T) { + // arrange helper := initClient() - initPVCWatcher(context.TODO(), helper) + + // action + err := initPVCAccessor(helper) helper.Activate() defer helper.Deactivate() + + // assert + assert.NoError(t, err) } func TestProcessPVC(t *testing.T) { - obj := &v1.PersistentVolumeClaim{ - TypeMeta: metav1.TypeMeta{Kind: "PersistentVolumeClaim"}, - ObjectMeta: metav1.ObjectMeta{Name: "fake-pvc"}, - Spec: v1.PersistentVolumeClaimSpec{ - Resources: v1.VolumeResourceRequirements{ - Requests: map[v1.ResourceName]resource.Quantity{ - v1.ResourceStorage: {}, - }}, - }, - } + obj := genFakePvc("fake-pvc") helper := initClient() helper.addPVC(obj) helper.deletePVC(obj) @@ -84,9 +97,14 @@ func TestProcessPVC(t *testing.T) { } func TestGetVolumeConfiguration(t *testing.T) { + // arrange helper := initClient() - helper.pvcIndexer = helper.pvcController.GetIndexer() + + // action + initPVCAccessor(helper) _, err := helper.GetVolumeConfiguration(context.TODO(), "fake-pvc") + + // assert if err == nil { t.Error("TestGetVolumeConfiguration failed") } diff --git a/utils/k8sutils/resource_accessor.go b/utils/k8sutils/resource_accessor.go new file mode 100644 index 00000000..3fb84633 --- /dev/null +++ b/utils/k8sutils/resource_accessor.go @@ -0,0 +1,114 @@ +/* + * + * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package k8sutils provides Kubernetes utilities +package k8sutils + +import ( + "fmt" + + "k8s.io/apimachinery/pkg/runtime" + "k8s.io/client-go/tools/cache" +) + +// ResourceAccessor provides the methods to get kubernetes resource. +type ResourceAccessor[T runtime.Object] struct { + informer cache.SharedIndexInformer +} + +// WithTransformer sets transformer for the resource. +func WithTransformer[T runtime.Object](transform cache.TransformFunc) func(*ResourceAccessor[T]) error { + return func(r *ResourceAccessor[T]) error { + if err := r.informer.SetTransform(transform); err != nil { + return fmt.Errorf("set transformer failed: %v", err) + } + + return nil + } +} + +// WithIndexers sets indexers for the resource. +func WithIndexers[T runtime.Object](indexers cache.Indexers) func(*ResourceAccessor[T]) error { + return func(r *ResourceAccessor[T]) error { + if err := r.informer.AddIndexers(indexers); err != nil { + return fmt.Errorf("add indexers failed: %w", err) + } + + return nil + } +} + +// WithHandler sets event handler for the resource. +func WithHandler[T runtime.Object](handler cache.ResourceEventHandlerFuncs) func(*ResourceAccessor[T]) error { + return func(r *ResourceAccessor[T]) error { + if _, err := r.informer.AddEventHandler(handler); err != nil { + return fmt.Errorf("add handler failed: %w", err) + } + + return nil + } +} + +// NewResourceAccessor returns a ResourceAccessor instance. +func NewResourceAccessor[T runtime.Object](informer cache.SharedIndexInformer, + options ...func(*ResourceAccessor[T]) error) (*ResourceAccessor[T], error) { + watcher := &ResourceAccessor[T]{} + watcher.informer = informer + + for _, option := range options { + if err := option(watcher); err != nil { + return nil, err + } + } + + return watcher, nil +} + +// GetByIndex gets resource by index. +// To use this method, you must add the indexer first. +func (rw *ResourceAccessor[T]) GetByIndex(indexName, indexValue string) (T, error) { + value, err := rw.getByIndex(indexName, indexValue) + if err != nil { + if err = rw.informer.GetIndexer().Resync(); err != nil { + return value, fmt.Errorf("resync %T resource failed: %w", value, err) + } + + value, err = rw.getByIndex(indexName, indexValue) + if err != nil { + return value, fmt.Errorf("get by index failed: %w", err) + } + } + + return value, nil +} + +func (rw *ResourceAccessor[T]) getByIndex(indexName, indexValue string) (T, error) { + var value T + items, err := rw.informer.GetIndexer().ByIndex(indexName, indexValue) + if err != nil { + return value, fmt.Errorf("could not search cache for %T by index %s", value, indexValue) + } else if len(items) == 0 { + return value, fmt.Errorf("%T object not found in cache by index %s", value, indexValue) + } else if len(items) > 1 { + return value, fmt.Errorf("multiple cached %T objects found by index %s", value, indexValue) + } + + value, ok := items[0].(T) + if !ok { + return value, fmt.Errorf("convert %v to %T error", items[0], value) + } + return value, nil +} diff --git a/utils/k8sutils/resource_accessor_test.go b/utils/k8sutils/resource_accessor_test.go new file mode 100644 index 00000000..5ccfec5b --- /dev/null +++ b/utils/k8sutils/resource_accessor_test.go @@ -0,0 +1,67 @@ +/* + * + * Copyright (c) Huawei Technologies Co., Ltd. 2024-2024. All rights reserved. + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package k8sutils provides Kubernetes utilities +package k8sutils + +import ( + "context" + "errors" + "sync" + "testing" + + "github.com/stretchr/testify/assert" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "k8s.io/client-go/informers" + "k8s.io/client-go/kubernetes/fake" + "k8s.io/client-go/tools/cache" +) + +func TestResourceAccessor_GetByIndex_Success(t *testing.T) { + // arrange + ctx := context.Background() + fakeClient := fake.NewSimpleClientset() + factory := informers.NewSharedInformerFactory(fakeClient, 0) + FactoryCh := make(chan struct{}) + go factory.Start(FactoryCh) + defer close(FactoryCh) + var wg sync.WaitGroup + wg.Add(1) + testPvcKey := func(obj any) ([]string, error) { + defer wg.Done() + pvc, ok := obj.(*corev1.PersistentVolumeClaim) + if !ok { + return nil, errors.New("obj is not of type *corev1.PersistentVolumeClaim") + } + + return []string{pvc.Name}, nil + } + accessor, _ := NewResourceAccessor[*corev1.PersistentVolumeClaim]( + factory.Core().V1().PersistentVolumeClaims().Informer(), + WithIndexers[*corev1.PersistentVolumeClaim](cache.Indexers{"test-index": testPvcKey})) + fakeClient.CoreV1().PersistentVolumeClaims(corev1.NamespaceDefault). + Create(ctx, genFakePvc("fake-pvc"), metav1.CreateOptions{}) + + // action + wg.Wait() + pvc, err := accessor.GetByIndex("test-index", "fake-pvc") + + // assert + assert.NoError(t, err) + assert.NotNil(t, pvc) + assert.Equal(t, pvc.Name, "fake-pvc") +} diff --git a/utils/log/file.go b/utils/log/file.go index 24eef442..7bde6d96 100644 --- a/utils/log/file.go +++ b/utils/log/file.go @@ -29,7 +29,7 @@ import ( "github.com/sirupsen/logrus" - "huawei-csi-driver/pkg/constants" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" ) const ( diff --git a/utils/notify/stop_notify.go b/utils/notify/stop_notify.go index 729f45fb..6a6eef87 100644 --- a/utils/notify/stop_notify.go +++ b/utils/notify/stop_notify.go @@ -18,7 +18,7 @@ package notify import ( - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) var stopChan = make(chan struct{}) diff --git a/utils/retry/retry.go b/utils/retry/retry.go new file mode 100644 index 00000000..e270e4e2 --- /dev/null +++ b/utils/retry/retry.go @@ -0,0 +1,54 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2020-2024. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +// Package retry provides a simple retry mechanism +package retry + +import ( + "time" +) + +type attempt struct { + attempts int + period time.Duration +} + +// Attempts sets the number of retry attempts +func Attempts(attempts int) *attempt { + return &attempt{ + attempts: attempts, + } +} + +// Period sets the period of each retry attempt +func (r *attempt) Period(period time.Duration) *attempt { + r.period = period + return r +} + +// Do run the retry function +func (r *attempt) Do(do func() error) error { + var err error + for i := 0; i < r.attempts; i++ { + if err = do(); err == nil { + return nil + } + + time.Sleep(r.period) + } + + return err +} diff --git a/utils/retry/retry_test.go b/utils/retry/retry_test.go new file mode 100644 index 00000000..6ab87596 --- /dev/null +++ b/utils/retry/retry_test.go @@ -0,0 +1,70 @@ +/* + * Copyright (c) Huawei Technologies Co., Ltd. 2020-2024. All rights reserved. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package retry_test + +import ( + "errors" + "testing" + "time" + + "github.com/stretchr/testify/assert" + + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/retry" +) + +func TestRetry_Success(t *testing.T) { + // arrange + attempts := 3 + period := time.Millisecond + count := 0 + + // action + err := retry.Attempts(attempts). + Period(period). + Do(func() error { + if count == attempts-1 { + return nil + } + count++ + return errors.New("error") + }) + + // assert + assert.NoError(t, err) + assert.Equal(t, count, attempts-1) +} + +func TestRetry_Error(t *testing.T) { + // arrange + attempts := 3 + period := time.Millisecond + count := 0 + + // action + now := time.Now() + err := retry.Attempts(attempts). + Period(period). + Do(func() error { + count++ + return errors.New("error") + }) + + // assert + assert.ErrorContains(t, err, "error") + assert.Equal(t, count, attempts) + assert.True(t, time.Since(now) > time.Duration(attempts)*period) +} diff --git a/utils/utils.go b/utils/utils.go index 399791b1..3e6a51e0 100644 --- a/utils/utils.go +++ b/utils/utils.go @@ -32,33 +32,30 @@ import ( "runtime/debug" "strconv" "strings" - "sync" "syscall" "time" "github.com/container-storage-interface/spec/lib/go/csi" "golang.org/x/sys/unix" + "google.golang.org/grpc/codes" "google.golang.org/grpc/metadata" + "google.golang.org/grpc/status" "k8s.io/apimachinery/pkg/api/resource" - "huawei-csi-driver/cli/helper" - "huawei-csi-driver/csi/app" - "huawei-csi-driver/pkg/constants" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/cli/helper" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( + DoradoV7Prefix = "V700" DoradoV6Prefix = "V600" OceanStorV5Prefix = "V500" longTimeout = 60 ) -var ( - createSymLock sync.Mutex - removeSymLock sync.Mutex -) - const ( iSCSIProtocol string = "iscsi" scsiProtocol string = "scsi" @@ -271,6 +268,10 @@ func GetFileSystemName(name string) string { return strings.Replace(name, "-", "_", -1) } +func GetPersistentVolumeName(name string) string { + return strings.Replace(name, "_", "-", -1) +} + func GetFSSnapshotName(name string) string { return strings.Replace(name, "-", "_", -1) } @@ -410,13 +411,17 @@ func ReflectCall(obj interface{}, method string, args ...interface{}) []reflect. } // GetProductVersion is to get the oceanStorage version by get info from the system -func GetProductVersion(systemInfo map[string]interface{}) (string, error) { +func GetProductVersion(systemInfo map[string]interface{}) (constants.OceanstorVersion, error) { productVersion, ok := systemInfo["PRODUCTVERSION"].(string) if !ok { return "", errors.New("there is no PRODUCTVERSION field in system info") } - if strings.HasPrefix(productVersion, DoradoV6Prefix) { + // There is no version evolution in the current oceandisk storage, + // so there is currently no version processing involved for oceandisk. + if strings.HasPrefix(productVersion, DoradoV7Prefix) { + return constants.OceanStorDoradoV7, nil + } else if strings.HasPrefix(productVersion, DoradoV6Prefix) { return constants.OceanStorDoradoV6, nil } else if strings.HasPrefix(productVersion, OceanStorV5Prefix) { return constants.OceanStorV5, nil @@ -444,8 +449,7 @@ func IsSupportFeature(features map[string]int, feature string) bool { } func TransVolumeCapacity(size int64, unit int64) int64 { - newSize := RoundUpSize(size, unit) - return newSize + return RoundUpSize(size, unit) } func RoundUpSize(volumeSizeBytes int64, allocationUnitBytes int64) int64 { @@ -573,13 +577,29 @@ func IgnoreExistCode(err error, checkExitCode []string) error { return err } -// IsCapacityAvailable indicates whether the volume size is an integer multiple of 512. -func IsCapacityAvailable(volumeSizeBytes int64, allocationUnitBytes int64) bool { +// IsCapacityAvailable indicates whether the volume size is an integer multiple of allocationUnitBytes. +func IsCapacityAvailable(volumeSizeBytes, allocationUnitBytes int64, parameters map[string]any) error { if allocationUnitBytes == 0 { - log.Warningf("IsCapacityAvailable.allocationUnitBytes is invalid, can't be zero") - return false + return errors.New("IsCapacityAvailable.allocationUnitBytes is invalid, can't be zero") } - return volumeSizeBytes%allocationUnitBytes == 0 + + disableVerifyCapacity := GetValueOrFallback(parameters, "disableVerifyCapacity", "false") + var disableVerify bool + if disableVerifyCapacity != "" { + var err error + disableVerify, err = strconv.ParseBool(disableVerifyCapacity) + if err != nil { + return status.Error(codes.InvalidArgument, + fmt.Sprintf("Parse %s to bool failed: %v", disableVerifyCapacity, err)) + } + } + + if disableVerify || volumeSizeBytes%allocationUnitBytes == 0 { + return nil + } + + return fmt.Errorf("failed to check volume: the capacity %d is not an integer or not multiple of %d", + volumeSizeBytes, allocationUnitBytes) } // TransToInt is to trans different type to int type. @@ -791,59 +811,6 @@ func IsPathSymlinkWithTimeout(targetPath string, duration time.Duration) (bool, } } -// CreateSymlink between source and target -func CreateSymlink(ctx context.Context, source string, target string) error { - // First check if File exists in the staging area, then remove the mount - // and then create a symlink to the devpath Serialize symlink request - - createSymLock.Lock() - defer createSymLock.Unlock() - - log.AddContext(ctx).Infof("Create symlink called for [%s] to [%s]", source, target) - - finfo, err := os.Lstat(target) - if err != nil && os.IsNotExist(err) { - log.AddContext(ctx).Infof("Mountpoint [%v] does not exist", target) - } else { - if finfo.Mode()&os.ModeSymlink == os.ModeSymlink { - log.AddContext(ctx).Infof("Path symlink already exists") - return nil - } - // As the file exists but no symlink created, - // then throw error and it should be handled by user - log.AddContext(ctx).Errorf("Already target path exists [%v].", target) - return err - } - log.AddContext(ctx).Infof("Creating symlink for [%s] to [%s]", source, target) - err = os.Symlink(source, target) - if err != nil { - log.AddContext(ctx).Errorf("Failed to create a link for [%v] to [%v]", source, target) - return err - } - return nil -} - -// RemoveSymlink from target path -func RemoveSymlink(ctx context.Context, target string) error { - log.AddContext(ctx).Infof("Remove symlink called for [%s]", target) - - removeSymLock.Lock() - defer removeSymLock.Unlock() - - // If the file is symlink delete it - symLink, err := IsPathSymlink(target) - if symLink { - clierr := os.Remove(target) - if clierr != nil { - log.AddContext(ctx).Errorf("Failed to delete the target [%v]", target) - return clierr - } - log.AddContext(ctx).Infof("Successfully deleted the target [%v]", target) - return nil - } - return err -} - // RemoveDir delete directory from filePath func RemoveDir(filePath, dir string) { if _, err := os.Lstat(filePath); err != nil && os.IsNotExist(err) { @@ -1065,3 +1032,65 @@ func RemoveString(slice []string, s string) []string { } return newSlice } + +// FormatRespErr formats resp error and returns code and msg. +func FormatRespErr(respErr map[string]interface{}) (int64, string, error) { + code, ok := respErr["code"] + if !ok { + return 0, "", fmt.Errorf("can not get code from respErr %v", respErr) + } + + floatCode, ok := code.(float64) + if !ok { + return 0, "", fmt.Errorf("can not convert resp code %v to float64", code) + } + + intCode := int64(floatCode) + msg, ok := respErr["description"] + if !ok { + return intCode, "", nil + } + + strMsg, ok := msg.(string) + if !ok { + return intCode, "", nil + } + + return intCode, strMsg, nil +} + +// GetValueOrFallback returns the value of the given key +// or the fallback value if the key is not present or type convert error. +func GetValueOrFallback[T any](m map[string]any, k string, fallback T) T { + v, exists := m[k] + if !exists { + return fallback + } + + val, ok := v.(T) + if !ok { + return fallback + } + + return val +} + +func GetValue[T any](m map[string]any, k string) (T, bool) { + v, exists := m[k] + if !exists { + return zeroValue[T](), false + } + + val, ok := v.(T) + if !ok { + return zeroValue[T](), false + } + + return val, true +} + +// zeroValue returns zero value of the given type. +func zeroValue[T any]() T { + var zero T + return zero +} diff --git a/utils/utils_test.go b/utils/utils_test.go index f857062c..323d5f42 100644 --- a/utils/utils_test.go +++ b/utils/utils_test.go @@ -25,17 +25,17 @@ import ( "github.com/stretchr/testify/require" - "huawei-csi-driver/pkg/constants" + "github.com/Huawei/eSDK_K8S_Plugin/v4/pkg/constants" "github.com/agiledragon/gomonkey/v2" "github.com/prashantv/gostub" "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" - "huawei-csi-driver/csi/app" - cfg "huawei-csi-driver/csi/app/config" - "huawei-csi-driver/utils/k8sutils" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app" + cfg "github.com/Huawei/eSDK_K8S_Plugin/v4/csi/app/config" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/k8sutils" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( @@ -285,3 +285,56 @@ func TestMain(m *testing.M) { m.Run() } + +func TestIsCapacityAvailable_Success(t *testing.T) { + // arrange + const validSize int64 = 1024 * 1024 * 1024 + const notValidSize int64 = 1024*1024*1024 + 511 + cases := []struct { + name string + size int64 + params map[string]any + hasError bool + }{ + { + name: "Empty_Map", + size: validSize, + params: map[string]any{}, + hasError: false, + }, + { + name: "Empty_Value", + size: validSize, + params: map[string]any{"disableVerifyCapacity": ""}, + hasError: false, + }, + { + name: "False", + size: validSize, + params: map[string]any{"disableVerifyCapacity": "false"}, + hasError: false, + }, + { + name: "True_Valid", + size: validSize, + params: map[string]any{"disableVerifyCapacity": "false"}, + hasError: false, + }, + { + name: "True_Not_Valid", + size: notValidSize, + params: map[string]any{"disableVerifyCapacity": "false"}, + hasError: true, + }, + } + + for _, c := range cases { + t.Run(c.name, func(t *testing.T) { + // action + err := IsCapacityAvailable(c.size, constants.AllocationUnitBytes, c.params) + + // assert + assert.Equal(t, c.hasError, err != nil) + }) + } +} diff --git a/utils/version/version.go b/utils/version/version.go index 686ea4a2..aa1f8d2d 100644 --- a/utils/version/version.go +++ b/utils/version/version.go @@ -23,7 +23,7 @@ import ( "os" "sync" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/utils/volume.go b/utils/volume.go index fd3528d0..02ef89d5 100644 --- a/utils/volume.go +++ b/utils/volume.go @@ -24,7 +24,7 @@ type Volume interface { GetLunWWN() (string, error) SetLunWWN(string) SetSize(int64) - GetSize() (int64, error) + GetSize() int64 SetDTreeParentName(string) GetDTreeParentName() string GetFilesystemMode() string @@ -32,6 +32,7 @@ type Volume interface { GetID() string SetID(string) } + type volume struct { id string name string @@ -73,12 +74,8 @@ func (vol *volume) SetSize(size int64) { } // GetSize gets volume size in volume object -func (vol *volume) GetSize() (int64, error) { - if 0 == vol.size { - return 0, errors.New("empty Size") - } - - return vol.size, nil +func (vol *volume) GetSize() int64 { + return vol.size } func (vol *volume) SetDTreeParentName(dTreeParentName string) { diff --git a/utils/wwn_helper.go b/utils/wwn_helper.go index 241380b8..c5cd11b4 100644 --- a/utils/wwn_helper.go +++ b/utils/wwn_helper.go @@ -25,7 +25,7 @@ import ( "path/filepath" "strings" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const ( diff --git a/utils/wwn_helper_test.go b/utils/wwn_helper_test.go index c1df5240..be599d8f 100644 --- a/utils/wwn_helper_test.go +++ b/utils/wwn_helper_test.go @@ -28,7 +28,7 @@ import ( "github.com/agiledragon/gomonkey/v2" - "huawei-csi-driver/utils/log" + "github.com/Huawei/eSDK_K8S_Plugin/v4/utils/log" ) const (