Merge pull request #37 from bingyanh/master

update manila Rocky

Manila/Rocky-eol/__init__.py

@@ -1 +1 @@
-"""Version: 2.3.RC2"""
+"""Version: 2.5.RC1"""

Manila/Rocky-eol/constants.py

@@ -128,7 +128,7 @@
     METRO_RUNNING_STATUS_TO_BE_SYNC) = (
     '1', '23', '35', '41', '93', '94', '100')
 
-VALID_PRODUCTS = ('V3', 'V5', 'Dorado')
+VALID_PRODUCTS = ('V3', 'V5', 'Dorado', 'V6')
 
 AVAILABLE_FEATURE_STATUS = (1, 2)
 VALID_NETWORK_TYPE = ('flat', 'vlan', 'vxlan', None)
@@ -139,3 +139,5 @@
 SNAPSHOT_ROLLBACK_COMPLETED = "0"
 
 FILESYSTEM_MODES = ('0', '2')
+RPC_CALL_TIMES = 2
+RPC_CALL_INTERVAL = 1

Manila/Rocky-eol/helper.py

@@ -537,6 +537,11 @@ def get_share_by_name(self, share_name, share_proto, vstore_id=None):
             url = "/CIFSHARE?filter=DESCRIPTION:%s&range=[0-100]" % share_name
             result = self.call(url, "GET", data)
 
+        if share_proto == 'CIFS' and result.get('data'):
+            for data in result.get('data'):
+                if data.get('NAME') == cifs_share:
+                    return data
+
         for data in result.get('data', []):
             return data
 
@@ -648,7 +653,7 @@ def update_qos_fs(self, qos_id, new_fs_list):
         _assert_result(result, 'Associate FS %s to Qos %s error.',
                        new_fs_list, qos_id)
 
-    def create_qos(self, qos, fs_id):
+    def create_qos(self, qos, fs_id, vstore_id):
         localtime = time.strftime('%Y%m%d%H%M%S', time.localtime())
         qos_name = constants.QOS_NAME_PREFIX + fs_id + '_' + localtime
         data = {"NAME": qos_name,
@@ -658,6 +663,7 @@ def create_qos(self, qos, fs_id):
                 "SCHEDULESTARTTIME": "1410969600",
                 "STARTTIME": "00:00",
                 "DURATION": "86400",
+                "vstoreId": vstore_id,
                 }
         data.update(qos)
         result = self.call("/ioclass", 'POST', data)
@@ -963,7 +969,7 @@ def sync_hypermetro_pair(self, pair_id):
         _assert_result(result, 'Sync HyperMetro pair %s error.', pair_id)
 
     def delete_hypermetro_pair(self, pair_id):
-        url = "/HyperMetroPair/%s" % pair_id
+        url = "/HyperMetroPair/%s?isOnlineDeleting=0" % pair_id
         result = self.call(url, "DELETE")
         if _error_code(result) == constants.ERROR_HYPERMETRO_NOT_EXIST:
             LOG.warning('Hypermetro pair %s to delete not exist.', pair_id)

Manila/Rocky-eol/huawei_config.py

@@ -18,7 +18,7 @@
 
 from oslo_log import log as logging
 from oslo_utils import strutils
-from xml.etree import ElementTree as ET
+from defusedxml import ElementTree as ET
 
 from manila import exception
 from manila.i18n import _
@@ -82,7 +82,7 @@ def _encode_authentication(self, tree, xml_root):
             need_encode = True
 
         if need_encode:
-            tree.write(self.config.manila_huawei_conf_file, 'UTF-8')
+            tree.write(self.config.manila_huawei_conf_file, encoding='UTF-8')
 
     def _nas_address(self, xml_root):
         text = xml_root.findtext('Storage/RestURL')

Manila/Rocky-eol/huawei_nas.py

@@ -188,10 +188,10 @@ def _check_lun_type(opts):
                 LOG.error(msg)
                 raise exception.InvalidInput(reason=msg)
 
-    def _add_smartx(self, opts, fs_id):
+    def _add_smartx(self, opts, fs_id, vstore_id):
         try:
             if opts['qos']:
-                self.smart_qos.add(opts['qos'], fs_id)
+                self.smart_qos.add(opts['qos'], fs_id, vstore_id)
             if opts['huawei_smartpartition']:
                 self.smart_partition.add(opts['partitionname'], fs_id)
             if opts['huawei_smartcache']:
@@ -318,7 +318,8 @@ def _create_filesystem(self, share, pool_name, share_fs_id=None,
                 self.configuration.timeout)
             fs_info = self._split_clone_fs(context, share, fs_id)
 
-        self._add_smartx(opts, fs_id)
+        vstore_id = fs_info.get('vstoreId')
+        self._add_smartx(opts, fs_id, vstore_id)
         self._add_hypermetro(context, opts, params, remote_vstore_id, fs_info)
         return fs_id
 
@@ -546,29 +547,44 @@ def delete_share(self, context, share, share_server=None):
     def update_replica_filesystem(self, replica_fs_id, params):
         self.helper.update_filesystem(replica_fs_id, params)
 
-    def _update_filesystem(self, fs_info, params):
-        fs_id = fs_info.get('ID')
-        if (not self.is_dorado_v6 and
-                json.loads(fs_info.get('HYPERMETROPAIRIDS'))):
+    def _check_and_get_hypermetro_pair(self, fs_info):
+        if json.loads(fs_info.get('HYPERMETROPAIRIDS')):
             metro_id = self._get_metro_id_from_fs_info(fs_info)
             metro_info = self.helper.get_hypermetro_pair_by_id(metro_id)
             if not metro_info:
                 msg = (_("The hypermetro pair %(metro_id)s does not exist.") %
                        {"metro_id": metro_id})
                 LOG.error(msg)
                 raise exception.InvalidInput(reason=msg)
+            return metro_info
+        return {}
 
-            remote_fs_id = self._get_remote_fs_id(fs_id, metro_info)
-            try:
-                context = manila_context.get_admin_context()
-                self.rpc_client.update_filesystem(context, self.remote_backend,
-                                                  remote_fs_id, params)
-            except Exception as err:
-                msg = (_("Failed to update remote filesystem %(fs_id)s. "
-                         "Reason: %(err)s") %
-                       {"fs_id": remote_fs_id, "err": err})
-                LOG.error(msg)
-                raise exception.InvalidInput(reason=msg)
+    def _is_dorado_v6_hypermetro_filesystem_not_active(self, fs_info):
+        return (self.is_dorado_v6
+                and bool(self._check_and_get_hypermetro_pair(fs_info))
+                and not self._check_is_active_client())
+
+    def _update_hypermetro_remote_filesystem(self, fs_info, params):
+        fs_id = fs_info.get('ID')
+        metro_info = self._check_and_get_hypermetro_pair(fs_info)
+
+        remote_fs_id = self._get_remote_fs_id(fs_id, metro_info)
+        try:
+            context = manila_context.get_admin_context()
+            self.rpc_client.update_filesystem(context, self.remote_backend,
+                                              remote_fs_id, params)
+        except Exception as err:
+            msg = (_("Failed to update remote filesystem %(fs_id)s. "
+                     "Reason: %(err)s") %
+                   {"fs_id": remote_fs_id, "err": err})
+            LOG.error(msg)
+            raise exception.InvalidInput(reason=msg)
+
+    def _update_filesystem(self, fs_info, params):
+        fs_id = fs_info.get('ID')
+        if (not self.is_dorado_v6 and
+                json.loads(fs_info.get('HYPERMETROPAIRIDS'))):
+            self._update_hypermetro_remote_filesystem(fs_info, params)
 
         if json.loads(fs_info.get('REMOTEREPLICATIONIDS')):
             replica_id = self._get_replica_id_from_fs_info(fs_info)
@@ -591,6 +607,11 @@ def _update_filesystem(self, fs_info, params):
                 LOG.error(msg)
                 raise exception.InvalidInput(reason=msg)
 
+        if (self.is_dorado_v6 and json.loads(fs_info.get('HYPERMETROPAIRIDS'))
+                and not self._check_is_active_client()):
+            self._update_hypermetro_remote_filesystem(fs_info, params)
+            return
+
         self.helper.update_filesystem(fs_id, params)
 
     def _get_fs_info_by_name(self, share_name, raise_exception=True):
@@ -645,16 +666,49 @@ def shrink_share(self, share, new_size, share_server=None):
         params = {"CAPACITY": size}
         self._update_filesystem(fs_info, params)
 
+    def rpc_create_hypermetro_snapshot(self, context,
+                                       share_name, snapshot_name):
+        fs_info = self._get_fs_info_by_name(share_name)
+        return self.helper.create_snapshot(fs_info['ID'], snapshot_name)
+
     def create_snapshot(self, context, snapshot, share_server=None):
         fs_info = self._get_fs_info_by_name(snapshot['share_name'])
-        snapshot_id = self.helper.create_snapshot(fs_info['ID'],
-                                                  snapshot['name'])
+        if self._is_dorado_v6_hypermetro_filesystem_not_active(fs_info):
+            snapshot_id = self.rpc_client.create_hypermetro_snapshot(
+                context, snapshot['share_name'], snapshot['name'],
+                self.remote_backend)
+        else:
+            snapshot_id = self.helper.create_snapshot(fs_info['ID'],
+                                                      snapshot['name'])
         LOG.info("Create snapshot %(snapshot)s from share %(share)s "
                  "successfully.", {"snapshot": snapshot["id"],
                                    "share": snapshot['share_name']})
         return {'provider_location': snapshot_id}
 
+    def rpc_delete_hypermetro_snapshot(self, context, share_name,
+                                       snapshot_name):
+        fs_info = self._get_fs_info_by_name(share_name)
+        snapshot_id = huawei_utils.snapshot_id(fs_info['ID'],
+                                               snapshot_name)
+        self.helper.delete_snapshot(snapshot_id)
+
     def delete_snapshot(self, context, snapshot, share_server=None):
+        fs_info = self._get_fs_info_by_name(snapshot['share_name'],
+                                            raise_exception=False)
+        if not fs_info:
+            LOG.error("The filestsyetm %s is not exist, return success.",
+                      snapshot['share_name'])
+            return
+
+        if self._is_dorado_v6_hypermetro_filesystem_not_active(fs_info):
+            self.rpc_client.delete_hypermetro_snapshot(context,
+                                                       snapshot['share_name'],
+                                                       snapshot['name'],
+                                                       self.remote_backend)
+            LOG.info("Delete snapshot %(snapshot)s successfully.",
+                     {"snapshot": snapshot["id"]})
+            return
+
         provider_location = snapshot.get('provider_location')
         if provider_location and '@' in provider_location:
             snapshot_id = provider_location
@@ -749,6 +803,7 @@ def _update_pool_info(self):
                     capacity.get('PROVISIONEDCAPACITY', 0.0),
                 'allocated_capacity_gb': capacity.get('CONSUMEDCAPACITY', 0.0),
                 'reserved_percentage': 0,
+                'reserved_snapshot_percentage': 0,
                 'qos': [self.feature_supports['SmartQoS'], False],
                 'huawei_smartcache':
                     [self.feature_supports['SmartCache'], False],
@@ -783,7 +838,7 @@ def _update_share_stats(self, date=None):
         data = {
             'share_backend_name': backend_name or 'HUAWEI_NAS_Driver',
             'vendor_name': 'Huawei',
-            'driver_version': '2.3.RC2',
+            'driver_version': '2.5.RC1',
             'storage_protocol': 'NFS_CIFS',
             'snapshot_support': (self.feature_supports['HyperSnap']
                                  and self.configuration.snapshot_support),
@@ -850,9 +905,19 @@ def _get_access_for_share_copy(self, share):
                  {'access': access, 'share': share['name']})
         return access
 
+    def rpc_create_share_from_hypermetro_snapshot(self, context, share,
+                                                  snapshot, share_server):
+        return self.create_share_from_snapshot(
+            context, share, snapshot, share_server)
+
     def create_share_from_snapshot(self, context, share,
                                    snapshot, share_server=None):
         share_fs_info = self._get_fs_info_by_name(snapshot['share_name'])
+        if self._is_dorado_v6_hypermetro_filesystem_not_active(share_fs_info):
+            return self.rpc_client.create_share_from_hypermetro_snapshot(
+                context, share, snapshot, share_server, self.remote_backend)
+
+        share_fs_info = self._get_fs_info_by_name(snapshot['share_name'])
         share_fs_id = share_fs_info['ID']
         snapshot_id = huawei_utils.snapshot_id(share_fs_id, snapshot['name'])
 
@@ -1907,9 +1972,19 @@ def _snapshot_rollback_finish():
                 constants.SNAPSHOT_ROLLBACK_TIMEOUT)
         LOG.info("Snapshot %s rollback successful.", snapshot_name)
 
+    def rpc_revert_to_hypermetro_snapshot(self, context, share_name,
+                                          snapshot_name):
+        self._revert_to_snapshot(share_name, snapshot_name)
+
     def revert_to_snapshot(self, context, snapshot, share_access_rules,
                            snapshot_access_rules, share_server=None):
-        self._revert_to_snapshot(snapshot['share_name'], snapshot['name'])
+        fs_info = self._get_fs_info_by_name(snapshot['share_name'])
+        if self._is_dorado_v6_hypermetro_filesystem_not_active(fs_info):
+            self.rpc_client.revert_to_hypermetro_snapshot(
+                context, snapshot['share_name'], snapshot['name'],
+                self.remote_backend)
+        else:
+            self._revert_to_snapshot(snapshot['share_name'], snapshot['name'])
 
     def rpc_update_snapshot(self, replica_share_name,
                             active_snapshot_name, replica_snapshot_name):

Manila/Rocky-eol/huawei_utils.py

@@ -95,7 +95,7 @@ def _get_string_param(k, v):
 def _get_opts_from_specs(specs, is_dorado):
     default_support = True if is_dorado else False
     opts_capability = {
-        'capabilities:dedupe': (_get_bool_param, default_support),
+        'capabilities:dedupe': (_get_bool_param, False),
         'capabilities:compression': (_get_bool_param, default_support),
         'capabilities:huawei_smartcache': (_get_bool_param, False),
         'capabilities:huawei_smartpartition': (_get_bool_param, False),

Manila/Rocky-eol/manager.py

@@ -81,3 +81,21 @@ def allow_access(self, context, params):
 
     def get_remote_fs_info(self, context, share_name):
         self.metro_mgr.get_remote_fs_info(share_name)
+
+    def create_hypermetro_snapshot(self, context, share_name, snapshot_name):
+        return self.driver.rpc_create_hypermetro_snapshot(context, share_name,
+                                                          snapshot_name)
+
+    def delete_hypermetro_snapshot(self, context, share_name, snapshot_name):
+        return self.driver.rpc_delete_hypermetro_snapshot(context, share_name,
+                                                          snapshot_name)
+
+    def revert_to_hypermetro_snapshot(self, context, share_name,
+                                      snapshot_name):
+        self.driver.rpc_revert_to_hypermetro_snapshot(context, share_name,
+                                                      snapshot_name)
+
+    def create_share_from_hypermetro_snapshot(self, context, share, snapshot,
+                                              share_server):
+        return self.driver.rpc_create_share_from_hypermetro_snapshot(
+            context, share, snapshot, share_server)

Manila/Rocky-eol/rpcapi.py

@@ -12,11 +12,13 @@
 #    WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the
 #    License for the specific language governing permissions and limitations
 #    under the License.
+import time
 
 import oslo_messaging as messaging
 
 from manila import rpc
 from manila.share import utils
+from manila.share.drivers.huawei import constants
 
 
 class HuaweiAPI(object):
@@ -35,6 +37,19 @@ def __init__(self):
                                   version=self.BASE_RPC_API_VERSION)
         self.client = rpc.get_client(target, version_cap='1.0')
 
+    @staticmethod
+    def _retry_rpc_call(call_times, interval, call_context_call, context,
+                        rpc_fun_name, **kwargs):
+        while call_times:
+            try:
+                return call_context_call(context, rpc_fun_name, **kwargs)
+            except Exception:
+                call_times -= 1
+                if not call_times:
+                    raise
+                time.sleep(interval)
+                continue
+
     def create_replica_pair(
             self, context, host, local_share_info, remote_device_wwn,
             remote_fs_id, local_replication):
@@ -217,3 +232,44 @@ def delete_replica_snapshot(
             replica_share_name=replica_share_name,
             replica_snapshot_name=replica_snapshot_name,
         )
+
+    def create_hypermetro_snapshot(self, context, share_name, snapshot_name,
+                                   host):
+        call_context = self.client.prepare(server=host, version='1.0')
+        return self._retry_rpc_call(constants.RPC_CALL_TIMES,
+                                    constants.RPC_CALL_INTERVAL,
+                                    call_context.call, context,
+                                    "create_hypermetro_snapshot",
+                                    share_name=share_name,
+                                    snapshot_name=snapshot_name)
+
+    def delete_hypermetro_snapshot(self, context, share_name, snapshot_name,
+                                   host):
+        call_context = self.client.prepare(server=host, version='1.0')
+        return self._retry_rpc_call(constants.RPC_CALL_TIMES,
+                                    constants.RPC_CALL_INTERVAL,
+                                    call_context.call, context,
+                                    "delete_hypermetro_snapshot",
+                                    share_name=share_name,
+                                    snapshot_name=snapshot_name)
+
+    def revert_to_hypermetro_snapshot(self, context, share_name, snapshot_name,
+                                      host):
+        call_context = self.client.prepare(server=host, version='1.0')
+        return self._retry_rpc_call(constants.RPC_CALL_TIMES,
+                                    constants.RPC_CALL_INTERVAL,
+                                    call_context.call, context,
+                                    "revert_to_hypermetro_snapshot",
+                                    share_name=share_name,
+                                    snapshot_name=snapshot_name)
+
+    def create_share_from_hypermetro_snapshot(self, context, share,
+                                              snapshot, share_server, host):
+        call_context = self.client.prepare(server=host, version='1.0')
+        return self._retry_rpc_call(constants.RPC_CALL_TIMES,
+                                    constants.RPC_CALL_INTERVAL,
+                                    call_context.call, context,
+                                    "create_share_from_hypermetro_snapshot",
+                                    share=share,
+                                    snapshot=snapshot,
+                                    share_server=share_server)