From 6c32829f145353646264bb666312ce2ff75e7455 Mon Sep 17 00:00:00 2001
From: sushanthakumar <sushanthakumar@huawei.com>
Date: Sun, 29 Aug 2021 23:39:53 +0530
Subject: [PATCH] Support NVMe_FC protocol

---
 src/connector/connector_utils.go              | 23 ++++++++++++++
 src/connector/nvme/nvme.go                    | 28 +++++++++++++++--
 src/connector/nvme/nvme_helper.go             | 31 +++++++++++++++++++
 ...huawei-csi-configmap-oceanstor-fcnvme.yaml | 18 +++++++++++
 4 files changed, 97 insertions(+), 3 deletions(-)
 create mode 100644 src/connector/nvme/nvme_helper.go
 create mode 100644 yamls/deploy/huawei-csi-configmap-oceanstor-fcnvme.yaml

diff --git a/src/connector/connector_utils.go b/src/connector/connector_utils.go
index 32a3872a..690f52c0 100644
--- a/src/connector/connector_utils.go
+++ b/src/connector/connector_utils.go
@@ -948,3 +948,26 @@ func RemoveRoCEDevice(device string) ([]string, string, error) {
 
 	return devices, multiPathName, nil
 }
+
+// RemoveNvmeFcDevice remove dm device if present
+func RemoveNvmeFcDevice(device string) (string, error) {
+	var multiPathName string
+	var err error
+	if strings.HasPrefix(device, "dm") {
+		multiPathName = device
+		// devices: nvme0n1, nvme2n1,
+		_, err = getDeviceFromDM(multiPathName)
+		if err != nil {
+			log.Warningf("Get the devices from the multipath %s error: %v", multiPathName, err)
+		}
+
+		// just flush the dm path. no need to delete device on host, when delete the storage mapping
+		// the device will be automatically deleted.
+		err := FlushDMDevice(multiPathName)
+		if err == nil {
+			multiPathName = ""
+		}
+	}
+
+	return multiPathName, nil
+}
diff --git a/src/connector/nvme/nvme.go b/src/connector/nvme/nvme.go
index 92f40e4d..80245e3d 100644
--- a/src/connector/nvme/nvme.go
+++ b/src/connector/nvme/nvme.go
@@ -4,11 +4,19 @@ import (
 	"connector"
 	"errors"
 	"fmt"
+	"sync"
 	"time"
 	"utils/log"
 )
 
-type FCNVMe struct{}
+type FCNVMe struct {
+	mutex sync.Mutex
+}
+
+const (
+	intNumTwo   = 2
+	intNumThree = 3
+)
 
 func init() {
 	connector.RegisterConnector(connector.FCNVMeDriver, &FCNVMe{})
@@ -22,7 +30,7 @@ func (fc *FCNVMe) ConnectVolume(conn map[string]interface{}) (string, error) {
 		log.Errorln(msg)
 		return "", errors.New(msg)
 	}
-	connectInfo := map[string]interface{} {
+	connectInfo := map[string]interface{}{
 		"protocol": "fc",
 	}
 	connector.ScanNVMe(connectInfo)
@@ -48,6 +56,20 @@ func (fc *FCNVMe) ConnectVolume(conn map[string]interface{}) (string, error) {
 }
 
 func (fc *FCNVMe) DisConnectVolume(tgtLunGuid string) error {
+	fc.mutex.Lock()
+	defer fc.mutex.Unlock()
 	log.Infof("FC-NVMe Start to disconnect volume ==> Volume Guid info: %v", tgtLunGuid)
-	return connector.DeleteDevice(tgtLunGuid)
+	for i := 0; i < 3; i++ {
+		err := tryDisConnectVolume(tgtLunGuid, true)
+		if err == nil {
+			return nil
+		}
+
+		log.Errorf("Failed to delete device in %d time(s), err: %v", i, err)
+		time.Sleep(time.Second * intNumTwo)
+	}
+
+	msg := fmt.Sprintf("Failed to delete volume %s.", tgtLunGuid)
+	log.Errorln(msg)
+	return errors.New(msg)
 }
diff --git a/src/connector/nvme/nvme_helper.go b/src/connector/nvme/nvme_helper.go
new file mode 100644
index 00000000..3510d5a8
--- /dev/null
+++ b/src/connector/nvme/nvme_helper.go
@@ -0,0 +1,31 @@
+package nvme
+
+import (
+	"connector"
+	"time"
+	"utils/log"
+)
+
+func tryDisConnectVolume(tgtLunWWN string, checkDeviceAvailable bool) error {
+	device, err := connector.GetDevice(nil, tgtLunWWN, checkDeviceAvailable)
+	if err != nil {
+		log.Warningf("Get device of WWN %s error: %v", tgtLunWWN, err)
+		return err
+	}
+
+	multiPathName, err := connector.RemoveNvmeFcDevice(device)
+	if err != nil {
+		log.Errorf("Remove device %s error: %v", device, err)
+		return err
+	}
+
+	if multiPathName != "" {
+		time.Sleep(time.Second * intNumThree)
+		err = connector.FlushDMDevice(device)
+		if err != nil {
+			return err
+		}
+	}
+
+	return nil
+}
diff --git a/yamls/deploy/huawei-csi-configmap-oceanstor-fcnvme.yaml b/yamls/deploy/huawei-csi-configmap-oceanstor-fcnvme.yaml
new file mode 100644
index 00000000..9c7a678d
--- /dev/null
+++ b/yamls/deploy/huawei-csi-configmap-oceanstor-fcnvme.yaml
@@ -0,0 +1,18 @@
+kind: ConfigMap
+apiVersion: v1
+metadata:
+  name: huawei-csi-configmap
+  namespace: kube-system
+data:
+  csi.json: |
+    {
+        "backends": [
+            {
+                "storage": "oceanstor-san",
+                "name": "***",
+                "urls": ["https://*.*.*.*:8088", "https://*.*.*.*:8088"],
+                "pools": ["***", "***"],
+                "parameters": {"protocol": "fc-nvme"}
+            }
+        ]
+    }