Refactor the storage function module (dragonflyoss#289)

* feat: refactor cdn storage module

Signed-off-by: santong <[email protected]>

cdnsystem/config/config.go

@@ -17,10 +17,14 @@
 package config
 
 import (
-	"fmt"
-	"io/ioutil"
 	"time"
 
+	"d7y.io/dragonfly/v2/cdnsystem/daemon/mgr/cdn/storage"
+	"d7y.io/dragonfly/v2/cdnsystem/daemon/mgr/cdn/storage/disk"
+	"d7y.io/dragonfly/v2/cdnsystem/daemon/mgr/cdn/storage/hybrid"
+	"d7y.io/dragonfly/v2/cdnsystem/plugins"
+	"d7y.io/dragonfly/v2/cdnsystem/storedriver"
+	"d7y.io/dragonfly/v2/cdnsystem/storedriver/local"
 	"d7y.io/dragonfly/v2/cmd/dependency/base"
 	"d7y.io/dragonfly/v2/pkg/unit"
 	"d7y.io/dragonfly/v2/pkg/util/net/iputils"
@@ -39,22 +43,9 @@ func New() *Config {
 type Config struct {
 	base.Options    `yaml:",inline" mapstructure:",squash"`
 	*BaseProperties `yaml:"base" mapstructure:"base"`
-	Plugins         map[PluginType][]*PluginProperties `yaml:"plugins" mapstructure:"plugins"`
-	ConfigServer    string                             `yaml:"configServer" mapstructure:"configServer"`
-}
-
-// Load loads config properties from the giving file.
-func (c *Config) Load(path string) error {
-	content, err := ioutil.ReadFile(path)
-	if err != nil {
-		return fmt.Errorf("failed to load yaml %s when reading file: %v", path, err)
-	}
 
-	if err = yaml.Unmarshal(content, c); err != nil {
-		return fmt.Errorf("failed to load yaml %s: %v", path, err)
-	}
-
-	return nil
+	Plugins      map[plugins.PluginType][]*plugins.PluginProperties `yaml:"plugins" mapstructure:"plugins"`
+	ConfigServer string                                             `yaml:"configServer" mapstructure:"configServer"`
 }
 
 func (c *Config) String() string {
@@ -64,32 +55,63 @@ func (c *Config) String() string {
 	return ""
 }
 
-// NewDefaultPlugins creates a Plugins instant with default values.
-func NewDefaultPlugins() map[PluginType][]*PluginProperties {
-	return map[PluginType][]*PluginProperties{
-		StoragePlugin: {
+// NewDefaultPlugins creates plugin instants with default values.
+func NewDefaultPlugins() map[plugins.PluginType][]*plugins.PluginProperties {
+	return map[plugins.PluginType][]*plugins.PluginProperties{
+		plugins.StorageDriverPlugin: {
 			{
-				Name:   "disk",
+				Name:   local.DiskDriverName,
 				Enable: true,
-				Config: map[string]interface{}{
-					"baseDir": "/tmp/cdnsystem",
-					"gcConfig": map[string]interface{}{
-						"youngGCThreshold":  "100G",
-						"fullGCThreshold":   "5G",
-						"cleanRatio":        1,
-						"intervalThreshold": "2h",
-					},
+				Config: &storedriver.Config{
+					BaseDir: DefaultDiskBaseDir,
 				},
 			}, {
-				Name:   "memory",
+				Name:   local.MemoryDriverName,
+				Enable: true,
+				Config: &storedriver.Config{
+					BaseDir: DefaultMemoryBaseDir,
+				},
+			},
+		}, plugins.StorageManagerPlugin: {
+			{
+				Name:   disk.StorageMode,
 				Enable: true,
-				Config: map[string]interface{}{
-					"baseDir": "/tmp/memory/dragonfly",
-					"gcConfig": map[string]interface{}{
-						"youngGCThreshold":  "100G",
-						"fullGCThreshold":   "5G",
-						"cleanRatio":        3,
-						"intervalThreshold": "2h",
+				Config: &storage.Config{
+					GCInitialDelay: 0 * time.Second,
+					GCInterval:     15 * time.Second,
+					DriverConfigs: map[string]*storage.DriverConfig{
+						local.DiskDriverName: {
+							GCConfig: &storage.GCConfig{
+								YoungGCThreshold:  100 * unit.GB,
+								FullGCThreshold:   5 * unit.GB,
+								CleanRatio:        1,
+								IntervalThreshold: 2 * time.Hour,
+							}},
+					},
+				},
+			}, {
+				Name:   hybrid.StorageMode,
+				Enable: false,
+				Config: &storage.Config{
+					GCInitialDelay: 0 * time.Second,
+					GCInterval:     15 * time.Second,
+					DriverConfigs: map[string]*storage.DriverConfig{
+						local.DiskDriverName: {
+							GCConfig: &storage.GCConfig{
+								YoungGCThreshold:  100 * unit.GB,
+								FullGCThreshold:   5 * unit.GB,
+								CleanRatio:        1,
+								IntervalThreshold: 2 * time.Hour,
+							},
+						},
+						local.MemoryDriverName: {
+							GCConfig: &storage.GCConfig{
+								YoungGCThreshold:  100 * unit.GB,
+								FullGCThreshold:   5 * unit.GB,
+								CleanRatio:        3,
+								IntervalThreshold: 2 * time.Hour,
+							},
+						},
 					},
 				},
 			},
@@ -107,9 +129,8 @@ func NewDefaultBaseProperties() *BaseProperties {
 		FailAccessInterval:      DefaultFailAccessInterval,
 		GCInitialDelay:          DefaultGCInitialDelay,
 		GCMetaInterval:          DefaultGCMetaInterval,
-		GCStorageInterval:       DefaultGCStorageInterval,
 		TaskExpireTime:          DefaultTaskExpireTime,
-		StoragePattern:          DefaultStoragePattern,
+		StorageMode:             DefaultStorageMode,
 		AdvertiseIP:             iputils.HostIP,
 	}
 }
@@ -150,15 +171,11 @@ type BaseProperties struct {
 	// default: 2min
 	GCMetaInterval time.Duration `yaml:"gcMetaInterval" mapstructure:"gcMetaInterval"`
 
-	// GCStorageInterval is the interval time to execute GC storage.
-	// default: 15s
-	GCStorageInterval time.Duration `yaml:"gcStorageInterval" mapstructure:"gcStorageInterval"`
-
 	// TaskExpireTime when a task is not accessed within the taskExpireTime,
 	// and it will be treated to be expired.
 	// default: 3min
 	TaskExpireTime time.Duration `yaml:"taskExpireTime" mapstructure:"taskExpireTime"`
 
-	// StoragePattern disk/hybrid/memory
-	StoragePattern string `yaml:"storagePattern" mapstructure:"storagePattern"`
+	// StorageMode disk/hybrid/memory
+	StorageMode string `yaml:"storageMode" mapstructure:"storageMode"`
 }

cdnsystem/config/constants.go

@@ -34,7 +34,7 @@ const (
 )
 
 const (
-	DefaultStoragePattern = "disk"
+	DefaultStorageMode = "disk"
 )
 
 const (
@@ -58,8 +58,6 @@ const (
 	// DefaultGCMetaInterval is the interval time to execute the GC meta.
 	DefaultGCMetaInterval = 2 * time.Minute
 
-	DefaultGCStorageInterval = 15 * time.Second
-
 	// DefaultTaskExpireTime when a task is not accessed within the taskExpireTime,
 	// and it will be treated to be expired.
 	DefaultTaskExpireTime = 3 * time.Minute

cdnsystem/config/store_path.go

@@ -0,0 +1,29 @@
+/*
+ *     Copyright 2020 The Dragonfly Authors
+ *
+ * Licensed under the Apache License, Version 2.0 (the "License");
+ * you may not use this file except in compliance with the License.
+ * You may obtain a copy of the License at
+ *
+ *      http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package config
+
+import (
+	"path/filepath"
+
+	"d7y.io/dragonfly/v2/pkg/basic"
+)
+
+var (
+	DefaultDiskBaseDir = filepath.Join(basic.HomeDir, "ftp")
+
+	DefaultMemoryBaseDir = "/dev/shm/dragonfly"
+)

cdnsystem/config/testdata/config/cdn.yaml

@@ -0,0 +1,94 @@
+---
+base:
+  # listenPort is the port cdn server listens on.
+  # default: 8003
+  listenPort: 8003
+
+  # DownloadPort is the port for download files from cdn.
+  # And you should start a file server firstly which listens on the download port.
+  # default: 8001
+  downloadPort: 8001
+
+  # SystemReservedBandwidth is the network bandwidth reserved for system software.
+  # default: 20 MB, in format of G(B)/g/M(B)/m/K(B)/k/B, pure number will also be parsed as Byte.
+  systemReservedBandwidth: 20M
+
+  # MaxBandwidth is the network bandwidth that cdn can use.
+  # default: 1G, in format of G(B)/g/M(B)/m/K(B)/k/B, pure number will also be parsed as Byte.
+  maxBandwidth: 1G
+
+  # FailAccessInterval is the interval time after failed to access the URL.
+  # If a task failed to be downloaded from the source, it will not be retried in the time since the last failure.
+  # default: 3m
+  failAccessInterval: 3m
+
+  # GCInitialDelay is the delay time from the start to the first GC execution.
+  # default: 6s
+  gcInitialDelay: 6s
+
+  # GCMetaInterval is the interval time to execute GC meta.
+  # default: 2m0s
+  gcMetaInterval: 2m
+
+  # TaskExpireTime when a task is not accessed within the taskExpireTime,
+  # and it will be treated to be expired.
+  # default: 3m0s
+  taskExpireTime: 3m
+
+  # storageMode is the mode of storage policy, [disk/hybrid]
+  storageMode: disk
+
+plugins:
+  storageDriver:
+    - name: disk
+      enable: true
+      config:
+        baseDir: /tmp/cdnsystem2
+    - name: memory
+      enable: true
+      config:
+        baseDir: /tmp/memory/dragonfly
+
+  storageManager:
+    - name: disk
+      enable: true
+      config:
+        gcInitialDelay: 5s
+        gcInterval: 15s
+        driverConfigs:
+          disk:
+            gcConfig:
+              youngGCThreshold: 100G
+              fullGCThreshold: 5G
+              cleanRatio: 1
+              intervalThreshold: 2h
+    - name: hybrid
+      enable: false
+      config:
+        gcInitialDelay: 5s
+        gcInterval: 15s
+        driverConfigs:
+          disk:
+            gcConfig:
+              youngGCThreshold: 100G
+              fullGCThreshold: 5G
+              cleanRatio: 1
+              intervalThreshold: 2h
+          memory:
+            gcConfig:
+              youngGCThreshold: 100G
+              fullGCThreshold: 5G
+              cleanRatio: 3
+              intervalThreshold: 2h
+
+# Console shows log on console
+# default: false
+console: false
+
+# Whether to enable debug level logger and enable pprof
+# default: false
+verbose: false
+
+# listen port for pprof, only valid when the verbose option is true
+# default is random port
+pprofPort: 0