From 21a6111b875deeaab7e80e40b45692ae9a522716 Mon Sep 17 00:00:00 2001 From: kongfei Date: Tue, 6 Sep 2022 22:46:03 +0800 Subject: [PATCH 1/3] fix config collapse when instances were written in more than one file(config) --- inputs/provider.go | 11 +++++++- pkg/cfg/cfg.go | 63 ++++++++++++++++++++++++++++++++++++++-------- 2 files changed, 62 insertions(+), 12 deletions(-) diff --git a/inputs/provider.go b/inputs/provider.go index eff0348c..c50478cb 100644 --- a/inputs/provider.go +++ b/inputs/provider.go @@ -80,6 +80,15 @@ func NewProvider(c *config.ConfigType, reloadFunc func()) (Provider, error) { providers = append(providers, provider) } } + // 不添加provider配置 则默认使用local + // 兼容老版本 + if len(providers) == 0 { + provider, err := newLocalProvider(c) + if err != nil { + return nil, err + } + providers = append(providers, provider) + } return &ProviderManager{ providers: providers, }, nil @@ -224,7 +233,7 @@ func (lp *LocalProvider) GetInputConfig(inputKey string) ([]cfg.ConfigWithFormat return nil, fmt.Errorf("failed to list files under: %s : %v", lp.configDir, err) } - cwf := make([]cfg.ConfigWithFormat, 0, 1) + cwf := make([]cfg.ConfigWithFormat, 0, len(files)) for _, f := range files { c, err := file.ReadBytes(path.Join(lp.configDir, inputFilePrefix+inputKey, f)) if err != nil { diff --git a/pkg/cfg/cfg.go b/pkg/cfg/cfg.go index a680aeb0..575dbda7 100644 --- a/pkg/cfg/cfg.go +++ b/pkg/cfg/cfg.go @@ -1,7 +1,10 @@ package cfg import ( + "bytes" "fmt" + "io" + "os" "path" "strings" @@ -32,7 +35,20 @@ func GuessFormat(fpath string) ConfigFormat { return TomlFormat } +func readFile(fname string) ([]byte, error) { + file, err := os.Open(fname) + if err != nil { + return nil, err + } + defer file.Close() + return io.ReadAll(file) +} + func LoadConfigByDir(configDir string, configPtr interface{}) error { + var ( + tBuf, yBuf, jBuf []byte + ) + loaders := []multiconfig.Loader{ &multiconfig.TagLoader{}, &multiconfig.EnvironmentLoader{}, @@ -44,26 +60,41 @@ func LoadConfigByDir(configDir string, configPtr interface{}) error { } for _, fpath := range files { - if strings.HasSuffix(fpath, ".toml") { - loaders = append(loaders, &multiconfig.TOMLLoader{Path: path.Join(configDir, fpath)}) - } - if strings.HasSuffix(fpath, ".json") { - loaders = append(loaders, &multiconfig.JSONLoader{Path: path.Join(configDir, fpath)}) + buf, err := readFile(path.Join(configDir, fpath)) + if err != nil { + return err } - if strings.HasSuffix(fpath, ".yaml") || strings.HasSuffix(fpath, ".yml") { - loaders = append(loaders, &multiconfig.YAMLLoader{Path: path.Join(configDir, fpath)}) + switch { + case strings.HasSuffix(fpath, "toml"): + tBuf = append(tBuf, buf...) + case strings.HasSuffix(fpath, "json"): + jBuf = append(jBuf, buf...) + case strings.HasSuffix(fpath, "yaml") || strings.HasSuffix(fpath, "yml"): + yBuf = append(yBuf, buf...) } } + if len(tBuf) != 0 { + loaders = append(loaders, &multiconfig.TOMLLoader{Reader: bytes.NewReader(tBuf)}) + } + if len(yBuf) != 0 { + loaders = append(loaders, &multiconfig.YAMLLoader{Reader: bytes.NewReader(yBuf)}) + } + if len(jBuf) != 0 { + loaders = append(loaders, &multiconfig.JSONLoader{Reader: bytes.NewReader(jBuf)}) + } + m := multiconfig.DefaultLoader{ Loader: multiconfig.MultiLoader(loaders...), Validator: multiconfig.MultiValidator(&multiconfig.RequiredValidator{}), } - return m.Load(configPtr) } func LoadConfigs(configs []ConfigWithFormat, configPtr interface{}) error { + var ( + tBuf, yBuf, jBuf []byte + ) loaders := []multiconfig.Loader{ &multiconfig.TagLoader{}, &multiconfig.EnvironmentLoader{}, @@ -71,14 +102,24 @@ func LoadConfigs(configs []ConfigWithFormat, configPtr interface{}) error { for _, c := range configs { switch c.Format { case TomlFormat: - loaders = append(loaders, &multiconfig.TOMLLoader{Reader: strings.NewReader(c.Config)}) + tBuf = append(tBuf, []byte(c.Config)...) case YamlFormat: - loaders = append(loaders, &multiconfig.YAMLLoader{Reader: strings.NewReader(c.Config)}) + yBuf = append(yBuf, []byte(c.Config)...) case JsonFormat: - loaders = append(loaders, &multiconfig.JSONLoader{Reader: strings.NewReader(c.Config)}) + jBuf = append(jBuf, []byte(c.Config)...) } } + if len(tBuf) != 0 { + loaders = append(loaders, &multiconfig.TOMLLoader{Reader: bytes.NewReader(tBuf)}) + } + if len(yBuf) != 0 { + loaders = append(loaders, &multiconfig.YAMLLoader{Reader: bytes.NewReader(yBuf)}) + } + if len(jBuf) != 0 { + loaders = append(loaders, &multiconfig.JSONLoader{Reader: bytes.NewReader(jBuf)}) + } + m := multiconfig.DefaultLoader{ Loader: multiconfig.MultiLoader(loaders...), Validator: multiconfig.MultiValidator(&multiconfig.RequiredValidator{}), From 11453594375d7a5174897ffa2e7129bd61104915 Mon Sep 17 00:00:00 2001 From: kongfei Date: Tue, 6 Sep 2022 23:05:07 +0800 Subject: [PATCH 2/3] default provider format --- inputs/provider.go | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/inputs/provider.go b/inputs/provider.go index c50478cb..0fd37d7d 100644 --- a/inputs/provider.go +++ b/inputs/provider.go @@ -61,7 +61,11 @@ type Provider interface { func NewProvider(c *config.ConfigType, reloadFunc func()) (Provider, error) { log.Println("I! use input provider:", c.Global.Providers) - + // 不添加provider配置 则默认使用local + // 兼容老版本 + if len(c.Global.Providers) == 0 { + c.Global.Providers = append(c.Global.Providers, "local") + } providers := make([]Provider, 0, len(c.Global.Providers)) for _, p := range c.Global.Providers { name := strings.ToLower(p) @@ -80,15 +84,7 @@ func NewProvider(c *config.ConfigType, reloadFunc func()) (Provider, error) { providers = append(providers, provider) } } - // 不添加provider配置 则默认使用local - // 兼容老版本 - if len(providers) == 0 { - provider, err := newLocalProvider(c) - if err != nil { - return nil, err - } - providers = append(providers, provider) - } + return &ProviderManager{ providers: providers, }, nil From 3d4b23d623e24cc623bc53a1e7c53e1172354ae1 Mon Sep 17 00:00:00 2001 From: kongfei Date: Tue, 6 Sep 2022 23:12:06 +0800 Subject: [PATCH 3/3] update readfile method --- pkg/cfg/cfg.go | 14 ++------------ 1 file changed, 2 insertions(+), 12 deletions(-) diff --git a/pkg/cfg/cfg.go b/pkg/cfg/cfg.go index 575dbda7..8ad3b315 100644 --- a/pkg/cfg/cfg.go +++ b/pkg/cfg/cfg.go @@ -3,8 +3,7 @@ package cfg import ( "bytes" "fmt" - "io" - "os" + "io/ioutil" "path" "strings" @@ -35,15 +34,6 @@ func GuessFormat(fpath string) ConfigFormat { return TomlFormat } -func readFile(fname string) ([]byte, error) { - file, err := os.Open(fname) - if err != nil { - return nil, err - } - defer file.Close() - return io.ReadAll(file) -} - func LoadConfigByDir(configDir string, configPtr interface{}) error { var ( tBuf, yBuf, jBuf []byte @@ -60,7 +50,7 @@ func LoadConfigByDir(configDir string, configPtr interface{}) error { } for _, fpath := range files { - buf, err := readFile(path.Join(configDir, fpath)) + buf, err := ioutil.ReadFile(path.Join(configDir, fpath)) if err != nil { return err }