From 261b9beb4c1538c779244c8c60d14ae4c287e0f1 Mon Sep 17 00:00:00 2001 From: sstolbov Date: Tue, 23 Apr 2024 17:51:59 +0500 Subject: [PATCH] Merging configuration from environment variables and configuration file (#15) Co-authored-by: stolbov_sa Co-authored-by: Mikhail Grigorev --- internal/pgscv/config.go | 84 +++++++-- internal/pgscv/config_test.go | 161 ++++++++++++++++++ .../pgscv-disable-collectors-example.yaml | 11 ++ .../testdata/pgscv-full-merge-example.yaml | 45 +++++ 4 files changed, 289 insertions(+), 12 deletions(-) create mode 100644 internal/pgscv/testdata/pgscv-disable-collectors-example.yaml create mode 100644 internal/pgscv/testdata/pgscv-full-merge-example.yaml diff --git a/internal/pgscv/config.go b/internal/pgscv/config.go index 9f70df9..87e1772 100644 --- a/internal/pgscv/config.go +++ b/internal/pgscv/config.go @@ -39,28 +39,88 @@ type Config struct { // NewConfig creates new config based on config file or return default config if config file is not specified. func NewConfig(configFilePath string) (*Config, error) { - if configFilePath == "" { - return newConfigFromEnv() + // Get configuration from file + var configFromFile *Config + if configFilePath != "" { + configRealPath, err := RealPath(configFilePath) + if err != nil { + return nil, err + } + log.Infoln("read configuration from ", configRealPath) + content, err := os.ReadFile(filepath.Clean(configRealPath)) + if err != nil { + return nil, err + } + configFromFile = &Config{Defaults: map[string]string{}} + err = yaml.Unmarshal(content, configFromFile) + if err != nil { + return nil, err + } } - configRealPath, err := RealPath(configFilePath) + // Get configuration from environment variables + configFromEnv, err := newConfigFromEnv() if err != nil { return nil, err } - log.Infoln("read configuration from ", configRealPath) - content, err := os.ReadFile(filepath.Clean(configRealPath)) - if err != nil { - return nil, err + + // Merge values from configFromFile and configFromEnv + if configFromFile != nil { + if configFromEnv.NoTrackMode { + configFromFile.NoTrackMode = configFromEnv.NoTrackMode + } + if configFromEnv.ListenAddress != "" { + configFromFile.ListenAddress = configFromEnv.ListenAddress + } + if len(configFromEnv.ServicesConnsSettings) > 0 { + configFromFile.ServicesConnsSettings = mergeServicesConnsSettings(configFromFile.ServicesConnsSettings, configFromEnv.ServicesConnsSettings) + } + for key, value := range configFromEnv.Defaults { + configFromFile.Defaults[key] = value + } + configFromFile.DisableCollectors = append(configFromFile.DisableCollectors, configFromEnv.DisableCollectors...) + configFromFile.CollectorsSettings = mergeCollectorsSettings(configFromFile.CollectorsSettings, configFromEnv.CollectorsSettings) + + if configFromEnv.Databases != "" { + // If set environment variable PGSCV_DATABASES and 'databases' settings from file is empty, then use PGSCV_DATABASES + if configFromFile.Databases == "" { + configFromFile.Databases = configFromEnv.Databases + } else { + // If set environment variable PGSCV_DATABASES and 'databases' settings from file is not empty, then use 'databases' settings from file + log.Debug("PGSCV_DATABASES environment setting was ignored, the settings from configuration file were used.") + } + } + // Set AuthConfig settings + if configFromEnv.AuthConfig != (http.AuthConfig{}) { + configFromFile.AuthConfig = configFromEnv.AuthConfig + } + return configFromFile, nil } - config := &Config{Defaults: map[string]string{}} + return configFromEnv, nil +} - err = yaml.Unmarshal(content, config) - if err != nil { - return nil, err +// Merge CollectorsSettings +func mergeCollectorsSettings(dest, src model.CollectorsSettings) model.CollectorsSettings { + if dest == nil { + return src } + for key, value := range src { + dest[key] = value + } + return dest +} - return config, nil +// Merge services ConnsSettings +func mergeServicesConnsSettings(dest, src service.ConnsSettings) service.ConnsSettings { + if dest == nil { + return src + } + for key, value := range src { + dest[key] = value + } + + return dest } // Read real config file path diff --git a/internal/pgscv/config_test.go b/internal/pgscv/config_test.go index 69bd149..4e99ad2 100644 --- a/internal/pgscv/config_test.go +++ b/internal/pgscv/config_test.go @@ -11,6 +11,167 @@ import ( "github.com/stretchr/testify/assert" ) +func TestMergsConfigWithEnvs(t *testing.T) { + var testcases = []struct { + name string + valid bool + file string + envvars map[string]string + want *Config + }{ + { + valid: true, // Completely valid variables + file: "testdata/pgscv-full-merge-example.yaml", + envvars: map[string]string{ + "PGSCV_LISTEN_ADDRESS": "127.0.0.1:12345", + "PGSCV_NO_TRACK_MODE": "yes", + "PGSCV_DATABASES": "exampledb-envs", + "PGSCV_DISABLE_COLLECTORS": "example/1,example/2, example/3", + "POSTGRES_DSN_EXAMPLE1": "postgres://pgscv1:password1@example_dsn1:5432", + "PATRONI_URL": "example_url", + "PATRONI_URL_EXAMPLE3": "postgres://pgscv3:password3@example_dsn3:5432", + "PGSCV_AUTH_USERNAME": "user", + "PGSCV_AUTH_PASSWORD": "pass", + "PGSCV_AUTH_KEYFILE": "keyfile1.key", + "PGSCV_AUTH_CERTFILE": "certfile1.cert", + }, + want: &Config{ + ListenAddress: "127.0.0.1:12345", + NoTrackMode: true, + Databases: "exampledb-envs", + DisableCollectors: []string{"fisrt-disabled-collector", "second-disabled-collector", "example/1", "example/2", "example/3"}, + ServicesConnsSettings: map[string]service.ConnSetting{ + "EXAMPLE1": {ServiceType: "postgres", Conninfo: "postgres://pgscv1:password1@example_dsn1:5432", BaseURL: ""}, + "EXAMPLE3": {ServiceType: "patroni", Conninfo: "", BaseURL: "postgres://pgscv3:password3@example_dsn3:5432"}, + "patroni": {ServiceType: "patroni", Conninfo: "", BaseURL: "example_url"}, + "postgres": {ServiceType: model.ServiceTypePostgresql, Conninfo: "host=127.0.0.1 port=5432 dbname=pgscv_fixtures user=pgscv", BaseURL: ""}, + }, + CollectorsSettings: model.CollectorsSettings{ + "postgres/custom": { + Subsystems: map[string]model.MetricsSubsystem{ + "activity": { + Query: "select datname as database,xact_commit,xact_rollback,blks_read as read,blks_write as write from pg_stat_database", + Metrics: model.Metrics{ + {ShortName: "xact_commit_total", Usage: "COUNTER", Labels: []string{"database"}, Value: "xact_commit", Description: "description"}, + {ShortName: "blocks_total", Usage: "COUNTER", Labels: []string{"database"}, + LabeledValues: map[string][]string{"access": {"read", "write"}}, Description: "description", + }, + }, + }, + "bgwriter": { + Query: "select maxwritten_clean from pg_stat_bgwriter", + Metrics: model.Metrics{ + {ShortName: "maxwritten_clean_total", Usage: "COUNTER", Value: "maxwritten_clean", Description: "description"}, + }, + }, + }, + }, + }, + AuthConfig: http.AuthConfig{ + Username: "user", + Password: "pass", + Keyfile: "keyfile1.key", + Certfile: "certfile1.cert", + }, + Defaults: map[string]string{ + "postgres_username": "testuser", "postgres_password": "testpassword", + "pgbouncer_username": "testuser2", "pgbouncer_password": "testapassword2", + }, + }, + }, + } + for _, tc := range testcases { + for k, v := range tc.envvars { + assert.NoError(t, os.Setenv(k, v)) + } + + t.Run(tc.name, func(t *testing.T) { + got, err := NewConfig(tc.file) + if tc.valid { + assert.NoError(t, err) + assert.Equal(t, tc.want, got) + } else { + assert.Error(t, err) + } + }) + for k := range tc.envvars { + assert.NoError(t, os.Unsetenv(k)) + } + } +} + +func TestNewConfigWithEnvs(t *testing.T) { + var testcases = []struct { + name string + valid bool + file string + envvars map[string]string + want *Config + }{ + { + valid: true, // Completely valid variables + file: "testdata/pgscv-disable-collectors-example.yaml", + envvars: map[string]string{ + "PGSCV_DISABLE_COLLECTORS": "example/1,example/2, example/3", + }, + want: &Config{ + ListenAddress: "127.0.0.1:12345", + NoTrackMode: false, + Databases: "", + DisableCollectors: []string{"system", "another-disabled-collector", "example/1", "example/2", "example/3"}, + ServicesConnsSettings: map[string]service.ConnSetting{ + "postgres": {ServiceType: model.ServiceTypePostgresql, Conninfo: "host=127.0.0.1 port=5432 dbname=pgscv_fixtures user=pgscv", BaseURL: ""}, + "pgbouncer:6432": {ServiceType: model.ServiceTypePgbouncer, Conninfo: "host=127.0.0.1 port=6432 dbname=pgbouncer user=pgscv password=pgscv"}, + }, + Defaults: map[string]string{}, + }, + }, + { + name: "valid: with services in envs", + valid: true, + file: "testdata/pgscv-services-example.yaml", + envvars: map[string]string{ + "DATABASE_DSN_demo_master": "example_dsn_2:5433", + }, + want: &Config{ + ListenAddress: "127.0.0.1:8080", + Defaults: map[string]string{}, + ServicesConnsSettings: service.ConnsSettings{ + "postgres:5432": {ServiceType: model.ServiceTypePostgresql, Conninfo: "host=127.0.0.1 port=5432 dbname=pgscv_fixtures user=pgscv"}, + "pgbouncer:6432": {ServiceType: model.ServiceTypePgbouncer, Conninfo: "host=127.0.0.1 port=6432 dbname=pgbouncer user=pgscv password=pgscv"}, + "demo_master": service.ConnSetting{ServiceType: "postgres", Conninfo: "example_dsn_2:5433", BaseURL: ""}, + }, + }, + }, + } + for _, tc := range testcases { + for k, v := range tc.envvars { + assert.NoError(t, os.Setenv(k, v)) + } + + t.Run(tc.name, func(t *testing.T) { + got, err := NewConfig(tc.file) + if tc.valid { + assert.NoError(t, err) + assert.Equal(t, tc.want, got) + } else { + assert.Error(t, err) + } + }) + for k := range tc.envvars { + assert.NoError(t, os.Unsetenv(k)) + } + } + + // try to open unknown file + _, err := NewConfig("testdata/nonexistent.yaml") + assert.Error(t, err) + + // try to open invalid file + _, err = NewConfig("testdata/invalid.txt") + assert.Error(t, err) +} + func TestNewConfig(t *testing.T) { var testcases = []struct { name string diff --git a/internal/pgscv/testdata/pgscv-disable-collectors-example.yaml b/internal/pgscv/testdata/pgscv-disable-collectors-example.yaml new file mode 100644 index 0000000..c521d4f --- /dev/null +++ b/internal/pgscv/testdata/pgscv-disable-collectors-example.yaml @@ -0,0 +1,11 @@ +disable_collectors: + - system + - another-disabled-collector +listen_address: "127.0.0.1:12345" +services: + "postgres": + service_type: "postgres" + conninfo: "host=127.0.0.1 port=5432 dbname=pgscv_fixtures user=pgscv" + "pgbouncer:6432": + service_type: "pgbouncer" + conninfo: "host=127.0.0.1 port=6432 dbname=pgbouncer user=pgscv password=pgscv" diff --git a/internal/pgscv/testdata/pgscv-full-merge-example.yaml b/internal/pgscv/testdata/pgscv-full-merge-example.yaml new file mode 100644 index 0000000..bb97226 --- /dev/null +++ b/internal/pgscv/testdata/pgscv-full-merge-example.yaml @@ -0,0 +1,45 @@ +listen_address: "127.0.0.1:8888" +services: + "postgres": + service_type: "postgres" + conninfo: "host=127.0.0.1 port=5432 dbname=pgscv_fixtures user=pgscv" +defaults: + postgres_username: "testuser" + postgres_password: "testpassword" + pgbouncer_username: "testuser2" + pgbouncer_password: "testapassword2" +disable_collectors: + - fisrt-disabled-collector + - second-disabled-collector +collectors: + postgres/custom: + echo: "example" + subsystems: + activity: + query: "select datname as database,xact_commit,xact_rollback,blks_read as read,blks_write as write from pg_stat_database" + metrics: + - name: xact_commit_total + usage: COUNTER + labels: + - database + value: xact_commit + description: "description" + - name: "blocks_total" + usage: COUNTER + labels: + - database + labeled_values: + access: [ "read", "write" ] + description: "description" + bgwriter: + query: "select maxwritten_clean from pg_stat_bgwriter" + metrics: + - name: "maxwritten_clean_total" + usage: COUNTER + value: maxwritten_clean + description: "description" +authentication: + username: user + password: supersecret + keyfile: example.key + certfile: example.cert \ No newline at end of file