Merging configuration from environment variables and configuration fi…

…le (#15)

Co-authored-by: stolbov_sa <[email protected]>
Co-authored-by: Mikhail Grigorev <[email protected]>

internal/pgscv/config.go

@@ -39,28 +39,88 @@ type Config struct {
 
 // NewConfig creates new config based on config file or return default config if config file is not specified.
 func NewConfig(configFilePath string) (*Config, error) {
-	if configFilePath == "" {
-		return newConfigFromEnv()
+	// Get configuration from file
+	var configFromFile *Config
+	if configFilePath != "" {
+		configRealPath, err := RealPath(configFilePath)
+		if err != nil {
+			return nil, err
+		}
+		log.Infoln("read configuration from ", configRealPath)
+		content, err := os.ReadFile(filepath.Clean(configRealPath))
+		if err != nil {
+			return nil, err
+		}
+		configFromFile = &Config{Defaults: map[string]string{}}
+		err = yaml.Unmarshal(content, configFromFile)
+		if err != nil {
+			return nil, err
+		}
 	}
 
-	configRealPath, err := RealPath(configFilePath)
+	// Get configuration from environment variables
+	configFromEnv, err := newConfigFromEnv()
 	if err != nil {
 		return nil, err
 	}
-	log.Infoln("read configuration from ", configRealPath)
-	content, err := os.ReadFile(filepath.Clean(configRealPath))
-	if err != nil {
-		return nil, err
+
+	// Merge values from configFromFile and configFromEnv
+	if configFromFile != nil {
+		if configFromEnv.NoTrackMode {
+			configFromFile.NoTrackMode = configFromEnv.NoTrackMode
+		}
+		if configFromEnv.ListenAddress != "" {
+			configFromFile.ListenAddress = configFromEnv.ListenAddress
+		}
+		if len(configFromEnv.ServicesConnsSettings) > 0 {
+			configFromFile.ServicesConnsSettings = mergeServicesConnsSettings(configFromFile.ServicesConnsSettings, configFromEnv.ServicesConnsSettings)
+		}
+		for key, value := range configFromEnv.Defaults {
+			configFromFile.Defaults[key] = value
+		}
+		configFromFile.DisableCollectors = append(configFromFile.DisableCollectors, configFromEnv.DisableCollectors...)
+		configFromFile.CollectorsSettings = mergeCollectorsSettings(configFromFile.CollectorsSettings, configFromEnv.CollectorsSettings)
+
+		if configFromEnv.Databases != "" {
+			// If set environment variable PGSCV_DATABASES and 'databases' settings from file is empty, then use PGSCV_DATABASES
+			if configFromFile.Databases == "" {
+				configFromFile.Databases = configFromEnv.Databases
+			} else {
+				// If set environment variable PGSCV_DATABASES and 'databases' settings from file is not empty, then use 'databases' settings from file
+				log.Debug("PGSCV_DATABASES environment setting was ignored, the settings from configuration file were used.")
+			}
+		}
+		// Set AuthConfig settings
+		if configFromEnv.AuthConfig != (http.AuthConfig{}) {
+			configFromFile.AuthConfig = configFromEnv.AuthConfig
+		}
+		return configFromFile, nil
 	}
 
-	config := &Config{Defaults: map[string]string{}}
+	return configFromEnv, nil
+}
 
-	err = yaml.Unmarshal(content, config)
-	if err != nil {
-		return nil, err
+// Merge CollectorsSettings
+func mergeCollectorsSettings(dest, src model.CollectorsSettings) model.CollectorsSettings {
+	if dest == nil {
+		return src
 	}
+	for key, value := range src {
+		dest[key] = value
+	}
+	return dest
+}
 
-	return config, nil
+// Merge services ConnsSettings
+func mergeServicesConnsSettings(dest, src service.ConnsSettings) service.ConnsSettings {
+	if dest == nil {
+		return src
+	}
+	for key, value := range src {
+		dest[key] = value
+	}
+
+	return dest
 }
 
 // Read real config file path

internal/pgscv/config_test.go

@@ -11,6 +11,167 @@ import (
 	"github.com/stretchr/testify/assert"
 )
 
+func TestMergsConfigWithEnvs(t *testing.T) {
+	var testcases = []struct {
+		name    string
+		valid   bool
+		file    string
+		envvars map[string]string
+		want    *Config
+	}{
+		{
+			valid: true, // Completely valid variables
+			file:  "testdata/pgscv-full-merge-example.yaml",
+			envvars: map[string]string{
+				"PGSCV_LISTEN_ADDRESS":     "127.0.0.1:12345",
+				"PGSCV_NO_TRACK_MODE":      "yes",
+				"PGSCV_DATABASES":          "exampledb-envs",
+				"PGSCV_DISABLE_COLLECTORS": "example/1,example/2, example/3",
+				"POSTGRES_DSN_EXAMPLE1":    "postgres://pgscv1:password1@example_dsn1:5432",
+				"PATRONI_URL":              "example_url",
+				"PATRONI_URL_EXAMPLE3":     "postgres://pgscv3:password3@example_dsn3:5432",
+				"PGSCV_AUTH_USERNAME":      "user",
+				"PGSCV_AUTH_PASSWORD":      "pass",
+				"PGSCV_AUTH_KEYFILE":       "keyfile1.key",
+				"PGSCV_AUTH_CERTFILE":      "certfile1.cert",
+			},
+			want: &Config{
+				ListenAddress:     "127.0.0.1:12345",
+				NoTrackMode:       true,
+				Databases:         "exampledb-envs",
+				DisableCollectors: []string{"fisrt-disabled-collector", "second-disabled-collector", "example/1", "example/2", "example/3"},
+				ServicesConnsSettings: map[string]service.ConnSetting{
+					"EXAMPLE1": {ServiceType: "postgres", Conninfo: "postgres://pgscv1:password1@example_dsn1:5432", BaseURL: ""},
+					"EXAMPLE3": {ServiceType: "patroni", Conninfo: "", BaseURL: "postgres://pgscv3:password3@example_dsn3:5432"},
+					"patroni":  {ServiceType: "patroni", Conninfo: "", BaseURL: "example_url"},
+					"postgres": {ServiceType: model.ServiceTypePostgresql, Conninfo: "host=127.0.0.1 port=5432 dbname=pgscv_fixtures user=pgscv", BaseURL: ""},
+				},
+				CollectorsSettings: model.CollectorsSettings{
+					"postgres/custom": {
+						Subsystems: map[string]model.MetricsSubsystem{
+							"activity": {
+								Query: "select datname as database,xact_commit,xact_rollback,blks_read as read,blks_write as write from pg_stat_database",
+								Metrics: model.Metrics{
+									{ShortName: "xact_commit_total", Usage: "COUNTER", Labels: []string{"database"}, Value: "xact_commit", Description: "description"},
+									{ShortName: "blocks_total", Usage: "COUNTER", Labels: []string{"database"},
+										LabeledValues: map[string][]string{"access": {"read", "write"}}, Description: "description",
+									},
+								},
+							},
+							"bgwriter": {
+								Query: "select maxwritten_clean from pg_stat_bgwriter",
+								Metrics: model.Metrics{
+									{ShortName: "maxwritten_clean_total", Usage: "COUNTER", Value: "maxwritten_clean", Description: "description"},
+								},
+							},
+						},
+					},
+				},
+				AuthConfig: http.AuthConfig{
+					Username: "user",
+					Password: "pass",
+					Keyfile:  "keyfile1.key",
+					Certfile: "certfile1.cert",
+				},
+				Defaults: map[string]string{
+					"postgres_username": "testuser", "postgres_password": "testpassword",
+					"pgbouncer_username": "testuser2", "pgbouncer_password": "testapassword2",
+				},
+			},
+		},
+	}
+	for _, tc := range testcases {
+		for k, v := range tc.envvars {
+			assert.NoError(t, os.Setenv(k, v))
+		}
+
+		t.Run(tc.name, func(t *testing.T) {
+			got, err := NewConfig(tc.file)
+			if tc.valid {
+				assert.NoError(t, err)
+				assert.Equal(t, tc.want, got)
+			} else {
+				assert.Error(t, err)
+			}
+		})
+		for k := range tc.envvars {
+			assert.NoError(t, os.Unsetenv(k))
+		}
+	}
+}
+
+func TestNewConfigWithEnvs(t *testing.T) {
+	var testcases = []struct {
+		name    string
+		valid   bool
+		file    string
+		envvars map[string]string
+		want    *Config
+	}{
+		{
+			valid: true, // Completely valid variables
+			file:  "testdata/pgscv-disable-collectors-example.yaml",
+			envvars: map[string]string{
+				"PGSCV_DISABLE_COLLECTORS": "example/1,example/2, example/3",
+			},
+			want: &Config{
+				ListenAddress:     "127.0.0.1:12345",
+				NoTrackMode:       false,
+				Databases:         "",
+				DisableCollectors: []string{"system", "another-disabled-collector", "example/1", "example/2", "example/3"},
+				ServicesConnsSettings: map[string]service.ConnSetting{
+					"postgres":       {ServiceType: model.ServiceTypePostgresql, Conninfo: "host=127.0.0.1 port=5432 dbname=pgscv_fixtures user=pgscv", BaseURL: ""},
+					"pgbouncer:6432": {ServiceType: model.ServiceTypePgbouncer, Conninfo: "host=127.0.0.1 port=6432 dbname=pgbouncer user=pgscv password=pgscv"},
+				},
+				Defaults: map[string]string{},
+			},
+		},
+		{
+			name:  "valid: with services in envs",
+			valid: true,
+			file:  "testdata/pgscv-services-example.yaml",
+			envvars: map[string]string{
+				"DATABASE_DSN_demo_master": "example_dsn_2:5433",
+			},
+			want: &Config{
+				ListenAddress: "127.0.0.1:8080",
+				Defaults:      map[string]string{},
+				ServicesConnsSettings: service.ConnsSettings{
+					"postgres:5432":  {ServiceType: model.ServiceTypePostgresql, Conninfo: "host=127.0.0.1 port=5432 dbname=pgscv_fixtures user=pgscv"},
+					"pgbouncer:6432": {ServiceType: model.ServiceTypePgbouncer, Conninfo: "host=127.0.0.1 port=6432 dbname=pgbouncer user=pgscv password=pgscv"},
+					"demo_master":    service.ConnSetting{ServiceType: "postgres", Conninfo: "example_dsn_2:5433", BaseURL: ""},
+				},
+			},
+		},
+	}
+	for _, tc := range testcases {
+		for k, v := range tc.envvars {
+			assert.NoError(t, os.Setenv(k, v))
+		}
+
+		t.Run(tc.name, func(t *testing.T) {
+			got, err := NewConfig(tc.file)
+			if tc.valid {
+				assert.NoError(t, err)
+				assert.Equal(t, tc.want, got)
+			} else {
+				assert.Error(t, err)
+			}
+		})
+		for k := range tc.envvars {
+			assert.NoError(t, os.Unsetenv(k))
+		}
+	}
+
+	// try to open unknown file
+	_, err := NewConfig("testdata/nonexistent.yaml")
+	assert.Error(t, err)
+
+	// try to open invalid file
+	_, err = NewConfig("testdata/invalid.txt")
+	assert.Error(t, err)
+}
+
 func TestNewConfig(t *testing.T) {
 	var testcases = []struct {
 		name  string

internal/pgscv/testdata/pgscv-disable-collectors-example.yaml

@@ -0,0 +1,11 @@
+disable_collectors:
+  - system
+  - another-disabled-collector
+listen_address: "127.0.0.1:12345"
+services:
+  "postgres":
+    service_type: "postgres"
+    conninfo: "host=127.0.0.1 port=5432 dbname=pgscv_fixtures user=pgscv"
+  "pgbouncer:6432":
+    service_type: "pgbouncer"
+    conninfo: "host=127.0.0.1 port=6432 dbname=pgbouncer user=pgscv password=pgscv"

internal/pgscv/testdata/pgscv-full-merge-example.yaml

@@ -0,0 +1,45 @@
+listen_address: "127.0.0.1:8888"
+services:
+  "postgres":
+    service_type: "postgres"
+    conninfo: "host=127.0.0.1 port=5432 dbname=pgscv_fixtures user=pgscv"
+defaults:
+  postgres_username: "testuser"
+  postgres_password: "testpassword"
+  pgbouncer_username: "testuser2"
+  pgbouncer_password: "testapassword2"
+disable_collectors:
+  - fisrt-disabled-collector
+  - second-disabled-collector
+collectors:
+  postgres/custom:
+    echo: "example"
+    subsystems:
+      activity:
+        query: "select datname as database,xact_commit,xact_rollback,blks_read as read,blks_write as write from pg_stat_database"
+        metrics:
+          - name: xact_commit_total
+            usage: COUNTER
+            labels:
+              - database
+            value: xact_commit
+            description: "description"
+          - name: "blocks_total"
+            usage: COUNTER
+            labels:
+              - database
+            labeled_values:
+              access: [ "read", "write" ]
+            description: "description"
+      bgwriter:
+        query: "select maxwritten_clean from pg_stat_bgwriter"
+        metrics:
+          - name: "maxwritten_clean_total"
+            usage: COUNTER
+            value: maxwritten_clean
+            description: "description"
+authentication:
+  username: user
+  password: supersecret
+  keyfile: example.key
+  certfile: example.cert