From ca76d501dc19a5a5cca6cf24341f0875847c9e06 Mon Sep 17 00:00:00 2001 From: "Dmitry V. Bulashev" Date: Mon, 21 Oct 2024 09:29:51 +0500 Subject: [PATCH 1/7] Do not skip an inaccessible database at agent startup --- internal/service/service.go | 13 ++++++------- 1 file changed, 6 insertions(+), 7 deletions(-) diff --git a/internal/service/service.go b/internal/service/service.go index fcf01d5..55f76d1 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -128,7 +128,7 @@ func (repo *Repository) addServicesFromConfig(config Config) { return } - // Check all passed connection settings and try to connect using them. In case of success, create a 'Service' instance + // Check all passed connection settings and try to connect using them. Create a 'Service' instance // in the repo. for k, cs := range config.ConnsSettings { var msg string @@ -154,15 +154,14 @@ func (repo *Repository) addServicesFromConfig(config Config) { // Check connection using created *ConnConfig, go next if connection failed. db, err := store.NewWithConfig(pgconfig) if err != nil { - log.Warnf("%s: %s, skip", cs.Conninfo, err) - continue + log.Warnf("%s: %s", cs.Conninfo, err) + } else { + msg = fmt.Sprintf("service [%s] available through: %s@%s:%d/%s", k, pgconfig.User, pgconfig.Host, pgconfig.Port, pgconfig.Database) + db.Close() } - db.Close() - - msg = fmt.Sprintf("service [%s] available through: %s@%s:%d/%s", k, pgconfig.User, pgconfig.Host, pgconfig.Port, pgconfig.Database) } - // Connection was successful, create 'Service' struct with service-related properties and add it to service repo. + // Create 'Service' struct with service-related properties and add it to service repo. s := Service{ ServiceID: k, ConnSettings: cs, From c6577ea8e3cea213e47a61a4146f62fd71ba1470 Mon Sep 17 00:00:00 2001 From: "Dmitry V. Bulashev" Date: Mon, 21 Oct 2024 09:35:27 +0500 Subject: [PATCH 2/7] remove test --- internal/service/service_test.go | 5 ----- 1 file changed, 5 deletions(-) diff --git a/internal/service/service_test.go b/internal/service/service_test.go index 63e9033..c234cf2 100644 --- a/internal/service/service_test.go +++ b/internal/service/service_test.go @@ -77,11 +77,6 @@ func TestRepository_addServicesFromConfig(t *testing.T) { config: Config{ConnsSettings: ConnsSettings{"test": {ServiceType: model.ServiceTypePostgresql, Conninfo: "invalid conninfo"}}}, expected: 1, }, - { - name: "unavailable service", - config: Config{ConnsSettings: ConnsSettings{"test": {ServiceType: model.ServiceTypePostgresql, Conninfo: "port=1"}}}, - expected: 1, - }, } for _, tc := range testCases { From 42e3736b3a90a07b4695d06c6a329311da65a57c Mon Sep 17 00:00:00 2001 From: "Dmitry V. Bulashev" Date: Fri, 25 Oct 2024 08:51:49 +0500 Subject: [PATCH 3/7] fixup --- deploy/demo-lab/.env | 1 + deploy/demo-lab/pgscv/pgscv.yaml | 1 + internal/pgscv/config.go | 56 +++++++++++++++++++------------- internal/pgscv/pgscv.go | 19 ++++++----- internal/service/service.go | 16 ++++++--- internal/service/service_test.go | 31 +++++++++++++++--- 6 files changed, 83 insertions(+), 41 deletions(-) diff --git a/deploy/demo-lab/.env b/deploy/demo-lab/.env index 4373bfe..18a53d1 100644 --- a/deploy/demo-lab/.env +++ b/deploy/demo-lab/.env @@ -10,6 +10,7 @@ LOG_LEVEL=debug #PGSCV_COLLECT_TOP_QUERY=15 #PGSCV_COLLECT_TOP_TABLE=15 #PGSCV_COLLECT_TOP_INDEX=15 +#PGSCV_TEST_DB_CONNECTION_ON_STARTUP=f #POSTGRES_DSN_db1=postgresql://pgscv:pgscv@host1:5432/pgbench1 #POSTGRES_DSN_db2=postgresql://pgscv:pgscv@host1:5432/pgbench2 #PGSCV_DATABASES="^pgbench.*$" diff --git a/deploy/demo-lab/pgscv/pgscv.yaml b/deploy/demo-lab/pgscv/pgscv.yaml index d21b4ab..6413f52 100644 --- a/deploy/demo-lab/pgscv/pgscv.yaml +++ b/deploy/demo-lab/pgscv/pgscv.yaml @@ -8,6 +8,7 @@ no_track_mode: false #collect_top_query: 10 #collect_top_table: 10 #collect_top_index: 10 +#test_db_connection_on_startup: false services: "postgres12": service_type: "postgres" diff --git a/internal/pgscv/config.go b/internal/pgscv/config.go index 19018e4..2d2f6e5 100644 --- a/internal/pgscv/config.go +++ b/internal/pgscv/config.go @@ -27,19 +27,19 @@ const ( // Config defines application's configuration. type Config struct { - NoTrackMode bool `yaml:"no_track_mode"` // controls tracking sensitive information (query texts, etc) - ListenAddress string `yaml:"listen_address"` // Network address and port where the application should listen on - ServicesConnsSettings service.ConnsSettings `yaml:"services"` // All connections settings for exact services - Defaults map[string]string `yaml:"defaults"` // Defaults - DisableCollectors []string `yaml:"disable_collectors"` // List of collectors which should be disabled. DEPRECATED in favor collectors settings - CollectorsSettings model.CollectorsSettings `yaml:"collectors"` // Collectors settings propagated from main YAML configuration - Databases string `yaml:"databases"` // Regular expression string specifies databases from which metrics should be collected - DatabasesRE *regexp.Regexp // Regular expression object compiled from Databases - AuthConfig http.AuthConfig `yaml:"authentication"` // TLS and Basic auth configuration - CollectTopTable int `yaml:"collect_top_table"` // Limit elements on Table collector - CollectTopIndex int `yaml:"collect_top_index"` // Limit elements on Indexes collector - CollectTopQuery int `yaml:"collect_top_query"` // Limit elements on Statements collector - + NoTrackMode bool `yaml:"no_track_mode"` // controls tracking sensitive information (query texts, etc) + ListenAddress string `yaml:"listen_address"` // Network address and port where the application should listen on + ServicesConnsSettings service.ConnsSettings `yaml:"services"` // All connections settings for exact services + Defaults map[string]string `yaml:"defaults"` // Defaults + DisableCollectors []string `yaml:"disable_collectors"` // List of collectors which should be disabled. DEPRECATED in favor collectors settings + CollectorsSettings model.CollectorsSettings `yaml:"collectors"` // Collectors settings propagated from main YAML configuration + Databases string `yaml:"databases"` // Regular expression string specifies databases from which metrics should be collected + DatabasesRE *regexp.Regexp // Regular expression object compiled from Databases + AuthConfig http.AuthConfig `yaml:"authentication"` // TLS and Basic auth configuration + CollectTopTable int `yaml:"collect_top_table"` // Limit elements on Table collector + CollectTopIndex int `yaml:"collect_top_index"` // Limit elements on Indexes collector + CollectTopQuery int `yaml:"collect_top_query"` // Limit elements on Statements collector + TestDbConnectionOnStartup bool `yaml:"test_db_connection_on_startup"` // Check connection settings and try to connect using them. In case of failure, don't create a Service instance. } // NewConfig creates new config based on config file or return default config if config file is not specified. @@ -65,6 +65,7 @@ func NewConfig(configFilePath string) (*Config, error) { // Get configuration from environment variables configFromEnv, err := newConfigFromEnv() + fmt.Println(configFromEnv.TestDbConnectionOnStartup) if err != nil { return nil, err } @@ -109,6 +110,9 @@ func NewConfig(configFilePath string) (*Config, error) { if configFromEnv.CollectTopQuery > 0 { configFromFile.CollectTopQuery = configFromEnv.CollectTopQuery } + if !configFromEnv.TestDbConnectionOnStartup { + configFromFile.TestDbConnectionOnStartup = false + } return configFromFile, nil } @@ -336,8 +340,9 @@ func newConfigFromEnv() (*Config, error) { log.Infoln("read configuration from environment") config := &Config{ - Defaults: map[string]string{}, - ServicesConnsSettings: map[string]service.ConnSetting{}, + Defaults: map[string]string{}, + ServicesConnsSettings: map[string]service.ConnSetting{}, + TestDbConnectionOnStartup: true, } for _, env := range os.Environ() { @@ -387,12 +392,7 @@ func newConfigFromEnv() (*Config, error) { case "PGSCV_LISTEN_ADDRESS": config.ListenAddress = value case "PGSCV_NO_TRACK_MODE": - switch value { - case "y", "yes", "Yes", "YES", "t", "true", "True", "TRUE", "1", "on": - config.NoTrackMode = true - default: - config.NoTrackMode = false - } + config.NoTrackMode = toBool(value) case "PGSCV_DATABASES": config.Databases = value case "PGSCV_DISABLE_COLLECTORS": @@ -423,12 +423,24 @@ func newConfigFromEnv() (*Config, error) { return nil, fmt.Errorf("invalid setting PGSCV_COLLECT_TOP_INDEX, value '%s', allowed only digits", value) } config.CollectTopIndex = collectTopIndex + case "PGSCV_TEST_DB_CONNECTION_ON_STARTUP": + config.TestDbConnectionOnStartup = toBool(value) } } - return config, nil } +func toBool(s string) bool { + switch s { + case "y", "yes", "Yes", "YES", "t", "true", "True", "TRUE", "1", "on": + return true + case "n", "no", "No", "NO", "f", "false", "False", "FALSE", "0", "off": + return false + default: + return false + } +} + // newDatabasesRegexp creates new regexp depending on passed string. func newDatabasesRegexp(s string) (*regexp.Regexp, error) { if s == "" { diff --git a/internal/pgscv/pgscv.go b/internal/pgscv/pgscv.go index 867c4d1..b1d7e6e 100644 --- a/internal/pgscv/pgscv.go +++ b/internal/pgscv/pgscv.go @@ -16,15 +16,16 @@ func Start(ctx context.Context, config *Config) error { serviceRepo := service.NewRepository() serviceConfig := service.Config{ - NoTrackMode: config.NoTrackMode, - ConnDefaults: config.Defaults, - ConnsSettings: config.ServicesConnsSettings, - DatabasesRE: config.DatabasesRE, - DisabledCollectors: config.DisableCollectors, - CollectorsSettings: config.CollectorsSettings, - CollectTopTable: config.CollectTopTable, - CollectTopIndex: config.CollectTopIndex, - CollectTopQuery: config.CollectTopQuery, + NoTrackMode: config.NoTrackMode, + ConnDefaults: config.Defaults, + ConnsSettings: config.ServicesConnsSettings, + DatabasesRE: config.DatabasesRE, + DisabledCollectors: config.DisableCollectors, + CollectorsSettings: config.CollectorsSettings, + CollectTopTable: config.CollectTopTable, + CollectTopIndex: config.CollectTopIndex, + CollectTopQuery: config.CollectTopQuery, + TestDbConnectionOnStartup: config.TestDbConnectionOnStartup, } if len(config.ServicesConnsSettings) == 0 { diff --git a/internal/service/service.go b/internal/service/service.go index 55f76d1..ebaa99f 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -41,10 +41,11 @@ type Config struct { DatabasesRE *regexp.Regexp DisabledCollectors []string // CollectorsSettings defines all collector settings propagated from main YAML configuration. - CollectorsSettings model.CollectorsSettings - CollectTopTable int - CollectTopIndex int - CollectTopQuery int + CollectorsSettings model.CollectorsSettings + CollectTopTable int + CollectTopIndex int + CollectTopQuery int + TestDbConnectionOnStartup bool } // Collector is an interface for prometheus.Collector. @@ -154,7 +155,12 @@ func (repo *Repository) addServicesFromConfig(config Config) { // Check connection using created *ConnConfig, go next if connection failed. db, err := store.NewWithConfig(pgconfig) if err != nil { - log.Warnf("%s: %s", cs.Conninfo, err) + if config.TestDbConnectionOnStartup { + log.Warnf("%s: %s skip", cs.Conninfo, err) + continue + } else { + log.Warnf("%s: %s", cs.Conninfo, err) + } } else { msg = fmt.Sprintf("service [%s] available through: %s@%s:%d/%s", k, pgconfig.User, pgconfig.Host, pgconfig.Port, pgconfig.Database) db.Close() diff --git a/internal/service/service_test.go b/internal/service/service_test.go index c234cf2..4b5d9dc 100644 --- a/internal/service/service_test.go +++ b/internal/service/service_test.go @@ -62,9 +62,12 @@ func TestRepository_addServicesFromConfig(t *testing.T) { }{ { name: "valid", - config: Config{ConnsSettings: ConnsSettings{ - "test": {ServiceType: model.ServiceTypePostgresql, Conninfo: "host=127.0.0.1 port=5432 user=pgscv dbname=pgscv_fixtures"}, - }}, + config: Config{ + TestDbConnectionOnStartup: true, + ConnsSettings: ConnsSettings{ + "test": {ServiceType: model.ServiceTypePostgresql, + Conninfo: "host=127.0.0.1 port=5432 user=pgscv dbname=pgscv_fixtures"}, + }}, expected: 2, }, { @@ -73,10 +76,28 @@ func TestRepository_addServicesFromConfig(t *testing.T) { expected: 1, }, { - name: "invalid service", - config: Config{ConnsSettings: ConnsSettings{"test": {ServiceType: model.ServiceTypePostgresql, Conninfo: "invalid conninfo"}}}, + name: "invalid service", + config: Config{ + TestDbConnectionOnStartup: true, + ConnsSettings: ConnsSettings{"test": {ServiceType: model.ServiceTypePostgresql, Conninfo: "invalid conninfo"}}}, expected: 1, }, + { + name: "unavailable service", + config: Config{ + TestDbConnectionOnStartup: true, + ConnsSettings: ConnsSettings{"test": {ServiceType: model.ServiceTypePostgresql, Conninfo: "port=1"}}, + }, + expected: 1, + }, + { + name: "unavailable service", + config: Config{ + TestDbConnectionOnStartup: false, + ConnsSettings: ConnsSettings{"test": {ServiceType: model.ServiceTypePostgresql, Conninfo: "port=1"}}, + }, + expected: 2, + }, } for _, tc := range testCases { From 4ea4467a9baf385d96a05740b6951165457aaf45 Mon Sep 17 00:00:00 2001 From: "Dmitry V. Bulashev" Date: Fri, 25 Oct 2024 09:06:18 +0500 Subject: [PATCH 4/7] fixup --- internal/pgscv/config_test.go | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/internal/pgscv/config_test.go b/internal/pgscv/config_test.go index 4e99ad2..a0b7ef4 100644 --- a/internal/pgscv/config_test.go +++ b/internal/pgscv/config_test.go @@ -585,7 +585,8 @@ func Test_newConfigFromEnv(t *testing.T) { Keyfile: "keyfile.key", Certfile: "certfile.cert", }, - Defaults: map[string]string{}, + Defaults: map[string]string{}, + TestDbConnectionOnStartup: true, }, }, { From 33dfc2295a538d34669d7a6c7b990e61e08a002e Mon Sep 17 00:00:00 2001 From: "Dmitry V. Bulashev" Date: Fri, 25 Oct 2024 09:36:10 +0500 Subject: [PATCH 5/7] fixup --- internal/pgscv/config_test.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/internal/pgscv/config_test.go b/internal/pgscv/config_test.go index a0b7ef4..d0fdb88 100644 --- a/internal/pgscv/config_test.go +++ b/internal/pgscv/config_test.go @@ -546,7 +546,11 @@ func Test_newConfigFromEnv(t *testing.T) { { valid: true, // No env variables envvars: map[string]string{}, - want: &Config{Defaults: map[string]string{}, ServicesConnsSettings: map[string]service.ConnSetting{}}, + want: &Config{ + Defaults: map[string]string{}, + ServicesConnsSettings: map[string]service.ConnSetting{}, + TestDbConnectionOnStartup: true, + }, }, { valid: true, // Completely valid variables From 6a5fe114ec2d312282f6e2b20d85d9be3f4ba506 Mon Sep 17 00:00:00 2001 From: Mikhail Grigorev Date: Tue, 29 Oct 2024 18:55:15 +0500 Subject: [PATCH 6/7] Replacing variable with skip_conn_error_mode --- internal/pgscv/config.go | 44 +++++++++++++++++--------------- internal/pgscv/config_test.go | 38 +++++++++++++-------------- internal/pgscv/pgscv.go | 23 +++++++++-------- internal/service/service.go | 16 ++++++------ internal/service/service_test.go | 31 ++++++++++++---------- 5 files changed, 79 insertions(+), 73 deletions(-) diff --git a/internal/pgscv/config.go b/internal/pgscv/config.go index 2d2f6e5..015bc3a 100644 --- a/internal/pgscv/config.go +++ b/internal/pgscv/config.go @@ -27,19 +27,19 @@ const ( // Config defines application's configuration. type Config struct { - NoTrackMode bool `yaml:"no_track_mode"` // controls tracking sensitive information (query texts, etc) - ListenAddress string `yaml:"listen_address"` // Network address and port where the application should listen on - ServicesConnsSettings service.ConnsSettings `yaml:"services"` // All connections settings for exact services - Defaults map[string]string `yaml:"defaults"` // Defaults - DisableCollectors []string `yaml:"disable_collectors"` // List of collectors which should be disabled. DEPRECATED in favor collectors settings - CollectorsSettings model.CollectorsSettings `yaml:"collectors"` // Collectors settings propagated from main YAML configuration - Databases string `yaml:"databases"` // Regular expression string specifies databases from which metrics should be collected - DatabasesRE *regexp.Regexp // Regular expression object compiled from Databases - AuthConfig http.AuthConfig `yaml:"authentication"` // TLS and Basic auth configuration - CollectTopTable int `yaml:"collect_top_table"` // Limit elements on Table collector - CollectTopIndex int `yaml:"collect_top_index"` // Limit elements on Indexes collector - CollectTopQuery int `yaml:"collect_top_query"` // Limit elements on Statements collector - TestDbConnectionOnStartup bool `yaml:"test_db_connection_on_startup"` // Check connection settings and try to connect using them. In case of failure, don't create a Service instance. + NoTrackMode bool `yaml:"no_track_mode"` // controls tracking sensitive information (query texts, etc) + ListenAddress string `yaml:"listen_address"` // Network address and port where the application should listen on + ServicesConnsSettings service.ConnsSettings `yaml:"services"` // All connections settings for exact services + Defaults map[string]string `yaml:"defaults"` // Defaults + DisableCollectors []string `yaml:"disable_collectors"` // List of collectors which should be disabled. DEPRECATED in favor collectors settings + CollectorsSettings model.CollectorsSettings `yaml:"collectors"` // Collectors settings propagated from main YAML configuration + Databases string `yaml:"databases"` // Regular expression string specifies databases from which metrics should be collected + DatabasesRE *regexp.Regexp // Regular expression object compiled from Databases + AuthConfig http.AuthConfig `yaml:"authentication"` // TLS and Basic auth configuration + CollectTopTable int `yaml:"collect_top_table"` // Limit elements on Table collector + CollectTopIndex int `yaml:"collect_top_index"` // Limit elements on Indexes collector + CollectTopQuery int `yaml:"collect_top_query"` // Limit elements on Statements collector + SkipConnErrorMode bool `yaml:"skip_conn_error_mode"` // Skipping connection errors and creating a Service instance. } // NewConfig creates new config based on config file or return default config if config file is not specified. @@ -65,7 +65,6 @@ func NewConfig(configFilePath string) (*Config, error) { // Get configuration from environment variables configFromEnv, err := newConfigFromEnv() - fmt.Println(configFromEnv.TestDbConnectionOnStartup) if err != nil { return nil, err } @@ -110,8 +109,8 @@ func NewConfig(configFilePath string) (*Config, error) { if configFromEnv.CollectTopQuery > 0 { configFromFile.CollectTopQuery = configFromEnv.CollectTopQuery } - if !configFromEnv.TestDbConnectionOnStartup { - configFromFile.TestDbConnectionOnStartup = false + if configFromEnv.SkipConnErrorMode { + configFromFile.SkipConnErrorMode = configFromEnv.SkipConnErrorMode } return configFromFile, nil } @@ -183,6 +182,10 @@ func (c *Config) Validate() error { log.Infoln("no-track disabled, for details check the documentation about 'no_track_mode' option.") } + if c.SkipConnErrorMode { + log.Infoln("skipping connection errors is enabled.") + } + // setup defaults if c.Defaults == nil { c.Defaults = map[string]string{} @@ -340,9 +343,8 @@ func newConfigFromEnv() (*Config, error) { log.Infoln("read configuration from environment") config := &Config{ - Defaults: map[string]string{}, - ServicesConnsSettings: map[string]service.ConnSetting{}, - TestDbConnectionOnStartup: true, + Defaults: map[string]string{}, + ServicesConnsSettings: map[string]service.ConnSetting{}, } for _, env := range os.Environ() { @@ -423,8 +425,8 @@ func newConfigFromEnv() (*Config, error) { return nil, fmt.Errorf("invalid setting PGSCV_COLLECT_TOP_INDEX, value '%s', allowed only digits", value) } config.CollectTopIndex = collectTopIndex - case "PGSCV_TEST_DB_CONNECTION_ON_STARTUP": - config.TestDbConnectionOnStartup = toBool(value) + case "PGSCV_SKIP_CONN_ERROR_MODE": + config.SkipConnErrorMode = toBool(value) } } return config, nil diff --git a/internal/pgscv/config_test.go b/internal/pgscv/config_test.go index d0fdb88..c7ce928 100644 --- a/internal/pgscv/config_test.go +++ b/internal/pgscv/config_test.go @@ -547,28 +547,28 @@ func Test_newConfigFromEnv(t *testing.T) { valid: true, // No env variables envvars: map[string]string{}, want: &Config{ - Defaults: map[string]string{}, - ServicesConnsSettings: map[string]service.ConnSetting{}, - TestDbConnectionOnStartup: true, + Defaults: map[string]string{}, + ServicesConnsSettings: map[string]service.ConnSetting{}, }, }, { valid: true, // Completely valid variables envvars: map[string]string{ - "PGSCV_LISTEN_ADDRESS": "127.0.0.1:12345", - "PGSCV_NO_TRACK_MODE": "yes", - "PGSCV_DATABASES": "exampledb", - "PGSCV_DISABLE_COLLECTORS": "example/1,example/2, example/3", - "POSTGRES_DSN": "example_dsn", - "POSTGRES_DSN_EXAMPLE1": "example_dsn", - "PGBOUNCER_DSN": "example_dsn", - "PGBOUNCER_DSN_EXAMPLE2": "example_dsn", - "PATRONI_URL": "example_url", - "PATRONI_URL_EXAMPLE3": "example_url", - "PGSCV_AUTH_USERNAME": "user", - "PGSCV_AUTH_PASSWORD": "pass", - "PGSCV_AUTH_KEYFILE": "keyfile.key", - "PGSCV_AUTH_CERTFILE": "certfile.cert", + "PGSCV_LISTEN_ADDRESS": "127.0.0.1:12345", + "PGSCV_NO_TRACK_MODE": "yes", + "PGSCV_DATABASES": "exampledb", + "PGSCV_DISABLE_COLLECTORS": "example/1,example/2, example/3", + "POSTGRES_DSN": "example_dsn", + "POSTGRES_DSN_EXAMPLE1": "example_dsn", + "PGBOUNCER_DSN": "example_dsn", + "PGBOUNCER_DSN_EXAMPLE2": "example_dsn", + "PATRONI_URL": "example_url", + "PATRONI_URL_EXAMPLE3": "example_url", + "PGSCV_AUTH_USERNAME": "user", + "PGSCV_AUTH_PASSWORD": "pass", + "PGSCV_AUTH_KEYFILE": "keyfile.key", + "PGSCV_AUTH_CERTFILE": "certfile.cert", + "PGSCV_SKIP_CONN_ERROR_MODE": "yes", }, want: &Config{ ListenAddress: "127.0.0.1:12345", @@ -589,8 +589,8 @@ func Test_newConfigFromEnv(t *testing.T) { Keyfile: "keyfile.key", Certfile: "certfile.cert", }, - Defaults: map[string]string{}, - TestDbConnectionOnStartup: true, + Defaults: map[string]string{}, + SkipConnErrorMode: true, }, }, { diff --git a/internal/pgscv/pgscv.go b/internal/pgscv/pgscv.go index b1d7e6e..a8d9c41 100644 --- a/internal/pgscv/pgscv.go +++ b/internal/pgscv/pgscv.go @@ -3,10 +3,11 @@ package pgscv import ( "context" "errors" + "sync" + "github.com/cherts/pgscv/internal/http" "github.com/cherts/pgscv/internal/log" "github.com/cherts/pgscv/internal/service" - "sync" ) // Start is the application's starting point. @@ -16,16 +17,16 @@ func Start(ctx context.Context, config *Config) error { serviceRepo := service.NewRepository() serviceConfig := service.Config{ - NoTrackMode: config.NoTrackMode, - ConnDefaults: config.Defaults, - ConnsSettings: config.ServicesConnsSettings, - DatabasesRE: config.DatabasesRE, - DisabledCollectors: config.DisableCollectors, - CollectorsSettings: config.CollectorsSettings, - CollectTopTable: config.CollectTopTable, - CollectTopIndex: config.CollectTopIndex, - CollectTopQuery: config.CollectTopQuery, - TestDbConnectionOnStartup: config.TestDbConnectionOnStartup, + NoTrackMode: config.NoTrackMode, + ConnDefaults: config.Defaults, + ConnsSettings: config.ServicesConnsSettings, + DatabasesRE: config.DatabasesRE, + DisabledCollectors: config.DisableCollectors, + CollectorsSettings: config.CollectorsSettings, + CollectTopTable: config.CollectTopTable, + CollectTopIndex: config.CollectTopIndex, + CollectTopQuery: config.CollectTopQuery, + SkipConnErrorMode: config.SkipConnErrorMode, } if len(config.ServicesConnsSettings) == 0 { diff --git a/internal/service/service.go b/internal/service/service.go index ebaa99f..8caedf3 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -41,11 +41,11 @@ type Config struct { DatabasesRE *regexp.Regexp DisabledCollectors []string // CollectorsSettings defines all collector settings propagated from main YAML configuration. - CollectorsSettings model.CollectorsSettings - CollectTopTable int - CollectTopIndex int - CollectTopQuery int - TestDbConnectionOnStartup bool + CollectorsSettings model.CollectorsSettings + CollectTopTable int + CollectTopIndex int + CollectTopQuery int + SkipConnErrorMode bool } // Collector is an interface for prometheus.Collector. @@ -155,11 +155,11 @@ func (repo *Repository) addServicesFromConfig(config Config) { // Check connection using created *ConnConfig, go next if connection failed. db, err := store.NewWithConfig(pgconfig) if err != nil { - if config.TestDbConnectionOnStartup { + if config.SkipConnErrorMode { + log.Warnf("%s: %s", cs.Conninfo, err) + } else { log.Warnf("%s: %s skip", cs.Conninfo, err) continue - } else { - log.Warnf("%s: %s", cs.Conninfo, err) } } else { msg = fmt.Sprintf("service [%s] available through: %s@%s:%d/%s", k, pgconfig.User, pgconfig.Host, pgconfig.Port, pgconfig.Database) diff --git a/internal/service/service_test.go b/internal/service/service_test.go index 4b5d9dc..013a9a5 100644 --- a/internal/service/service_test.go +++ b/internal/service/service_test.go @@ -1,10 +1,11 @@ package service import ( + "testing" + "github.com/cherts/pgscv/internal/model" "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/assert" - "testing" ) func TestRepository_addService(t *testing.T) { @@ -63,13 +64,23 @@ func TestRepository_addServicesFromConfig(t *testing.T) { { name: "valid", config: Config{ - TestDbConnectionOnStartup: true, + SkipConnErrorMode: false, ConnsSettings: ConnsSettings{ "test": {ServiceType: model.ServiceTypePostgresql, Conninfo: "host=127.0.0.1 port=5432 user=pgscv dbname=pgscv_fixtures"}, }}, expected: 2, }, + { + name: "valid_skip_error", + config: Config{ + SkipConnErrorMode: true, + ConnsSettings: ConnsSettings{ + "test": {ServiceType: model.ServiceTypePostgresql, + Conninfo: "host=127.0.0.1 port=5430 user=pgscv dbname=pgscv_fixtures"}, + }}, + expected: 2, + }, { name: "empty conn settings", config: Config{}, @@ -78,26 +89,18 @@ func TestRepository_addServicesFromConfig(t *testing.T) { { name: "invalid service", config: Config{ - TestDbConnectionOnStartup: true, - ConnsSettings: ConnsSettings{"test": {ServiceType: model.ServiceTypePostgresql, Conninfo: "invalid conninfo"}}}, + SkipConnErrorMode: true, + ConnsSettings: ConnsSettings{"test": {ServiceType: model.ServiceTypePostgresql, Conninfo: "invalid conninfo"}}}, expected: 1, }, { name: "unavailable service", config: Config{ - TestDbConnectionOnStartup: true, - ConnsSettings: ConnsSettings{"test": {ServiceType: model.ServiceTypePostgresql, Conninfo: "port=1"}}, + SkipConnErrorMode: false, + ConnsSettings: ConnsSettings{"test": {ServiceType: model.ServiceTypePostgresql, Conninfo: "port=1"}}, }, expected: 1, }, - { - name: "unavailable service", - config: Config{ - TestDbConnectionOnStartup: false, - ConnsSettings: ConnsSettings{"test": {ServiceType: model.ServiceTypePostgresql, Conninfo: "port=1"}}, - }, - expected: 2, - }, } for _, tc := range testCases { From 618cfa3c838712a85394a434bf7acac8f93f34ca Mon Sep 17 00:00:00 2001 From: Mikhail Grigorev Date: Tue, 29 Oct 2024 18:56:58 +0500 Subject: [PATCH 7/7] Replacing variable with skip_conn_error_mode --- deploy/demo-lab/.env | 2 +- deploy/demo-lab/pgscv/pgscv.yaml | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/deploy/demo-lab/.env b/deploy/demo-lab/.env index 18a53d1..53de3d9 100644 --- a/deploy/demo-lab/.env +++ b/deploy/demo-lab/.env @@ -10,7 +10,7 @@ LOG_LEVEL=debug #PGSCV_COLLECT_TOP_QUERY=15 #PGSCV_COLLECT_TOP_TABLE=15 #PGSCV_COLLECT_TOP_INDEX=15 -#PGSCV_TEST_DB_CONNECTION_ON_STARTUP=f +#PGSCV_SKIP_CONN_ERROR_MODE=t #POSTGRES_DSN_db1=postgresql://pgscv:pgscv@host1:5432/pgbench1 #POSTGRES_DSN_db2=postgresql://pgscv:pgscv@host1:5432/pgbench2 #PGSCV_DATABASES="^pgbench.*$" diff --git a/deploy/demo-lab/pgscv/pgscv.yaml b/deploy/demo-lab/pgscv/pgscv.yaml index 6413f52..9626b05 100644 --- a/deploy/demo-lab/pgscv/pgscv.yaml +++ b/deploy/demo-lab/pgscv/pgscv.yaml @@ -8,7 +8,7 @@ no_track_mode: false #collect_top_query: 10 #collect_top_table: 10 #collect_top_index: 10 -#test_db_connection_on_startup: false +#skip_conn_error_mode: true services: "postgres12": service_type: "postgres"