diff --git a/cmd/pgscv.go b/cmd/pgscv.go index 6e9b5b4..1fead64 100644 --- a/cmd/pgscv.go +++ b/cmd/pgscv.go @@ -8,6 +8,7 @@ import ( "gopkg.in/alecthomas/kingpin.v2" "os" "os/signal" + "runtime/pprof" "syscall" ) @@ -38,6 +39,20 @@ func main() { os.Exit(1) } + if config.CpuProfile != "" { + f, err := os.Create(config.CpuProfile) + if err != nil { + log.Errorln(err) + os.Exit(1) + } + err = pprof.StartCPUProfile(f) + if err != nil { + log.Errorln(err) + os.Exit(1) + } + defer pprof.StopCPUProfile() + } + if err := config.Validate(); err != nil { log.Errorln("validate config failed: ", err) os.Exit(1) diff --git a/internal/collector/collector.go b/internal/collector/collector.go index f5a6738..447fc77 100644 --- a/internal/collector/collector.go +++ b/internal/collector/collector.go @@ -1,7 +1,12 @@ package collector import ( + "fmt" + "os" + "runtime" + "runtime/pprof" "sync" + "time" "github.com/cherts/pgscv/internal/filter" "github.com/cherts/pgscv/internal/log" @@ -201,13 +206,37 @@ func (n PgscvCollector) Collect(out chan<- prometheus.Metric) { pipelineIn := make(chan prometheus.Metric) // Run collectors. - wgCollector.Add(len(n.Collectors)) - for name, c := range n.Collectors { - go func(name string, c Collector) { + wgCollector.Add(1) + go func(n PgscvCollector) { + defer wgCollector.Done() + for name, c := range n.Collectors { + var m runtime.MemStats + var start time.Time + if n.Config.LogCollectorStatistics { + start = time.Now() + runtime.ReadMemStats(&m) + log.Info(fmt.Sprintf("Start Collector: %s, Alloc = %v MiB", name, m.Alloc/1024/1024)) + } collect(name, n.Config, c, pipelineIn) - wgCollector.Done() - }(name, c) - } + runtime.GC() + if n.Config.LogCollectorStatistics { + runtime.ReadMemStats(&m) + log.Info(fmt.Sprintf("Stop Collector: %s, Alloc = %v MiB, duration = %s", name, m.Alloc/1024/1024, time.Since(start))) + } + } + if n.Config.MemProfile != "" { + f, err := os.Create(n.Config.MemProfile) + if err != nil { + log.Errorln(err) + os.Exit(1) + } + if pprof.WriteHeapProfile(f) != nil { + log.Errorln(err) + os.Exit(1) + } + _ = f.Close() + } + }(n) // Run sender. wgSender.Add(1) diff --git a/internal/collector/config.go b/internal/collector/config.go index 509bb7f..c41b017 100644 --- a/internal/collector/config.go +++ b/internal/collector/config.go @@ -31,6 +31,10 @@ type Config struct { DatabasesRE *regexp.Regexp // Settings defines collectors settings propagated from main YAML configuration. Settings model.CollectorsSettings + // path to store heap profile + MemProfile string + // logging: collecting duration, mem allocation + LogCollectorStatistics bool } // postgresServiceConfig defines Postgres-specific stuff required during collecting Postgres metrics. diff --git a/internal/pgscv/config.go b/internal/pgscv/config.go index 87e1772..e35c48c 100644 --- a/internal/pgscv/config.go +++ b/internal/pgscv/config.go @@ -26,15 +26,18 @@ const ( // Config defines application's configuration. type Config struct { - NoTrackMode bool `yaml:"no_track_mode"` // controls tracking sensitive information (query texts, etc) - ListenAddress string `yaml:"listen_address"` // Network address and port where the application should listen on - ServicesConnsSettings service.ConnsSettings `yaml:"services"` // All connections settings for exact services - Defaults map[string]string `yaml:"defaults"` // Defaults - DisableCollectors []string `yaml:"disable_collectors"` // List of collectors which should be disabled. DEPRECATED in favor collectors settings - CollectorsSettings model.CollectorsSettings `yaml:"collectors"` // Collectors settings propagated from main YAML configuration - Databases string `yaml:"databases"` // Regular expression string specifies databases from which metrics should be collected - DatabasesRE *regexp.Regexp // Regular expression object compiled from Databases - AuthConfig http.AuthConfig `yaml:"authentication"` // TLS and Basic auth configuration + NoTrackMode bool `yaml:"no_track_mode"` // controls tracking sensitive information (query texts, etc) + ListenAddress string `yaml:"listen_address"` // Network address and port where the application should listen on + ServicesConnsSettings service.ConnsSettings `yaml:"services"` // All connections settings for exact services + Defaults map[string]string `yaml:"defaults"` // Defaults + DisableCollectors []string `yaml:"disable_collectors"` // List of collectors which should be disabled. DEPRECATED in favor collectors settings + CollectorsSettings model.CollectorsSettings `yaml:"collectors"` // Collectors settings propagated from main YAML configuration + Databases string `yaml:"databases"` // Regular expression string specifies databases from which metrics should be collected + DatabasesRE *regexp.Regexp // Regular expression object compiled from Databases + AuthConfig http.AuthConfig `yaml:"authentication"` // TLS and Basic auth configuration + CpuProfile string `yaml:"cpu_profile"` // path to store cpu profile + MemProfile string `yaml:"mem_profile"` // path to store heap profile + LogCollectorStatistics bool `yaml:"log_collector_statistics"` // logging: collecting duration, mem allocation } // NewConfig creates new config based on config file or return default config if config file is not specified. @@ -94,6 +97,15 @@ func NewConfig(configFilePath string) (*Config, error) { if configFromEnv.AuthConfig != (http.AuthConfig{}) { configFromFile.AuthConfig = configFromEnv.AuthConfig } + if configFromEnv.MemProfile != "" { + configFromFile.MemProfile = configFromEnv.MemProfile + } + if configFromEnv.CpuProfile != "" { + configFromFile.CpuProfile = configFromEnv.CpuProfile + } + if configFromEnv.LogCollectorStatistics { + configFromFile.LogCollectorStatistics = configFromEnv.LogCollectorStatistics + } return configFromFile, nil } @@ -226,6 +238,15 @@ func (c *Config) Validate() error { c.AuthConfig.EnableAuth = enableAuth c.AuthConfig.EnableTLS = enableTLS + if c.LogCollectorStatistics { + log.Infoln("Enabled LogCollectorStatistics") + } + if c.MemProfile != "" { + log.Infoln("Enabled MemProfile") + } + if c.CpuProfile != "" { + log.Infoln("Enabled CpuProfile") + } return nil } @@ -370,6 +391,17 @@ func newConfigFromEnv() (*Config, error) { config.AuthConfig.Keyfile = value case "PGSCV_AUTH_CERTFILE": config.AuthConfig.Certfile = value + case "PGSCV_CPUPROFILE_FILE": + config.CpuProfile = value + case "PGSCV_MEMPROFILE_FILE": + config.MemProfile = value + case "PGSCV_LOG_COLLECTOR": + switch value { + case "y", "yes", "Yes", "YES", "t", "true", "True", "TRUE", "1", "on": + config.LogCollectorStatistics = true + default: + config.LogCollectorStatistics = false + } } } diff --git a/internal/pgscv/pgscv.go b/internal/pgscv/pgscv.go index 56dfa7f..f1d070a 100644 --- a/internal/pgscv/pgscv.go +++ b/internal/pgscv/pgscv.go @@ -14,14 +14,15 @@ func Start(ctx context.Context, config *Config) error { log.Debug("start application") serviceRepo := service.NewRepository() - serviceConfig := service.Config{ - NoTrackMode: config.NoTrackMode, - ConnDefaults: config.Defaults, - ConnsSettings: config.ServicesConnsSettings, - DatabasesRE: config.DatabasesRE, - DisabledCollectors: config.DisableCollectors, - CollectorsSettings: config.CollectorsSettings, + NoTrackMode: config.NoTrackMode, + ConnDefaults: config.Defaults, + ConnsSettings: config.ServicesConnsSettings, + DatabasesRE: config.DatabasesRE, + DisabledCollectors: config.DisableCollectors, + CollectorsSettings: config.CollectorsSettings, + MemProfile: config.MemProfile, + LogCollectorStatistics: config.LogCollectorStatistics, } if len(config.ServicesConnsSettings) == 0 { diff --git a/internal/service/service.go b/internal/service/service.go index 5846e8c..a4e81ec 100644 --- a/internal/service/service.go +++ b/internal/service/service.go @@ -33,7 +33,6 @@ type Service struct { // Config defines service's configuration. type Config struct { - RuntimeMode int NoTrackMode bool ConnDefaults map[string]string `yaml:"defaults"` // Defaults ConnsSettings ConnsSettings @@ -42,6 +41,10 @@ type Config struct { DisabledCollectors []string // CollectorsSettings defines all collector settings propagated from main YAML configuration. CollectorsSettings model.CollectorsSettings + // path to store heap profile + MemProfile string + // logging: collecting duration, mem allocation + LogCollectorStatistics bool } // Collector is an interface for prometheus.Collector. @@ -184,11 +187,13 @@ func (repo *Repository) setupServices(config Config) error { if service.Collector == nil { factories := collector.Factories{} collectorConfig := collector.Config{ - NoTrackMode: config.NoTrackMode, - ServiceType: service.ConnSettings.ServiceType, - ConnString: service.ConnSettings.Conninfo, - Settings: config.CollectorsSettings, - DatabasesRE: config.DatabasesRE, + NoTrackMode: config.NoTrackMode, + ServiceType: service.ConnSettings.ServiceType, + ConnString: service.ConnSettings.Conninfo, + Settings: config.CollectorsSettings, + DatabasesRE: config.DatabasesRE, + MemProfile: config.MemProfile, + LogCollectorStatistics: config.LogCollectorStatistics, } switch service.ConnSettings.ServiceType {