-
Notifications
You must be signed in to change notification settings - Fork 45
Commit
This commit does not belong to any branch on this repository, and may belong to a fork outside of the repository.
Merge pull request #320 from kinvolk/enhance_metrics
Enhance application metrics and Add http endpoint level metrics
- Loading branch information
Showing
7 changed files
with
168 additions
and
77 deletions.
There are no files selected for viewing
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
| Original file line number | Diff line number | Diff line change |
|---|---|---|
| @@ -0,0 +1,117 @@ | ||
| package main | ||
|
|
||
| import ( | ||
| "fmt" | ||
| "os" | ||
| "time" | ||
|
|
||
| "github.com/prometheus/client_golang/prometheus" | ||
| ) | ||
|
|
||
| const ( | ||
| defaultMetricsUpdateInterval = 5 * time.Second | ||
| ) | ||
|
|
||
| var ( | ||
| appInstancePerChannelGaugeMetric = prometheus.NewGaugeVec( | ||
| prometheus.GaugeOpts{ | ||
| Namespace: "nebraska", | ||
| Name: "application_instances_per_channel", | ||
| Help: "Number of applications from specific channel running on instances", | ||
| }, | ||
| []string{ | ||
| "application", | ||
| "version", | ||
| "channel", | ||
| }, | ||
| ) | ||
|
|
||
| failedUpdatesGaugeMetric = prometheus.NewGaugeVec( | ||
| prometheus.GaugeOpts{ | ||
| Namespace: "nebraska", | ||
| Name: "failed_updates", | ||
| Help: "Number of failed updates of an application", | ||
| }, | ||
| []string{ | ||
| "application", | ||
| }, | ||
| ) | ||
| ) | ||
|
|
||
| // registerNebraskaMetrics registers the application metrics collector with the DefaultRegistrer. | ||
| func registerNebraskaMetrics() error { | ||
| err := prometheus.Register(appInstancePerChannelGaugeMetric) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| err = prometheus.Register(failedUpdatesGaugeMetric) | ||
| if err != nil { | ||
| return err | ||
| } | ||
| return nil | ||
| } | ||
|
|
||
| // getMetricsRefreshInterval returns the metrics update Interval key is set in the environment as time.Duration, | ||
| // NEBRASKA_METRICS_UPDATE_INTERVAL. The variable must be a string acceptable by time.ParseDuration | ||
| // If not returns the default update interval. | ||
| func getMetricsRefreshInterval() time.Duration { | ||
| refreshIntervalEnvValue := os.Getenv("NEBRASKA_METRICS_UPDATE_INTERVAL") | ||
| if refreshIntervalEnvValue == "" { | ||
| return defaultMetricsUpdateInterval | ||
| } | ||
|
|
||
| refreshInterval, err := time.ParseDuration(refreshIntervalEnvValue) | ||
| if err != nil || refreshInterval <= 0 { | ||
| logger.Warn("invalid NEBRASKA_METRICS_UPDATE_INTERVAL value, it must be acceptable by time.ParseDuration and positive", "value", refreshIntervalEnvValue) | ||
| return defaultMetricsUpdateInterval | ||
| } | ||
| return refreshInterval | ||
| } | ||
|
|
||
| // registerAndInstrumentMetrics registers the application metrics and instruments them in configurable intervals. | ||
| func registerAndInstrumentMetrics(ctl *controller) error { | ||
| // register application metrics | ||
| err := registerNebraskaMetrics() | ||
| if err != nil { | ||
| return err | ||
| } | ||
|
|
||
| refreshInterval := getMetricsRefreshInterval() | ||
|
|
||
| metricsTicker := time.Tick(refreshInterval) | ||
|
|
||
| go func() { | ||
| for { | ||
| <-metricsTicker | ||
| err := calculateMetrics(ctl) | ||
| if err != nil { | ||
| logger.Error("registerAndInstrumentMetrics updating the metrics", "error", err.Error()) | ||
| } | ||
| } | ||
| }() | ||
|
|
||
| return nil | ||
| } | ||
|
|
||
| // calculateMetrics calculates the application metrics and updates the respective metric. | ||
| func calculateMetrics(ctl *controller) error { | ||
| aipcMetrics, err := ctl.api.GetAppInstancesPerChannelMetrics() | ||
| if err != nil { | ||
| return fmt.Errorf("failed to get app instances per channel metrics: %w", err) | ||
| } | ||
|
|
||
| for _, metric := range aipcMetrics { | ||
| appInstancePerChannelGaugeMetric.WithLabelValues(metric.ApplicationName, metric.Version, metric.ChannelName).Set(float64(metric.InstancesCount)) | ||
| } | ||
|
|
||
| fuMetrics, err := ctl.api.GetFailedUpdatesMetrics() | ||
| if err != nil { | ||
| return fmt.Errorf("failed to get failed update metrics: %w", err) | ||
| } | ||
|
|
||
| for _, metric := range fuMetrics { | ||
| failedUpdatesGaugeMetric.WithLabelValues(metric.ApplicationName).Set(float64(metric.FailureCount)) | ||
| } | ||
|
|
||
| return nil | ||
| } |
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
This file contains bidirectional Unicode text that may be interpreted or compiled differently than what appears below. To review, open the file in an editor that reveals hidden Unicode characters.
Learn more about bidirectional Unicode characters
Oops, something went wrong.