From 71c4b21f8387669d0e98b4ec4ab073476fcfdaf8 Mon Sep 17 00:00:00 2001 From: yolossn Date: Fri, 8 Jan 2021 13:24:27 +0530 Subject: [PATCH 1/4] backend: Remove existing metrics endpoint Currently we generate the metrics using fmt functions, the ideal way is to use the prometheus client_golang package. Also calculating the metrics in the request-response cycle can lead to timeouts. Signed-off-by: yolossn --- cmd/nebraska/controller.go | 63 -------------------------------------- cmd/nebraska/nebraska.go | 6 ---- 2 files changed, 69 deletions(-) diff --git a/cmd/nebraska/controller.go b/cmd/nebraska/controller.go index 3ac59d6f9..abffb99a3 100644 --- a/cmd/nebraska/controller.go +++ b/cmd/nebraska/controller.go @@ -806,69 +806,6 @@ func (ctl *controller) getActivity(c *gin.Context) { } } -// ---------------------------------------------------------------------------- -// Metrics -// - -const ( - appInstancesPerChannelMetricsProlog = `# HELP nebraska_application_instances_per_channel A number of applications from specific channel running on instances -# TYPE nebraska_application_instances_per_channel gauge` - failedUpdatesMetricsProlog = `# HELP nebraska_failed_updates A number of failed updates of an application -# TYPE nebraska_failed_updates gauge` -) - -func escapeMetricString(str string) string { - str = strings.Replace(str, `\`, `\\`, -1) - str = strings.Replace(str, `"`, `\"`, -1) - str = strings.Replace(str, "\n", `\n`, -1) - return str -} - -func (ctl *controller) getMetrics(c *gin.Context) { - teamID := c.GetString("team_id") - - nowUnixMillis := time.Now().Unix() * 1000 - aipcMetrics, err := ctl.api.GetAppInstancesPerChannelMetrics(teamID) - if err != nil { - logger.Error("getMetrics - getting app instances per channel metrics", "error", err.Error(), "teamID", teamID) - httpError(c, http.StatusBadRequest) - return - } - fuMetrics, err := ctl.api.GetFailedUpdatesMetrics(teamID) - if err != nil { - logger.Error("getMetrics - getting failed updates metrics", "error", err.Error(), "teamID", teamID) - httpError(c, http.StatusBadRequest) - return - } - - // "version" specifies a version of prometheus text file - // format. For details see: - // - // https://github.com/prometheus/docs/blob/master/content/docs/instrumenting/exposition_formats.md#basic-info - c.Writer.Header().Set("Content-Type", "text/plain; version=0.0.4") - c.Writer.WriteHeader(http.StatusOK) - needEmptyLine := false - if len(aipcMetrics) > 0 { - if needEmptyLine { - fmt.Fprintf(c.Writer, "\n") - } - fmt.Fprintf(c.Writer, "%s\n", appInstancesPerChannelMetricsProlog) - for _, metric := range aipcMetrics { - fmt.Fprintf(c.Writer, `nebraska_application_instances_per_channel{application="%s",version="%s",channel="%s"} %d %d%s`, escapeMetricString(metric.ApplicationName), escapeMetricString(metric.Version), escapeMetricString(metric.ChannelName), metric.InstancesCount, nowUnixMillis, "\n") - } - needEmptyLine = true - } - if len(fuMetrics) > 0 { - if needEmptyLine { - fmt.Fprintf(c.Writer, "\n") - } - fmt.Fprintf(c.Writer, "%s\n", failedUpdatesMetricsProlog) - for _, metric := range fuMetrics { - fmt.Fprintf(c.Writer, `nebraska_failed_updates{application="%s"} %d %d%s`, escapeMetricString(metric.ApplicationName), metric.FailureCount, nowUnixMillis, "\n") - } - } -} - // ---------------------------------------------------------------------------- // OMAHA server // diff --git a/cmd/nebraska/nebraska.go b/cmd/nebraska/nebraska.go index e705db221..9b20ce743 100644 --- a/cmd/nebraska/nebraska.go +++ b/cmd/nebraska/nebraska.go @@ -303,12 +303,6 @@ func setupRoutes(ctl *controller, httpLog bool) *gin.Engine { flatcarPkgsRouter.Static("/", *flatcarPackagesPath) } - // Metrics - metricsRouter := wrappedEngine.Group("/metrics", "metrics") - setupRouter(metricsRouter, "metrics", httpLog) - metricsRouter.Use(ctl.authenticate) - metricsRouter.GET("/", ctl.getMetrics) - // Serve frontend static content staticRouter := wrappedEngine.Group("/", "static") staticRouter.Use(ctl.authenticate) From 0e022c43aa81f51bf698913b414df9056f7ae7ea Mon Sep 17 00:00:00 2001 From: yolossn Date: Fri, 8 Jan 2021 13:50:13 +0530 Subject: [PATCH 2/4] backend: Add gin-prometheus middleware The middleware adds endpoint level instrumentation that provides metrics like total request count, request duration, etc. These metrics will be useful for identifying endpoints which don't meet the performance requirements. Signed-off-by: yolossn --- cmd/nebraska/nebraska.go | 16 ++++++++++++++++ go.mod | 1 + go.sum | 19 +++++++++++++++++++ 3 files changed, 36 insertions(+) diff --git a/cmd/nebraska/nebraska.go b/cmd/nebraska/nebraska.go index 9b20ce743..d4649257e 100644 --- a/cmd/nebraska/nebraska.go +++ b/cmd/nebraska/nebraska.go @@ -12,6 +12,7 @@ import ( "strings" "time" + "github.com/Depado/ginprom" "github.com/gin-gonic/gin" log "github.com/mgutz/logxi/v1" @@ -230,7 +231,22 @@ func setupRoutes(ctl *controller, httpLog bool) *gin.Engine { if httpLog { setupRequestLifetimeLogging(engine) } + + // Setup Middlewares + + // Recovery middleware to recover from panics engine.Use(gin.Recovery()) + + // Prometheus Metrics Middleware + setupRouter(engine, "top", httpLog) + p := ginprom.New( + ginprom.Engine(engine), + ginprom.Namespace("nebraska"), + ginprom.Subsystem("gin"), + ginprom.Path("/metrics"), + ) + engine.Use(p.Instrument()) + setupRouter(engine, "top", httpLog) wrappedEngine := wrapRouter(engine, httpLog) diff --git a/go.mod b/go.mod index 5fa214d94..a904c96af 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.13 replace github.com/coreos/go-omaha => github.com/kinvolk/go-omaha v0.0.0-20201126073937-801f305b087b require ( + github.com/Depado/ginprom v1.5.0 github.com/blang/semver v3.5.1+incompatible github.com/coreos/go-omaha v0.0.0 // replaced with github.com/kinvolk/go-omaha github.com/doug-martin/goqu/v9 v9.10.0 diff --git a/go.sum b/go.sum index dc66e9944..13ceb2e3b 100644 --- a/go.sum +++ b/go.sum @@ -36,6 +36,8 @@ github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/DATA-DOG/go-sqlmock v1.3.3 h1:CWUqKXe0s8A2z6qCgkP4Kru7wC11YoAnoupUKFDnH08= github.com/DATA-DOG/go-sqlmock v1.3.3/go.mod h1:f/Ixk793poVmq4qj/V1dPUg2JEAKC73Q5eFN3EC/SaM= +github.com/Depado/ginprom v1.5.0 h1:eOn6F20eNe7FSXFOaTaJNJf3aaCJp1V/tR0FXHssWjU= +github.com/Depado/ginprom v1.5.0/go.mod h1:h9T9g3Cc7ThDS8vZhAiGlh31F8Hu46/E+TsD5rFHQo8= github.com/Knetic/govaluate v3.0.1-0.20171022003610-9aa49832a739+incompatible/go.mod h1:r7JcOSlj0wfOMncg0iLm8Leh48TZaKVeNIfJntJ2wa0= github.com/OneOfOne/xxhash v1.2.2/go.mod h1:HSdplMjZKSmBqAxg5vPj2TmRDmfkzw+cTzAElWljhcU= github.com/OpenPeeDeeP/depguard v1.0.1 h1:VlW4R6jmBIv3/u1JNlawEvJMM4J+dPORPaZasQee8Us= @@ -51,6 +53,8 @@ github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRF github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0= github.com/apache/thrift v0.12.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= github.com/apache/thrift v0.13.0/go.mod h1:cp2SuWMxlEZw2r+iP2GNCdIi4C1qmUzdZFSVb+bacwQ= +github.com/appleboy/gofight/v2 v2.1.2 h1:VOy3jow4vIK8BRQJoC/I9muxyYlJ2yb9ht2hZoS3rf4= +github.com/appleboy/gofight/v2 v2.1.2/go.mod h1:frW+U1QZEdDgixycTj4CygQ48yLTUhplt43+Wczp3rw= github.com/armon/circbuf v0.0.0-20150827004946-bbbad097214e/go.mod h1:3U/XgcO3hCbHZ8TKRvWD2dDTCfh9M9ya+I9JpbB7O8o= github.com/armon/consul-api v0.0.0-20180202201655-eb2c6b5be1b6/go.mod h1:grANhF5doyWs3UAsr3K4I6qtAmlQcZDesFNEHPZAzj8= github.com/armon/go-metrics v0.0.0-20180917152333-f0300d1749da/go.mod h1:Q73ZrmVTwzkszR9V5SSuryQ31EELlFMUz1kKyl939pY= @@ -61,6 +65,7 @@ github.com/aws/aws-sdk-go v1.27.0/go.mod h1:KmX6BPdI08NWTb3/sm4ZGu5ShLoqVDhKgpiN github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= github.com/beorn7/perks v0.0.0-20180321164747-3a771d992973/go.mod h1:Dwedo/Wpr24TaqPxmxbtue+5NUziq4I4S80YR8gNf3Q= github.com/beorn7/perks v1.0.0/go.mod h1:KWe93zE9D1o94FZ5RNwFwVgaQK1VOXiVxmqh+CedLV8= +github.com/beorn7/perks v1.0.1 h1:VlbKKnNfV8bJzeqoa4cOKqO6bYr3WgKZxO8Z16+hsOM= github.com/beorn7/perks v1.0.1/go.mod h1:G2ZrVWU2WbWT9wwq4/hrbKbnv/1ERSJQ0ibhJ6rlkpw= github.com/bgentry/speakeasy v0.1.0/go.mod h1:+zsyZBPWlz7T6j88CTgSN5bM796AkVf0kBD4zp0CCIs= github.com/blang/semver v3.5.1+incompatible h1:cQNTCjp13qL8KC3Nbxr/y2Bqb63oX6wdnnjpJbkM4JQ= @@ -70,7 +75,9 @@ github.com/bombsimon/wsl v1.2.5/go.mod h1:43lEF/i0kpXbLCeDXL9LMT8c92HyBywXb0AsgM github.com/casbin/casbin/v2 v2.1.2/go.mod h1:YcPU1XXisHhLzuxH9coDNf2FbKpjGlbCg3n9yuLkIJQ= github.com/cenkalti/backoff v2.2.1+incompatible/go.mod h1:90ReRw6GdpyfrHakVjL/QHaoyV4aDUVVkXQJJJ3NXXM= github.com/census-instrumentation/opencensus-proto v0.2.1/go.mod h1:f6KPmirojxKA12rnyqOA5BBL4O983OfeGPqjHWSTneU= +github.com/cespare/xxhash v1.1.0 h1:a6HrQnmkObjyL+Gs60czilIUGqrzKutQD6XZog3p+ko= github.com/cespare/xxhash v1.1.0/go.mod h1:XrSqR1VqqWfGrhpAt58auRo0WTKS1nRRg3ghfAqPWnc= +github.com/cespare/xxhash/v2 v2.1.1 h1:6MnRN8NT7+YBpUIWxHtefFZOKTAPgGjpQSxqLNn0+qY= github.com/cespare/xxhash/v2 v2.1.1/go.mod h1:VGX0DQ3Q6kWi7AoAeZDth3/j3BFtOZR5XLFGgcrjCOs= github.com/chzyer/logex v1.1.10/go.mod h1:+Ywpsq7O8HXn0nuIou7OrIPyXbp3wmkHB+jjWRnGsAI= github.com/chzyer/readline v0.0.0-20180603132655-2972be24d48e/go.mod h1:nSuG5e5PlCu98SY8svDHJxuZscDgtXS6KTTbou5AhLI= @@ -138,8 +145,10 @@ github.com/franela/goreq v0.0.0-20171204163338-bcd34c9993f8/go.mod h1:ZhphrRTfi2 github.com/fsnotify/fsnotify v1.4.7 h1:IXs+QLmnXW2CcXuY+8Mzv/fWEsPGWxqefPtCP5CnV9I= github.com/fsnotify/fsnotify v1.4.7/go.mod h1:jwhsz4b93w/PPRr/qN1Yymfu8t87LnFCMoQvtojpjFo= github.com/ghodss/yaml v1.0.0/go.mod h1:4dBDuWmgqj2HViK6kFavaiC9ZROes6MMH2rRYeMEF04= +github.com/gin-contrib/sse v0.0.0-20190301062529-5545eab6dad3/go.mod h1:VJ0WA2NBN22VlZ2dKZQPAPnyWw5XTlK1KymzLKsr59s= github.com/gin-contrib/sse v0.1.0 h1:Y/yl/+YNO8GZSjAhjMsSuLt29uWRFHdHYUb5lYOV9qE= github.com/gin-contrib/sse v0.1.0/go.mod h1:RHrZQHXnP2xjPF+u1gW/2HnVO7nvIa9PG3Gm+fLHvGI= +github.com/gin-gonic/gin v1.4.0/go.mod h1:OW2EZn3DO8Ln9oIKOvM++LBO+5UPHJJDH72/q/3rZdM= github.com/gin-gonic/gin v1.6.3 h1:ahKqKTFpO5KTPHxWZjEdPScmYaGtLo8Y4DMHoEsnp14= github.com/gin-gonic/gin v1.6.3/go.mod h1:75u5sXoLsGZoRN5Sgbi1eraJ4GU3++wFwWzhwvtwp4M= github.com/go-critic/go-critic v0.3.5-0.20190904082202-d79a9f0c64db h1:GYXWx7Vr3+zv833u+8IoXbNnQY0AdXsxAgI0kX7xcwA= @@ -492,6 +501,7 @@ github.com/mattn/go-sqlite3 v1.11.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsO github.com/mattn/go-sqlite3 v1.12.0 h1:u/x3mp++qUxvYfulZ4HKOvVO0JWhk7HtE8lWhbGz/Do= github.com/mattn/go-sqlite3 v1.12.0/go.mod h1:FPy6KqzDD04eiIsT53CuJW3U88zkxoIYsOqkbpncsNc= github.com/mattn/goveralls v0.0.2/go.mod h1:8d1ZMHsd7fW6IRPKQh46F2WRpyib5/X4FOpevwGNQEw= +github.com/matttproud/golang_protobuf_extensions v1.0.1 h1:4hp9jkHxhMHkqkrB3Ix0jegS5sx/RkqARlsWZ6pIwiU= github.com/matttproud/golang_protobuf_extensions v1.0.1/go.mod h1:D8He9yQNgCq6Z5Ld7szi9bcBfOoFv/3dc6xSMkL2PC0= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d h1:5PJl274Y63IEHC+7izoQE9x6ikvDFZS2mDVS3drnohI= github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d/go.mod h1:01TrycV0kFyexm33Z7vhZRXopbI8J3TDReVlkTgMUxE= @@ -570,20 +580,27 @@ github.com/prometheus/client_golang v0.9.3-0.20190127221311-3c4408c8b829/go.mod github.com/prometheus/client_golang v0.9.3/go.mod h1:/TN21ttK/J9q6uSwhBd54HahCDft0ttaMvbicHlPoso= github.com/prometheus/client_golang v1.0.0/go.mod h1:db9x61etRT2tGnBNRi70OPL5FsnadC4Ky3P0J6CfImo= github.com/prometheus/client_golang v1.3.0/go.mod h1:hJaj2vgQTGQmVCsAACORcieXFeDPbaTKGT+JTgUa3og= +github.com/prometheus/client_golang v1.5.0 h1:Ctq0iGpCmr3jeP77kbF2UxgvRwzWWz+4Bh9/vJTyg1A= +github.com/prometheus/client_golang v1.5.0/go.mod h1:e9GMxYsXl05ICDXkRhurwBS4Q3OK1iX/F2sw+iXX5zU= github.com/prometheus/client_model v0.0.0-20180712105110-5c3871d89910/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190115171406-56726106282f/go.mod h1:MbSGuTsp3dbXC40dX6PRTWyKYBIrTGTE9sqQNg2J8bo= github.com/prometheus/client_model v0.0.0-20190129233127-fd36f4220a90/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.0.0-20190812154241-14fe0d1b01d4/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/client_model v0.1.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= +github.com/prometheus/client_model v0.2.0 h1:uq5h0d+GuxiXLJLNABMgp2qUWDPiLvgCzz2dUR+/W/M= +github.com/prometheus/client_model v0.2.0/go.mod h1:xMI15A0UPsDsEKsMN9yxemIoYk6Tm2C1GtYGdfGttqA= github.com/prometheus/common v0.0.0-20181113130724-41aa239b4cce/go.mod h1:daVV7qP5qjZbuso7PdcryaAu0sAZbrN9i7WWcTMWvro= github.com/prometheus/common v0.2.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.0/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.4.1/go.mod h1:TNfzLD0ON7rHzMJeJkieUDPYmFC7Snx/y86RQel1bk4= github.com/prometheus/common v0.7.0/go.mod h1:DjGbpBbp5NYNiECxcL/VnbXCCaQpKd3tt26CguLLsqA= +github.com/prometheus/common v0.9.1 h1:KOMtN28tlbam3/7ZKEYKHhKoJZYYj3gMH4uc62x7X7U= +github.com/prometheus/common v0.9.1/go.mod h1:yhUN8i9wzaXS3w1O07YhxHEBxD+W35wd8bs7vj7HSQ4= github.com/prometheus/procfs v0.0.0-20181005140218-185b4288413d/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190117184657-bf6a532e95b1/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= github.com/prometheus/procfs v0.0.0-20190507164030-5867b95ac084/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= github.com/prometheus/procfs v0.0.2/go.mod h1:TjEm7ze935MbeOT/UhFTIMYKhuLP4wbCsTZCD3I8kEA= +github.com/prometheus/procfs v0.0.8 h1:+fpWZdT24pJBiqJdAwYBjPSk+5YmQzYNPYzQsdzLkt8= github.com/prometheus/procfs v0.0.8/go.mod h1:7Qr8sr6344vo1JqZ6HhLceV9o3AJ1Ff+GxbHq6oeK9A= github.com/prometheus/tsdb v0.7.1/go.mod h1:qhTCs0VvXwvX/y3TZrWD7rabWM+ijKTux40TwIPHuXU= github.com/quasilyte/go-consistent v0.0.0-20190521200055-c6f3937de18c/go.mod h1:5STLWrekHfjyYwxBRVRXNOSewLJ3PWfDJd1VyTS21fI= @@ -1052,6 +1069,8 @@ gopkg.in/errgo.v2 v2.1.0/go.mod h1:hNsd1EY+bozCKY1Ytp96fpM3vjJbqLJn88ws8XvfDNI= gopkg.in/fsnotify.v1 v1.4.7 h1:xOHLXZwVvI9hhs+cLKq5+I5onOuwQLhQwiu63xxlHs4= gopkg.in/fsnotify.v1 v1.4.7/go.mod h1:Tz8NjZHkW78fSQdbUxIjBTcgA1z1m8ZHf0WmKUhAMys= gopkg.in/gcfg.v1 v1.2.3/go.mod h1:yesOnuUOFQAhST5vPY4nbZsb/huCgGGXlipJsBn0b3o= +gopkg.in/go-playground/assert.v1 v1.2.1/go.mod h1:9RXL0bg/zibRAgZUYszZSwO/z8Y/a8bDuhia5mkpMnE= +gopkg.in/go-playground/validator.v8 v8.18.2/go.mod h1:RX2a/7Ha8BgOhfk7j780h4/u/RRjR0eouCJSH80/M2Y= gopkg.in/gorp.v1 v1.7.2 h1:j3DWlAyGVv8whO7AcIWznQ2Yj7yJkn34B8s63GViAAw= gopkg.in/gorp.v1 v1.7.2/go.mod h1:Wo3h+DBQZIxATwftsglhdD/62zRFPhGhTiu5jUJmCaw= gopkg.in/guregu/null.v4 v4.0.0 h1:1Wm3S1WEA2I26Kq+6vcW+w0gcDo44YKYD7YIEJNHDjg= From 8ba02ab60abdafe3f2ef35a7d51182d760b0ed9d Mon Sep 17 00:00:00 2001 From: yolossn Date: Fri, 8 Jan 2021 14:13:48 +0530 Subject: [PATCH 3/4] backend: Remove teamID from metrics api The application metrics doesn't need teamID, as application metrics should provide general metrics and not specific to a team. Signed-off-by: yolossn --- pkg/api/metrics.go | 12 ++++++------ pkg/api/metrics_test.go | 4 ++-- 2 files changed, 8 insertions(+), 8 deletions(-) diff --git a/pkg/api/metrics.go b/pkg/api/metrics.go index 4d02e392e..a1c424581 100644 --- a/pkg/api/metrics.go +++ b/pkg/api/metrics.go @@ -8,7 +8,7 @@ var ( appInstancesPerChannelMetricSQL string = fmt.Sprintf(` SELECT a.name AS app_name, ia.version AS version, c.name AS channel_name, count(ia.version) AS instances_count FROM instance_application ia, application a, channel c, groups g -WHERE a.team_id = $1 AND a.id = ia.application_id AND ia.group_id = g.id AND g.channel_id = c.id AND %s +WHERE a.id = ia.application_id AND ia.group_id = g.id AND g.channel_id = c.id AND %s GROUP BY app_name, version, channel_name ORDER BY app_name, version, channel_name `, ignoreFakeInstanceCondition("ia.instance_id")) @@ -16,7 +16,7 @@ ORDER BY app_name, version, channel_name failedUpdatesSQL string = fmt.Sprintf(` SELECT a.name AS app_name, count(*) as fail_count FROM application a, event e, event_type et -WHERE a.team_id = $1 AND a.id = e.application_id AND e.event_type_id = et.id AND et.result = 0 AND et.type = 3 AND %s +WHERE a.id = e.application_id AND e.event_type_id = et.id AND et.result = 0 AND et.type = 3 AND %s GROUP BY app_name ORDER BY app_name `, ignoreFakeInstanceCondition("e.instance_id")) @@ -29,9 +29,9 @@ type AppInstancesPerChannelMetric struct { InstancesCount int `db:"instances_count" json:"instances_count"` } -func (api *API) GetAppInstancesPerChannelMetrics(teamID string) ([]AppInstancesPerChannelMetric, error) { +func (api *API) GetAppInstancesPerChannelMetrics() ([]AppInstancesPerChannelMetric, error) { var metrics []AppInstancesPerChannelMetric - rows, err := api.db.Queryx(appInstancesPerChannelMetricSQL, teamID) + rows, err := api.db.Queryx(appInstancesPerChannelMetricSQL) if err != nil { return nil, err } @@ -55,9 +55,9 @@ type FailedUpdatesMetric struct { FailureCount int `db:"fail_count" json:"fail_count"` } -func (api *API) GetFailedUpdatesMetrics(teamID string) ([]FailedUpdatesMetric, error) { +func (api *API) GetFailedUpdatesMetrics() ([]FailedUpdatesMetric, error) { var metrics []FailedUpdatesMetric - rows, err := api.db.Queryx(failedUpdatesSQL, teamID) + rows, err := api.db.Queryx(failedUpdatesSQL) if err != nil { return nil, err } diff --git a/pkg/api/metrics_test.go b/pkg/api/metrics_test.go index 0fb3f7308..296db9ebc 100644 --- a/pkg/api/metrics_test.go +++ b/pkg/api/metrics_test.go @@ -11,7 +11,7 @@ func TestGetAppInstancesPerChannelMetrics(t *testing.T) { defer a.Close() // defaultTeamID constant is defined in users_test.go - metrics, err := a.GetAppInstancesPerChannelMetrics(defaultTeamID) + metrics, err := a.GetAppInstancesPerChannelMetrics() require.NoError(t, err) expectedMetrics := []AppInstancesPerChannelMetric{ { @@ -72,7 +72,7 @@ func TestGetFailedUpdatesMetrics(t *testing.T) { defer a.Close() // defaultTeamID constant is defined in users_test.go - metrics, err := a.GetFailedUpdatesMetrics(defaultTeamID) + metrics, err := a.GetFailedUpdatesMetrics() require.NoError(t, err) expectedMetrics := []FailedUpdatesMetric{ { From ea4763b4a06623b3505620f0d98168742abe6abc Mon Sep 17 00:00:00 2001 From: yolossn Date: Fri, 8 Jan 2021 15:54:47 +0530 Subject: [PATCH 4/4] backend: Add application metrics This implementation of application metrics uses prometheus client_golang package and separates the instrumentation logic from the request-response cycle. The metric update interval can be configured using env variable. Signed-off-by: yolossn --- cmd/nebraska/metrics.go | 117 +++++++++++++++++++++++++++++++++++++++ cmd/nebraska/nebraska.go | 10 +++- go.mod | 1 + 3 files changed, 126 insertions(+), 2 deletions(-) create mode 100644 cmd/nebraska/metrics.go diff --git a/cmd/nebraska/metrics.go b/cmd/nebraska/metrics.go new file mode 100644 index 000000000..c15e925ed --- /dev/null +++ b/cmd/nebraska/metrics.go @@ -0,0 +1,117 @@ +package main + +import ( + "fmt" + "os" + "time" + + "github.com/prometheus/client_golang/prometheus" +) + +const ( + defaultMetricsUpdateInterval = 5 * time.Second +) + +var ( + appInstancePerChannelGaugeMetric = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "nebraska", + Name: "application_instances_per_channel", + Help: "Number of applications from specific channel running on instances", + }, + []string{ + "application", + "version", + "channel", + }, + ) + + failedUpdatesGaugeMetric = prometheus.NewGaugeVec( + prometheus.GaugeOpts{ + Namespace: "nebraska", + Name: "failed_updates", + Help: "Number of failed updates of an application", + }, + []string{ + "application", + }, + ) +) + +// registerNebraskaMetrics registers the application metrics collector with the DefaultRegistrer. +func registerNebraskaMetrics() error { + err := prometheus.Register(appInstancePerChannelGaugeMetric) + if err != nil { + return err + } + err = prometheus.Register(failedUpdatesGaugeMetric) + if err != nil { + return err + } + return nil +} + +// getMetricsRefreshInterval returns the metrics update Interval key is set in the environment as time.Duration, +// NEBRASKA_METRICS_UPDATE_INTERVAL. The variable must be a string acceptable by time.ParseDuration +// If not returns the default update interval. +func getMetricsRefreshInterval() time.Duration { + refreshIntervalEnvValue := os.Getenv("NEBRASKA_METRICS_UPDATE_INTERVAL") + if refreshIntervalEnvValue == "" { + return defaultMetricsUpdateInterval + } + + refreshInterval, err := time.ParseDuration(refreshIntervalEnvValue) + if err != nil || refreshInterval <= 0 { + logger.Warn("invalid NEBRASKA_METRICS_UPDATE_INTERVAL value, it must be acceptable by time.ParseDuration and positive", "value", refreshIntervalEnvValue) + return defaultMetricsUpdateInterval + } + return refreshInterval +} + +// registerAndInstrumentMetrics registers the application metrics and instruments them in configurable intervals. +func registerAndInstrumentMetrics(ctl *controller) error { + // register application metrics + err := registerNebraskaMetrics() + if err != nil { + return err + } + + refreshInterval := getMetricsRefreshInterval() + + metricsTicker := time.Tick(refreshInterval) + + go func() { + for { + <-metricsTicker + err := calculateMetrics(ctl) + if err != nil { + logger.Error("registerAndInstrumentMetrics updating the metrics", "error", err.Error()) + } + } + }() + + return nil +} + +// calculateMetrics calculates the application metrics and updates the respective metric. +func calculateMetrics(ctl *controller) error { + aipcMetrics, err := ctl.api.GetAppInstancesPerChannelMetrics() + if err != nil { + return fmt.Errorf("failed to get app instances per channel metrics: %w", err) + } + + for _, metric := range aipcMetrics { + appInstancePerChannelGaugeMetric.WithLabelValues(metric.ApplicationName, metric.Version, metric.ChannelName).Set(float64(metric.InstancesCount)) + } + + fuMetrics, err := ctl.api.GetFailedUpdatesMetrics() + if err != nil { + return fmt.Errorf("failed to get failed update metrics: %w", err) + } + + for _, metric := range fuMetrics { + failedUpdatesGaugeMetric.WithLabelValues(metric.ApplicationName).Set(float64(metric.FailureCount)) + } + + return nil +} diff --git a/cmd/nebraska/nebraska.go b/cmd/nebraska/nebraska.go index d4649257e..736fb0fbf 100644 --- a/cmd/nebraska/nebraska.go +++ b/cmd/nebraska/nebraska.go @@ -145,6 +145,12 @@ func mainWithError() error { engine := setupRoutes(ctl, *httpLog) + // Register Application metrics and Instrument. + err = registerAndInstrumentMetrics(ctl) + if err != nil { + return err + } + var params []string if os.Getenv("PORT") == "" { params = append(params, ":8000") @@ -237,8 +243,9 @@ func setupRoutes(ctl *controller, httpLog bool) *gin.Engine { // Recovery middleware to recover from panics engine.Use(gin.Recovery()) - // Prometheus Metrics Middleware setupRouter(engine, "top", httpLog) + + // Prometheus Metrics Middleware p := ginprom.New( ginprom.Engine(engine), ginprom.Namespace("nebraska"), @@ -247,7 +254,6 @@ func setupRoutes(ctl *controller, httpLog bool) *gin.Engine { ) engine.Use(p.Instrument()) - setupRouter(engine, "top", httpLog) wrappedEngine := wrapRouter(engine, httpLog) ctl.auth.SetupRouter(wrappedEngine) diff --git a/go.mod b/go.mod index a904c96af..6ba84eb82 100644 --- a/go.mod +++ b/go.mod @@ -32,6 +32,7 @@ require ( github.com/mattn/go-colorable v0.1.8 // indirect github.com/mgutz/ansi v0.0.0-20200706080929-d51e80ef957d // indirect github.com/mgutz/logxi v0.0.0-20161027140823-aebf8a7d67ab + github.com/prometheus/client_golang v1.5.0 github.com/rubenv/sql-migrate v0.0.0-20200616145509-8d140a17f351 github.com/stretchr/testify v1.6.1 github.com/ugorji/go v1.1.13 // indirect