From c764849a8f45a487b94e6f87a28e1f1b9ffeca88 Mon Sep 17 00:00:00 2001 From: Martti T Date: Mon, 22 May 2023 15:45:27 +0300 Subject: [PATCH] new prometheus middleware (#94) new prometheus middleware --- .github/workflows/echo-contrib.yml | 70 ++--- Makefile | 10 +- echoprometheus/README.md | 154 ++++++++++ echoprometheus/prometheus.go | 437 +++++++++++++++++++++++++++++ echoprometheus/prometheus_test.go | 329 ++++++++++++++++++++++ go.mod | 2 +- prometheus/prometheus.go | 3 + 7 files changed, 955 insertions(+), 50 deletions(-) create mode 100644 echoprometheus/README.md create mode 100644 echoprometheus/prometheus.go create mode 100644 echoprometheus/prometheus_test.go diff --git a/.github/workflows/echo-contrib.yml b/.github/workflows/echo-contrib.yml index 8450b64..30aaec9 100644 --- a/.github/workflows/echo-contrib.yml +++ b/.github/workflows/echo-contrib.yml @@ -4,73 +4,54 @@ on: push: branches: - master - paths: - - '**.go' - - 'go.*' - - '_fixture/**' - - '.github/**' - - 'codecov.yml' pull_request: branches: - master - paths: - - '**.go' - - 'go.*' - - '_fixture/**' - - '.github/**' - - 'codecov.yml' workflow_dispatch: +permissions: + contents: read # to fetch code (actions/checkout) + +env: + # run coverage and benchmarks only with the latest Go version + LATEST_GO_VERSION: "1.20" + jobs: test: - env: - latest: '1.19' strategy: matrix: os: [ubuntu-latest, macos-latest, windows-latest] - go: [ '1.17', '1.18', '1.19'] + # Each major Go release is supported until there are two newer major releases. https://golang.org/doc/devel/release.html#policy + # Echo tests with last four major releases (unless there are pressing vulnerabilities) + # As we depend on `golang.org/x/` libraries which only support last 2 Go releases we could have situations when + # we derive from last four major releases promise. + go: ["1.18", "1.19", "1.20"] name: ${{ matrix.os }} @ Go ${{ matrix.go }} runs-on: ${{ matrix.os }} steps: - - name: Set up Go ${{ matrix.go }} - uses: actions/setup-go@v3 - with: - go-version: ${{ matrix.go }} - - name: Checkout Code uses: actions/checkout@v3 - with: - ref: ${{ github.ref }} - - name: Run static checks - if: matrix.go == env.latest && matrix.os == 'ubuntu-latest' - run: | - go install honnef.co/go/tools/cmd/staticcheck@latest - staticcheck -tests=false ./... + - name: Set up Go ${{ matrix.go }} + uses: actions/setup-go@v4 + with: + go-version: ${{ matrix.go }} - name: Run Tests - run: | - go test -race --coverprofile=coverage.coverprofile --covermode=atomic ./... + run: go test -race --coverprofile=coverage.coverprofile --covermode=atomic ./... - name: Upload coverage to Codecov - if: success() && matrix.go == env.latest && matrix.os == 'ubuntu-latest' + if: success() && matrix.go == env.LATEST_GO_VERSION && matrix.os == 'ubuntu-latest' uses: codecov/codecov-action@v3 with: + token: fail_ci_if_error: false + benchmark: needs: test - strategy: - matrix: - os: [ubuntu-latest] - go: [1.19] - name: Benchmark comparison ${{ matrix.os }} @ Go ${{ matrix.go }} - runs-on: ${{ matrix.os }} + name: Benchmark comparison + runs-on: ubuntu-latest steps: - - name: Set up Go ${{ matrix.go }} - uses: actions/setup-go@v3 - with: - go-version: ${{ matrix.go }} - - name: Checkout Code (Previous) uses: actions/checkout@v3 with: @@ -82,6 +63,11 @@ jobs: with: path: new + - name: Set up Go ${{ matrix.go }} + uses: actions/setup-go@v4 + with: + go-version: ${{ env.LATEST_GO_VERSION }} + - name: Install Dependencies run: go install golang.org/x/perf/cmd/benchstat@latest @@ -97,4 +83,4 @@ jobs: - name: Run Benchstat run: | - benchstat previous/benchmark.txt new/benchmark.txt + benchstat previous/benchmark.txt new/benchmark.txt \ No newline at end of file diff --git a/Makefile b/Makefile index 1d41fbc..2e1aa3e 100644 --- a/Makefile +++ b/Makefile @@ -1,15 +1,11 @@ PKG := "github.com/labstack/echo-contrib" PKG_LIST := $(shell go list ${PKG}/...) -tag: - @git tag `grep -P '^\tversion = ' echo.go|cut -f2 -d'"'` - @git tag|grep -v ^v - .DEFAULT_GOAL := check check: lint vet race ## Check project init: - @go get -u honnef.co/go/tools/cmd/staticcheck@latest + @go install honnef.co/go/tools/cmd/staticcheck@latest format: ## Format the source code @find ./ -type f -name "*.go" -exec gofmt -w {} \; @@ -32,6 +28,6 @@ benchmark: ## Run benchmarks help: ## Display this help screen @grep -h -E '^[a-zA-Z_-]+:.*?## .*$$' $(MAKEFILE_LIST) | awk 'BEGIN {FS = ":.*?## "}; {printf "\033[36m%-30s\033[0m %s\n", $$1, $$2}' -goversion ?= "1.16" -test_version: ## Run tests inside Docker with given version (defaults to 1.15 oldest supported). Example: make test_version goversion=1.15 +goversion ?= "1.18" +test_version: ## Run tests inside Docker with given version (defaults to 1.18 oldest supported). Example: make test_version goversion=1.18 @docker run --rm -it -v $(shell pwd):/project golang:$(goversion) /bin/sh -c "cd /project && make race" diff --git a/echoprometheus/README.md b/echoprometheus/README.md new file mode 100644 index 0000000..71e7af8 --- /dev/null +++ b/echoprometheus/README.md @@ -0,0 +1,154 @@ +# Usage + +``` +package main + +import ( + "github.com/labstack/echo/v4" + "github.com/labstack/echo-contrib/prometheus/echoprometheus" +) + +func main() { + e := echo.New() + // Enable metrics middleware + e.Use(echoprometheus.NewMiddleware("myapp")) + e.GET("/metrics", echoprometheus.NewHandler()) + + e.Logger.Fatal(e.Start(":1323")) +} +``` + + +# How to migrate + +## Creating and adding middleware to the application + +Older `prometheus` middleware +```go + e := echo.New() + p := prometheus.NewPrometheus("echo", nil) + p.Use(e) +``` + +With the new `echoprometheus` middleware +```go + e := echo.New() + e.Use(echoprometheus.NewMiddleware("myapp")) // register middleware to gather metrics from requests + e.GET("/metrics", echoprometheus.NewHandler()) // register route to serve gathered metrics in Prometheus format +``` + +## Replacement for `Prometheus.MetricsList` field, `NewMetric(m *Metric, subsystem string)` function and `prometheus.Metric` struct + +The `NewMetric` function allowed to create custom metrics with the old `prometheus` middleware. This helper is no longer available +to avoid the added complexity. It is recommended to use native Prometheus metrics and register those yourself. + +This can be done now as follows: +```go + e := echo.New() + + customRegistry := prometheus.NewRegistry() // create custom registry for your custom metrics + customCounter := prometheus.NewCounter( // create new counter metric. This is replacement for `prometheus.Metric` struct + prometheus.CounterOpts{ + Name: "custom_requests_total", + Help: "How many HTTP requests processed, partitioned by status code and HTTP method.", + }, + ) + if err := customRegistry.Register(customCounter); err != nil { // register your new counter metric with metrics registry + log.Fatal(err) + } + + e.Use(NewMiddlewareWithConfig(MiddlewareConfig{ + AfterNext: func(c echo.Context, err error) { + customCounter.Inc() // use our custom metric in middleware. after every request increment the counter + }, + Registerer: customRegistry, // use our custom registry instead of default Prometheus registry + })) + e.GET("/metrics", NewHandlerWithConfig(HandlerConfig{Gatherer: customRegistry})) // register route for getting gathered metrics data from our custom Registry +``` + +## Replacement for `Prometheus.MetricsPath` + +`MetricsPath` was used to skip metrics own route from Prometheus metrics. Skipping is no longer done and requests to Prometheus +route will be included in gathered metrics. + +To restore the old behaviour the `/metrics` path needs to be excluded from counting using the Skipper function: +```go +conf := echoprometheus.MiddlewareConfig{ + Skipper: func(c echo.Context) bool { + return c.Path() == "/metrics" + }, +} +e.Use(echoprometheus.NewMiddlewareWithConfig(conf)) +``` + +## Replacement for `Prometheus.RequestCounterURLLabelMappingFunc` and `Prometheus.RequestCounterHostLabelMappingFunc` + +These function fields were used to define how "URL" or "Host" attribute in Prometheus metric lines are created. + +These can now be substituted by using `LabelFuncs`: +```go + e.Use(echoprometheus.NewMiddlewareWithConfig(echoprometheus.MiddlewareConfig{ + LabelFuncs: map[string]echoprometheus.LabelValueFunc{ + "scheme": func(c echo.Context, err error) string { // additional custom label + return c.Scheme() + }, + "url": func(c echo.Context, err error) string { // overrides default 'url' label value + return "x_" + c.Request().URL.Path + }, + "host": func(c echo.Context, err error) string { // overrides default 'host' label value + return "y_" + c.Request().Host + }, + }, + })) +``` + +Will produce Prometheus line as +`echo_request_duration_seconds_count{code="200",host="y_example.com",method="GET",scheme="http",url="x_/ok",scheme="http"} 1` + + +## Replacement for `Metric.Buckets` and modifying default metrics + +The `echoprometheus` middleware registers the following metrics by default: + +* Counter `requests_total` +* Histogram `request_duration_seconds` +* Histogram `response_size_bytes` +* Histogram `request_size_bytes` + +You can modify their definition before these metrics are registed with `CounterOptsFunc` and `HistogramOptsFunc` callbacks + +Example: +```go + e.Use(NewMiddlewareWithConfig(MiddlewareConfig{ + HistogramOptsFunc: func(opts prometheus.HistogramOpts) prometheus.HistogramOpts { + if opts.Name == "request_duration_seconds" { + opts.Buckets = []float64{1.0 * bKB, 2.0 * bKB, 5.0 * bKB, 10.0 * bKB, 100 * bKB, 500 * bKB, 1.0 * bMB, 2.5 * bMB, 5.0 * bMB, 10.0 * bMB} + } + return opts + }, + CounterOptsFunc: func(opts prometheus.CounterOpts) prometheus.CounterOpts { + if opts.Name == "requests_total" { + opts.ConstLabels = prometheus.Labels{"my_const": "123"} + } + return opts + }, + })) +``` + +## Replacement for `PushGateway` struct and related methods + +Function `RunPushGatewayGatherer` starts pushing collected metrics and block until context completes or ErrorHandler returns an error. +This function should be run in separate goroutine. + +Example: +```go + go func() { + config := echoprometheus.PushGatewayConfig{ + PushGatewayURL: "https://host:9080", + PushInterval: 10 * time.Millisecond, + } + if err := echoprometheus.RunPushGatewayGatherer(context.Background(), config); !errors.Is(err, context.Canceled) { + log.Fatal(err) + } + }() +``` \ No newline at end of file diff --git a/echoprometheus/prometheus.go b/echoprometheus/prometheus.go new file mode 100644 index 0000000..15164bd --- /dev/null +++ b/echoprometheus/prometheus.go @@ -0,0 +1,437 @@ +/* +Package echoprometheus provides middleware to add Prometheus metrics. +*/ +package echoprometheus + +import ( + "bytes" + "context" + "errors" + "fmt" + "github.com/labstack/echo/v4" + "github.com/labstack/echo/v4/middleware" + "github.com/labstack/gommon/log" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" + "github.com/prometheus/common/expfmt" + "io" + "net/http" + "sort" + "strconv" + "time" +) + +const ( + defaultSubsystem = "echo" +) + +const ( + _ = iota // ignore first value by assigning to blank identifier + bKB float64 = 1 << (10 * iota) + bMB +) + +// sizeBuckets is the buckets for request/response size. Here we define a spectrum from 1KB through 1NB up to 10MB. +var sizeBuckets = []float64{1.0 * bKB, 2.0 * bKB, 5.0 * bKB, 10.0 * bKB, 100 * bKB, 500 * bKB, 1.0 * bMB, 2.5 * bMB, 5.0 * bMB, 10.0 * bMB} + +// MiddlewareConfig contains the configuration for creating prometheus middleware collecting several default metrics. +type MiddlewareConfig struct { + // Skipper defines a function to skip middleware. + Skipper middleware.Skipper + + // Namespace is components of the fully-qualified name of the Metric (created by joining Namespace,Subsystem and Name components with "_") + // Optional + Namespace string + + // Subsystem is components of the fully-qualified name of the Metric (created by joining Namespace,Subsystem and Name components with "_") + // Defaults to: "echo" + Subsystem string + + // LabelFuncs allows adding custom labels in addition to default labels. When key has same name with default label + // it replaces default one. + LabelFuncs map[string]LabelValueFunc + + // HistogramOptsFunc allows to change options for metrics of type histogram before metric is registered to Registerer + HistogramOptsFunc func(opts prometheus.HistogramOpts) prometheus.HistogramOpts + + // CounterOptsFunc allows to change options for metrics of type counter before metric is registered to Registerer + CounterOptsFunc func(opts prometheus.CounterOpts) prometheus.CounterOpts + + // Registerer sets the prometheus.Registerer instance the middleware will register these metrics with. + // Defaults to: prometheus.DefaultRegisterer + Registerer prometheus.Registerer + + // BeforeNext is callback that is executed before next middleware/handler is called. Useful for case when you have own + // metrics that need data to be stored for AfterNext. + BeforeNext func(c echo.Context) + + // AfterNext is callback that is executed after next middleware/handler returns. Useful for case when you have own + // metrics that need incremented/observed. + AfterNext func(c echo.Context, err error) + + timeNow func() time.Time +} + +type LabelValueFunc func(c echo.Context, err error) string + +// HandlerConfig contains the configuration for creating HTTP handler for metrics. +type HandlerConfig struct { + // Gatherer sets the prometheus.Gatherer instance the middleware will use when generating the metric endpoint handler. + // Defaults to: prometheus.DefaultGatherer + Gatherer prometheus.Gatherer +} + +// PushGatewayConfig contains the configuration for pushing to a Prometheus push gateway. +type PushGatewayConfig struct { + // PushGatewayURL is push gateway URL in format http://domain:port + PushGatewayURL string + + // PushInterval in ticker interval for pushing gathered metrics to the Gateway + // Defaults to: 1 minute + PushInterval time.Duration + + // Gatherer sets the prometheus.Gatherer instance the middleware will use when generating the metric endpoint handler. + // Defaults to: prometheus.DefaultGatherer + Gatherer prometheus.Gatherer + + // ErrorHandler is function that is called when errors occur. When callback returns error StartPushGateway also returns. + ErrorHandler func(err error) error + + // ClientTransport specifies the mechanism by which individual HTTP POST requests are made. + // Defaults to: http.DefaultTransport + ClientTransport http.RoundTripper +} + +// NewHandler creates new instance of Handler using Prometheus default registry. +func NewHandler() echo.HandlerFunc { + return NewHandlerWithConfig(HandlerConfig{}) +} + +// NewHandlerWithConfig creates new instance of Handler using given configuration. +func NewHandlerWithConfig(config HandlerConfig) echo.HandlerFunc { + if config.Gatherer == nil { + config.Gatherer = prometheus.DefaultGatherer + } + h := promhttp.HandlerFor(config.Gatherer, promhttp.HandlerOpts{}) + + if r, ok := config.Gatherer.(prometheus.Registerer); ok { + h = promhttp.InstrumentMetricHandler(r, h) + } + + return func(c echo.Context) error { + h.ServeHTTP(c.Response(), c.Request()) + return nil + } +} + +// NewMiddleware creates new instance of middleware using Prometheus default registry. +func NewMiddleware(subsystem string) echo.MiddlewareFunc { + return NewMiddlewareWithConfig(MiddlewareConfig{Subsystem: subsystem}) +} + +// NewMiddlewareWithConfig creates new instance of middleware using given configuration. +func NewMiddlewareWithConfig(config MiddlewareConfig) echo.MiddlewareFunc { + mw, err := config.ToMiddleware() + if err != nil { + panic(err) + } + return mw +} + +// ToMiddleware converts configuration to middleware or returns an error. +func (conf MiddlewareConfig) ToMiddleware() (echo.MiddlewareFunc, error) { + if conf.timeNow == nil { + conf.timeNow = time.Now + } + if conf.Subsystem == "" { + conf.Subsystem = defaultSubsystem + } + if conf.Registerer == nil { + conf.Registerer = prometheus.DefaultRegisterer + } + if conf.CounterOptsFunc == nil { + conf.CounterOptsFunc = func(opts prometheus.CounterOpts) prometheus.CounterOpts { + return opts + } + } + if conf.HistogramOptsFunc == nil { + conf.HistogramOptsFunc = func(opts prometheus.HistogramOpts) prometheus.HistogramOpts { + return opts + } + } + + labelNames, customValuers := createLabels(conf.LabelFuncs) + + requestCount := prometheus.NewCounterVec( + conf.CounterOptsFunc(prometheus.CounterOpts{ + Namespace: conf.Namespace, + Subsystem: conf.Subsystem, + Name: "requests_total", + Help: "How many HTTP requests processed, partitioned by status code and HTTP method.", + }), + labelNames, + ) + // we do not allow skipping or replacing default collector but developer can use `conf.CounterOptsFunc` to rename + // this middleware default collector, so they can have own collector with that same name. + // and we treat all register errors as returnable failures + if err := conf.Registerer.Register(requestCount); err != nil { + return nil, err + } + + requestDuration := prometheus.NewHistogramVec( + conf.HistogramOptsFunc(prometheus.HistogramOpts{ + Namespace: conf.Namespace, + Subsystem: conf.Subsystem, + Name: "request_duration_seconds", + Help: "The HTTP request latencies in seconds.", + // Here, we use the prometheus defaults which are for ~10s request length max: []float64{.005, .01, .025, .05, .1, .25, .5, 1, 2.5, 5, 10} + Buckets: prometheus.DefBuckets, + }), + labelNames, + ) + if err := conf.Registerer.Register(requestDuration); err != nil { + return nil, err + } + + responseSize := prometheus.NewHistogramVec( + conf.HistogramOptsFunc(prometheus.HistogramOpts{ + Namespace: conf.Namespace, + Subsystem: conf.Subsystem, + Name: "response_size_bytes", + Help: "The HTTP response sizes in bytes.", + Buckets: sizeBuckets, + }), + labelNames, + ) + if err := conf.Registerer.Register(responseSize); err != nil { + return nil, err + } + + requestSize := prometheus.NewHistogramVec( + conf.HistogramOptsFunc(prometheus.HistogramOpts{ + Namespace: conf.Namespace, + Subsystem: conf.Subsystem, + Name: "request_size_bytes", + Help: "The HTTP request sizes in bytes.", + Buckets: sizeBuckets, + }), + labelNames, + ) + if err := conf.Registerer.Register(requestSize); err != nil { + return nil, err + } + + return func(next echo.HandlerFunc) echo.HandlerFunc { + return func(c echo.Context) error { + // NB: we do not skip metrics handler path by default. This can be added with custom Skipper but for default + // behaviour we measure metrics path request/response metrics also + if conf.Skipper != nil && conf.Skipper(c) { + return next(c) + } + + if conf.BeforeNext != nil { + conf.BeforeNext(c) + } + reqSz := computeApproximateRequestSize(c.Request()) + + start := conf.timeNow() + err := next(c) + elapsed := float64(conf.timeNow().Sub(start)) / float64(time.Second) + + if conf.AfterNext != nil { + conf.AfterNext(c, err) + } + + url := c.Path() // contains route path ala `/users/:id` + if url == "" { + // as of Echo v4.10.1 path is empty for 404 cases (when router did not find any matching routes) + // in this case we use actual path from request to have some distinction in Prometheus + url = c.Request().URL.Path + } + + status := c.Response().Status + if err != nil { + var httpError *echo.HTTPError + if errors.As(err, &httpError) { + status = httpError.Code + } + if status == 0 || status == http.StatusOK { + status = http.StatusInternalServerError + } + } + + values := make([]string, len(labelNames)) + values[0] = strconv.Itoa(status) + values[1] = c.Request().Method + values[2] = c.Request().Host + values[3] = url + for _, cv := range customValuers { + values[cv.index] = cv.valueFunc(c, err) + } + + requestDuration.WithLabelValues(values...).Observe(elapsed) + requestCount.WithLabelValues(values...).Inc() + requestSize.WithLabelValues(values...).Observe(float64(reqSz)) + responseSize.WithLabelValues(values...).Observe(float64(c.Response().Size)) + + return err + } + }, nil +} + +type customLabelValuer struct { + index int + label string + valueFunc LabelValueFunc +} + +func createLabels(customLabelFuncs map[string]LabelValueFunc) ([]string, []customLabelValuer) { + labelNames := []string{"code", "method", "host", "url"} + if len(customLabelFuncs) == 0 { + return labelNames, nil + } + + customValuers := make([]customLabelValuer, 0) + // we create valuers in two passes for a reason - first to get fixed order, and then we know to assign correct indexes + for label, labelFunc := range customLabelFuncs { + customValuers = append(customValuers, customLabelValuer{ + label: label, + valueFunc: labelFunc, + }) + } + sort.Slice(customValuers, func(i, j int) bool { + return customValuers[i].label < customValuers[j].label + }) + + for cvIdx, cv := range customValuers { + idx := containsAt(labelNames, cv.label) + if idx == -1 { + idx = len(labelNames) + labelNames = append(labelNames, cv.label) + } + customValuers[cvIdx].index = idx + } + return labelNames, customValuers +} + +func containsAt[K comparable](haystack []K, needle K) int { + for i, v := range haystack { + if v == needle { + return i + } + } + return -1 +} + +func computeApproximateRequestSize(r *http.Request) int { + s := 0 + if r.URL != nil { + s = len(r.URL.Path) + } + + s += len(r.Method) + s += len(r.Proto) + for name, values := range r.Header { + s += len(name) + for _, value := range values { + s += len(value) + } + } + s += len(r.Host) + + // N.B. r.Form and r.MultipartForm are assumed to be included in r.URL. + + if r.ContentLength != -1 { + s += int(r.ContentLength) + } + return s +} + +// RunPushGatewayGatherer starts pushing collected metrics and waits for it context to complete or ErrorHandler to return error. +// +// Example: +// ``` +// +// go func() { +// config := echoprometheus.PushGatewayConfig{ +// PushGatewayURL: "https://host:9080", +// PushInterval: 10 * time.Millisecond, +// } +// if err := echoprometheus.RunPushGatewayGatherer(context.Background(), config); !errors.Is(err, context.Canceled) { +// log.Fatal(err) +// } +// }() +// +// ``` +func RunPushGatewayGatherer(ctx context.Context, config PushGatewayConfig) error { + if config.PushGatewayURL == "" { + return errors.New("push gateway URL is missing") + } + if config.PushInterval <= 0 { + config.PushInterval = 1 * time.Minute + } + if config.Gatherer == nil { + config.Gatherer = prometheus.DefaultGatherer + } + if config.ErrorHandler == nil { + config.ErrorHandler = func(err error) error { + log.Error(err) + return nil + } + } + + client := &http.Client{ + Transport: config.ClientTransport, + } + out := &bytes.Buffer{} + + ticker := time.NewTicker(config.PushInterval) + defer ticker.Stop() + for { + select { + case <-ticker.C: + out.Reset() + err := WriteGatheredMetrics(out, config.Gatherer) + if err != nil { + if hErr := config.ErrorHandler(fmt.Errorf("failed to create metrics: %w", err)); hErr != nil { + return hErr + } + continue + } + req, err := http.NewRequestWithContext(ctx, http.MethodPost, config.PushGatewayURL, out) + if err != nil { + if hErr := config.ErrorHandler(fmt.Errorf("failed to create push gateway request: %w", err)); hErr != nil { + return hErr + } + continue + } + res, err := client.Do(req) + if err != nil { + if hErr := config.ErrorHandler(fmt.Errorf("error sending to push gateway: %w", err)); hErr != nil { + return hErr + } + } + if res.StatusCode != http.StatusOK { + if hErr := config.ErrorHandler(echo.NewHTTPError(res.StatusCode, "post metrics request did not succeed")); hErr != nil { + return hErr + } + } + case <-ctx.Done(): + return ctx.Err() + } + } +} + +// WriteGatheredMetrics gathers collected metrics and writes them to given writer +func WriteGatheredMetrics(writer io.Writer, gatherer prometheus.Gatherer) error { + metricFamilies, err := gatherer.Gather() + if err != nil { + return err + } + for _, mf := range metricFamilies { + if _, err := expfmt.MetricFamilyToText(writer, mf); err != nil { + return err + } + } + return nil +} diff --git a/echoprometheus/prometheus_test.go b/echoprometheus/prometheus_test.go new file mode 100644 index 0000000..c56c08d --- /dev/null +++ b/echoprometheus/prometheus_test.go @@ -0,0 +1,329 @@ +package echoprometheus + +import ( + "bytes" + "context" + "errors" + "fmt" + "github.com/labstack/echo/v4" + "github.com/prometheus/client_golang/prometheus" + "github.com/stretchr/testify/assert" + "net/http" + "net/http/httptest" + "strings" + "testing" + "time" +) + +func TestCustomRegistryMetrics(t *testing.T) { + e := echo.New() + + customRegistry := prometheus.NewRegistry() + e.Use(NewMiddlewareWithConfig(MiddlewareConfig{Registerer: customRegistry})) + e.GET("/metrics", NewHandlerWithConfig(HandlerConfig{Gatherer: customRegistry})) + + assert.Equal(t, http.StatusNotFound, request(e, "/ping?test=1")) + + s, code := requestBody(e, "/metrics") + assert.Equal(t, http.StatusOK, code) + assert.Contains(t, s, `echo_request_duration_seconds_count{code="404",host="example.com",method="GET",url="/ping"} 1`) +} + +func TestDefaultRegistryMetrics(t *testing.T) { + e := echo.New() + + e.Use(NewMiddleware("myapp")) + e.GET("/metrics", NewHandler()) + + assert.Equal(t, http.StatusNotFound, request(e, "/ping?test=1")) + + s, code := requestBody(e, "/metrics") + assert.Equal(t, http.StatusOK, code) + assert.Contains(t, s, `myapp_request_duration_seconds_count{code="404",host="example.com",method="GET",url="/ping"} 1`) + + unregisterDefaults("myapp") +} + +func TestPrometheus_Buckets(t *testing.T) { + e := echo.New() + + customRegistry := prometheus.NewRegistry() + e.Use(NewMiddlewareWithConfig(MiddlewareConfig{Registerer: customRegistry})) + e.GET("/metrics", NewHandlerWithConfig(HandlerConfig{Gatherer: customRegistry})) + + assert.Equal(t, http.StatusNotFound, request(e, "/ping")) + + body, code := requestBody(e, "/metrics") + assert.Equal(t, http.StatusOK, code) + assert.Contains(t, body, `echo_request_duration_seconds_bucket{code="404",host="example.com",method="GET",url="/ping",le="0.005"}`, "duration should have time bucket (like, 0.005s)") + assert.NotContains(t, body, `echo_request_duration_seconds_bucket{code="404",host="example.com",method="GET",url="/ping",le="512000"}`, "duration should NOT have a size bucket (like, 512K)") + assert.Contains(t, body, `echo_request_size_bytes_bucket{code="404",host="example.com",method="GET",url="/ping",le="1024"}`, "request size should have a 1024k (size) bucket") + assert.NotContains(t, body, `echo_request_size_bytes_bucket{code="404",host="example.com",method="GET",url="/ping",le="0.005"}`, "request size should NOT have time bucket (like, 0.005s)") + assert.Contains(t, body, `echo_response_size_bytes_bucket{code="404",host="example.com",method="GET",url="/ping",le="1024"}`, "response size should have a 1024k (size) bucket") + assert.NotContains(t, body, `echo_response_size_bytes_bucket{code="404",host="example.com",method="GET",url="/ping",le="0.005"}`, "response size should NOT have time bucket (like, 0.005s)") +} + +func TestMiddlewareConfig_Skipper(t *testing.T) { + e := echo.New() + + customRegistry := prometheus.NewRegistry() + e.Use(NewMiddlewareWithConfig(MiddlewareConfig{ + Skipper: func(c echo.Context) bool { + hasSuffix := strings.HasSuffix(c.Path(), "ignore") + return hasSuffix + }, + Registerer: customRegistry, + })) + + e.GET("/test", func(c echo.Context) error { + return c.String(http.StatusOK, "OK") + }) + e.GET("/test_ignore", func(c echo.Context) error { + return c.String(http.StatusOK, "OK") + }) + + assert.Equal(t, http.StatusNotFound, request(e, "/ping")) + assert.Equal(t, http.StatusOK, request(e, "/test")) + assert.Equal(t, http.StatusOK, request(e, "/test_ignore")) + + out := &bytes.Buffer{} + assert.NoError(t, WriteGatheredMetrics(out, customRegistry)) + + body := out.String() + assert.Contains(t, body, `echo_request_duration_seconds_count{code="200",host="example.com",method="GET",url="/test"} 1`) + assert.Contains(t, body, `echo_request_duration_seconds_count{code="404",host="example.com",method="GET",url="/ping"} 1`) + assert.Contains(t, body, `echo_request_duration_seconds_count{code="404",host="example.com",method="GET",url="/ping"} 1`) + assert.NotContains(t, body, `test_ignore`) // because we skipped +} + +func TestMetricsForErrors(t *testing.T) { + e := echo.New() + customRegistry := prometheus.NewRegistry() + e.Use(NewMiddlewareWithConfig(MiddlewareConfig{ + Skipper: func(c echo.Context) bool { + return strings.HasSuffix(c.Path(), "ignore") + }, + Subsystem: "myapp", + Registerer: customRegistry, + })) + e.GET("/metrics", NewHandlerWithConfig(HandlerConfig{Gatherer: customRegistry})) + + e.GET("/handler_for_ok", func(c echo.Context) error { + return c.JSON(http.StatusOK, "OK") + }) + e.GET("/handler_for_nok", func(c echo.Context) error { + return c.JSON(http.StatusConflict, "NOK") + }) + e.GET("/handler_for_error", func(c echo.Context) error { + return echo.NewHTTPError(http.StatusBadGateway, "BAD") + }) + + assert.Equal(t, http.StatusOK, request(e, "/handler_for_ok")) + assert.Equal(t, http.StatusConflict, request(e, "/handler_for_nok")) + assert.Equal(t, http.StatusConflict, request(e, "/handler_for_nok")) + assert.Equal(t, http.StatusBadGateway, request(e, "/handler_for_error")) + + body, code := requestBody(e, "/metrics") + assert.Equal(t, http.StatusOK, code) + assert.Contains(t, body, fmt.Sprintf("%s_requests_total", "myapp")) + assert.Contains(t, body, `myapp_requests_total{code="200",host="example.com",method="GET",url="/handler_for_ok"} 1`) + assert.Contains(t, body, `myapp_requests_total{code="409",host="example.com",method="GET",url="/handler_for_nok"} 2`) + assert.Contains(t, body, `myapp_requests_total{code="502",host="example.com",method="GET",url="/handler_for_error"} 1`) +} + +func TestMiddlewareConfig_LabelFuncs(t *testing.T) { + e := echo.New() + customRegistry := prometheus.NewRegistry() + e.Use(NewMiddlewareWithConfig(MiddlewareConfig{ + LabelFuncs: map[string]LabelValueFunc{ + "scheme": func(c echo.Context, err error) string { // additional custom label + return c.Scheme() + }, + "method": func(c echo.Context, err error) string { // overrides default 'method' label value + return "overridden_" + c.Request().Method + }, + }, + Registerer: customRegistry, + })) + e.GET("/metrics", NewHandlerWithConfig(HandlerConfig{Gatherer: customRegistry})) + + e.GET("/ok", func(c echo.Context) error { + return c.JSON(http.StatusOK, "OK") + }) + + assert.Equal(t, http.StatusOK, request(e, "/ok")) + + body, code := requestBody(e, "/metrics") + assert.Equal(t, http.StatusOK, code) + assert.Contains(t, body, `echo_request_duration_seconds_count{code="200",host="example.com",method="overridden_GET",scheme="http",url="/ok"} 1`) +} + +func TestMiddlewareConfig_HistogramOptsFunc(t *testing.T) { + e := echo.New() + customRegistry := prometheus.NewRegistry() + e.Use(NewMiddlewareWithConfig(MiddlewareConfig{ + HistogramOptsFunc: func(opts prometheus.HistogramOpts) prometheus.HistogramOpts { + if opts.Name == "request_duration_seconds" { + opts.ConstLabels = prometheus.Labels{"my_const": "123"} + } + return opts + }, + Registerer: customRegistry, + })) + e.GET("/metrics", NewHandlerWithConfig(HandlerConfig{Gatherer: customRegistry})) + + e.GET("/ok", func(c echo.Context) error { + return c.JSON(http.StatusOK, "OK") + }) + + assert.Equal(t, http.StatusOK, request(e, "/ok")) + + body, code := requestBody(e, "/metrics") + assert.Equal(t, http.StatusOK, code) + + // has const label + assert.Contains(t, body, `echo_request_duration_seconds_count{code="200",host="example.com",method="GET",my_const="123",url="/ok"} 1`) + // does not have const label + assert.Contains(t, body, `echo_request_size_bytes_count{code="200",host="example.com",method="GET",url="/ok"} 1`) +} + +func TestMiddlewareConfig_CounterOptsFunc(t *testing.T) { + e := echo.New() + customRegistry := prometheus.NewRegistry() + e.Use(NewMiddlewareWithConfig(MiddlewareConfig{ + CounterOptsFunc: func(opts prometheus.CounterOpts) prometheus.CounterOpts { + if opts.Name == "requests_total" { + opts.ConstLabels = prometheus.Labels{"my_const": "123"} + } + return opts + }, + Registerer: customRegistry, + })) + e.GET("/metrics", NewHandlerWithConfig(HandlerConfig{Gatherer: customRegistry})) + + e.GET("/ok", func(c echo.Context) error { + return c.JSON(http.StatusOK, "OK") + }) + + assert.Equal(t, http.StatusOK, request(e, "/ok")) + + body, code := requestBody(e, "/metrics") + assert.Equal(t, http.StatusOK, code) + + // has const label + assert.Contains(t, body, `echo_requests_total{code="200",host="example.com",method="GET",my_const="123",url="/ok"} 1`) + // does not have const label + assert.Contains(t, body, `echo_request_size_bytes_count{code="200",host="example.com",method="GET",url="/ok"} 1`) +} + +func TestMiddlewareConfig_AfterNextFuncs(t *testing.T) { + e := echo.New() + + customRegistry := prometheus.NewRegistry() + customCounter := prometheus.NewCounter( + prometheus.CounterOpts{ + Name: "custom_requests_total", + Help: "How many HTTP requests processed, partitioned by status code and HTTP method.", + }, + ) + if err := customRegistry.Register(customCounter); err != nil { + t.Fatal(err) + } + + e.Use(NewMiddlewareWithConfig(MiddlewareConfig{ + AfterNext: func(c echo.Context, err error) { + customCounter.Inc() // use our custom metric in middleware + }, + Registerer: customRegistry, + })) + e.GET("/metrics", NewHandlerWithConfig(HandlerConfig{Gatherer: customRegistry})) + + e.GET("/ok", func(c echo.Context) error { + return c.JSON(http.StatusOK, "OK") + }) + + assert.Equal(t, http.StatusOK, request(e, "/ok")) + + body, code := requestBody(e, "/metrics") + assert.Equal(t, http.StatusOK, code) + assert.Contains(t, body, `custom_requests_total 1`) +} + +func TestRunPushGatewayGatherer(t *testing.T) { + receivedMetrics := false + svr := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) { + receivedMetrics = true + w.WriteHeader(http.StatusBadRequest) + w.Write([]byte("OK")) + })) + defer svr.Close() + + ctx, cancel := context.WithTimeout(context.Background(), 250*time.Millisecond) + defer cancel() + + config := PushGatewayConfig{ + PushGatewayURL: svr.URL, + PushInterval: 10 * time.Millisecond, + ErrorHandler: func(err error) error { + return err // to force return after first request + }, + } + err := RunPushGatewayGatherer(ctx, config) + + assert.EqualError(t, err, "code=400, message=post metrics request did not succeed") + assert.True(t, receivedMetrics) +} + +func requestBody(e *echo.Echo, path string) (string, int) { + req := httptest.NewRequest(http.MethodGet, path, nil) + rec := httptest.NewRecorder() + e.ServeHTTP(rec, req) + + return rec.Body.String(), rec.Code +} + +func request(e *echo.Echo, path string) int { + _, code := requestBody(e, path) + return code +} + +func unregisterDefaults(subsystem string) { + // this is extremely hacky way to unregister our middleware metrics that it registers to prometheus default registry + // Metrics/collector can be unregistered only by their instance but we do not have their instance, so we need to + // create similar collector to register it and get error back with that existing collector we actually want to + // unregister + p := prometheus.DefaultRegisterer + + unRegisterCollector := func(opts prometheus.Opts) { + dummyDuplicate := prometheus.NewCounterVec(prometheus.CounterOpts(opts), []string{"code", "method", "host", "url"}) + err := p.Register(dummyDuplicate) + if err == nil { + return + } + var arErr prometheus.AlreadyRegisteredError + if errors.As(err, &arErr) { + p.Unregister(arErr.ExistingCollector) + } + } + + unRegisterCollector(prometheus.Opts{ + Subsystem: subsystem, + Name: "requests_total", + Help: "How many HTTP requests processed, partitioned by status code and HTTP method.", + }) + unRegisterCollector(prometheus.Opts{ + Subsystem: subsystem, + Name: "request_duration_seconds", + Help: "The HTTP request latencies in seconds.", + }) + unRegisterCollector(prometheus.Opts{ + Subsystem: subsystem, + Name: "response_size_bytes", + Help: "The HTTP response sizes in bytes.", + }) + unRegisterCollector(prometheus.Opts{ + Subsystem: subsystem, + Name: "request_size_bytes", + Help: "The HTTP request sizes in bytes.", + }) +} diff --git a/go.mod b/go.mod index 286bc46..4e35042 100644 --- a/go.mod +++ b/go.mod @@ -1,6 +1,6 @@ module github.com/labstack/echo-contrib -go 1.17 +go 1.18 require ( github.com/casbin/casbin/v2 v2.64.0 diff --git a/prometheus/prometheus.go b/prometheus/prometheus.go index 56ac38d..26ee6b8 100644 --- a/prometheus/prometheus.go +++ b/prometheus/prometheus.go @@ -138,6 +138,7 @@ type Metric struct { } // Prometheus contains the metrics gathered by the instance and its path +// Deprecated: use echoprometheus package instead type Prometheus struct { reqCnt *prometheus.CounterVec reqDur, reqSz, resSz *prometheus.HistogramVec @@ -172,6 +173,7 @@ type PushGateway struct { } // NewPrometheus generates a new set of metrics with a certain subsystem name +// Deprecated: use echoprometheus package instead func NewPrometheus(subsystem string, skipper middleware.Skipper, customMetricsList ...[]*Metric) *Prometheus { var metricsList []*Metric if skipper == nil { @@ -297,6 +299,7 @@ func (p *Prometheus) startPushTicker() { } // NewMetric associates prometheus.Collector based on Metric.Type +// Deprecated: use echoprometheus package instead func NewMetric(m *Metric, subsystem string) prometheus.Collector { var metric prometheus.Collector switch m.Type {