From d28232c4be2449bfaaa9891f2565363cf22bd77f Mon Sep 17 00:00:00 2001 From: Adam Wolfe Gordon Date: Mon, 18 Nov 2019 10:53:26 -0800 Subject: [PATCH] Add gRPC metrics via grpc-ecosystem interceptor Use the standard gRPC middleware to record RPC metrics such as durations in both gRPC services. --- emojivoto-emoji-svc/cmd/server.go | 53 +++++++++++++++++++++++++----- emojivoto-voting-svc/cmd/server.go | 9 ++++- go.mod | 1 + go.sum | 2 ++ kustomize/deployment/emoji.yml | 7 ++++ 5 files changed, 63 insertions(+), 9 deletions(-) diff --git a/emojivoto-emoji-svc/cmd/server.go b/emojivoto-emoji-svc/cmd/server.go index 1fab8a2e..abf95fce 100644 --- a/emojivoto-emoji-svc/cmd/server.go +++ b/emojivoto-emoji-svc/cmd/server.go @@ -4,19 +4,25 @@ import ( "fmt" "log" "net" + "net/http" "os" + "os/signal" + "syscall" "time" + "contrib.go.opencensus.io/exporter/ocagent" "github.com/buoyantio/emojivoto/emojivoto-emoji-svc/api" "github.com/buoyantio/emojivoto/emojivoto-emoji-svc/emoji" - "google.golang.org/grpc" - "contrib.go.opencensus.io/exporter/ocagent" + "github.com/grpc-ecosystem/go-grpc-prometheus" + "github.com/prometheus/client_golang/prometheus/promhttp" "go.opencensus.io/plugin/ocgrpc" "go.opencensus.io/trace" + "google.golang.org/grpc" ) var ( - grpcPort = os.Getenv("GRPC_PORT") + grpcPort = os.Getenv("GRPC_PORT") + promPort = os.Getenv("PROM_PORT") ocagentHost = os.Getenv("OC_AGENT_HOST") ) @@ -28,7 +34,7 @@ func main() { oce, err := ocagent.NewExporter( ocagent.WithInsecure(), - ocagent.WithReconnectionPeriod(5 * time.Second), + ocagent.WithReconnectionPeriod(5*time.Second), ocagent.WithAddress(ocagentHost), ocagent.WithServiceName("voting")) if err != nil { @@ -43,8 +49,39 @@ func main() { panic(err) } - grpcServer := grpc.NewServer(grpc.StatsHandler(&ocgrpc.ServerHandler{})) - api.NewGrpServer(grpcServer, allEmoji) - log.Printf("Starting grpc server on GRPC_PORT=[%s]", grpcPort) - grpcServer.Serve(lis) + errs := make(chan error, 1) + + if promPort != "" { + // Start prometheus server + go func() { + log.Printf("Starting prom metrics on PROM_PORT=[%s]", promPort) + http.Handle("/metrics", promhttp.Handler()) + err := http.ListenAndServe(fmt.Sprintf(":%s", promPort), nil) + errs <- err + }() + } + + // Start grpc server + go func() { + grpc_prometheus.EnableHandlingTimeHistogram() + grpcServer := grpc.NewServer( + grpc.StatsHandler(&ocgrpc.ServerHandler{}), + grpc.StreamInterceptor(grpc_prometheus.StreamServerInterceptor), + grpc.UnaryInterceptor(grpc_prometheus.UnaryServerInterceptor), + ) + api.NewGrpServer(grpcServer, allEmoji) + log.Printf("Starting grpc server on GRPC_PORT=[%s]", grpcPort) + err := grpcServer.Serve(lis) + errs <- err + }() + + // Catch shutdown + go func() { + sig := make(chan os.Signal, 1) + signal.Notify(sig, syscall.SIGINT, syscall.SIGQUIT) + s := <-sig + errs <- fmt.Errorf("caught signal %v", s) + }() + + log.Fatal(<-errs) } diff --git a/emojivoto-voting-svc/cmd/server.go b/emojivoto-voting-svc/cmd/server.go index 0c28689d..f31a4b43 100644 --- a/emojivoto-voting-svc/cmd/server.go +++ b/emojivoto-voting-svc/cmd/server.go @@ -14,6 +14,7 @@ import ( "github.com/buoyantio/emojivoto/emojivoto-voting-svc/voting" "contrib.go.opencensus.io/exporter/ocagent" + "github.com/grpc-ecosystem/go-grpc-prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" "go.opencensus.io/plugin/ocgrpc" "go.opencensus.io/trace" @@ -63,8 +64,14 @@ func main() { // Start grpc server go func() { - grpcServer := grpc.NewServer(grpc.StatsHandler(&ocgrpc.ServerHandler{})) + grpc_prometheus.EnableHandlingTimeHistogram() + grpcServer := grpc.NewServer( + grpc.StatsHandler(&ocgrpc.ServerHandler{}), + grpc.StreamInterceptor(grpc_prometheus.StreamServerInterceptor), + grpc.UnaryInterceptor(grpc_prometheus.UnaryServerInterceptor), + ) api.NewGrpServer(grpcServer, poll) + grpc_prometheus.Register(grpcServer) log.Printf("Starting grpc server on GRPC_PORT=[%s]", grpcPort) err := grpcServer.Serve(lis) errs <- err diff --git a/go.mod b/go.mod index 801a7add..951e1ff1 100644 --- a/go.mod +++ b/go.mod @@ -5,6 +5,7 @@ go 1.13 require ( contrib.go.opencensus.io/exporter/ocagent v0.6.0 github.com/golang/protobuf v1.3.2 + github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 github.com/prometheus/client_golang v1.2.1 go.opencensus.io v0.22.1 google.golang.org/grpc v1.22.0 diff --git a/go.sum b/go.sum index e8548d94..6ab59711 100644 --- a/go.sum +++ b/go.sum @@ -45,6 +45,8 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/ github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs= github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc= github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho= +github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk= github.com/grpc-ecosystem/grpc-gateway v1.9.4 h1:5xLhQjsk4zqPf9EHCrja2qFZMx+yBqkO3XgJ14bNnU0= github.com/grpc-ecosystem/grpc-gateway v1.9.4/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= diff --git a/kustomize/deployment/emoji.yml b/kustomize/deployment/emoji.yml index 55b544eb..095397c0 100644 --- a/kustomize/deployment/emoji.yml +++ b/kustomize/deployment/emoji.yml @@ -24,11 +24,15 @@ spec: - env: - name: GRPC_PORT value: "8080" + - name: PROM_PORT + value: "8801" image: buoyantio/emojivoto-emoji-svc:v9 name: emoji-svc ports: - containerPort: 8080 name: grpc + - containerPort: 8801 + name: prom resources: requests: cpu: 100m @@ -45,3 +49,6 @@ spec: - name: grpc port: 8080 targetPort: 8080 + - name: prom + port: 8801 + targetPort: 8801