Add gRPC metrics via grpc-ecosystem interceptor

Use the standard gRPC middleware to record RPC metrics such as durations
in both gRPC services.

emojivoto-emoji-svc/cmd/server.go

@@ -4,19 +4,25 @@ import (
 	"fmt"
 	"log"
 	"net"
+	"net/http"
 	"os"
+	"os/signal"
+	"syscall"
 	"time"
 
+	"contrib.go.opencensus.io/exporter/ocagent"
 	"github.com/buoyantio/emojivoto/emojivoto-emoji-svc/api"
 	"github.com/buoyantio/emojivoto/emojivoto-emoji-svc/emoji"
-	"google.golang.org/grpc"
-	"contrib.go.opencensus.io/exporter/ocagent"
+	"github.com/grpc-ecosystem/go-grpc-prometheus"
+	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"go.opencensus.io/plugin/ocgrpc"
 	"go.opencensus.io/trace"
+	"google.golang.org/grpc"
 )
 
 var (
-	grpcPort = os.Getenv("GRPC_PORT")
+	grpcPort    = os.Getenv("GRPC_PORT")
+	promPort    = os.Getenv("PROM_PORT")
 	ocagentHost = os.Getenv("OC_AGENT_HOST")
 )
 
@@ -28,7 +34,7 @@ func main() {
 
 	oce, err := ocagent.NewExporter(
 		ocagent.WithInsecure(),
-		ocagent.WithReconnectionPeriod(5 * time.Second),
+		ocagent.WithReconnectionPeriod(5*time.Second),
 		ocagent.WithAddress(ocagentHost),
 		ocagent.WithServiceName("voting"))
 	if err != nil {
@@ -43,8 +49,39 @@ func main() {
 		panic(err)
 	}
 
-	grpcServer := grpc.NewServer(grpc.StatsHandler(&ocgrpc.ServerHandler{}))
-	api.NewGrpServer(grpcServer, allEmoji)
-	log.Printf("Starting grpc server on GRPC_PORT=[%s]", grpcPort)
-	grpcServer.Serve(lis)
+	errs := make(chan error, 1)
+
+	if promPort != "" {
+		// Start prometheus server
+		go func() {
+			log.Printf("Starting prom metrics on PROM_PORT=[%s]", promPort)
+			http.Handle("/metrics", promhttp.Handler())
+			err := http.ListenAndServe(fmt.Sprintf(":%s", promPort), nil)
+			errs <- err
+		}()
+	}
+
+	// Start grpc server
+	go func() {
+		grpc_prometheus.EnableHandlingTimeHistogram()
+		grpcServer := grpc.NewServer(
+			grpc.StatsHandler(&ocgrpc.ServerHandler{}),
+			grpc.StreamInterceptor(grpc_prometheus.StreamServerInterceptor),
+			grpc.UnaryInterceptor(grpc_prometheus.UnaryServerInterceptor),
+		)
+		api.NewGrpServer(grpcServer, allEmoji)
+		log.Printf("Starting grpc server on GRPC_PORT=[%s]", grpcPort)
+		err := grpcServer.Serve(lis)
+		errs <- err
+	}()
+
+	// Catch shutdown
+	go func() {
+		sig := make(chan os.Signal, 1)
+		signal.Notify(sig, syscall.SIGINT, syscall.SIGQUIT)
+		s := <-sig
+		errs <- fmt.Errorf("caught signal %v", s)
+	}()
+
+	log.Fatal(<-errs)
 }

emojivoto-voting-svc/cmd/server.go

@@ -14,6 +14,7 @@ import (
 	"github.com/buoyantio/emojivoto/emojivoto-voting-svc/voting"
 
 	"contrib.go.opencensus.io/exporter/ocagent"
+	"github.com/grpc-ecosystem/go-grpc-prometheus"
 	"github.com/prometheus/client_golang/prometheus/promhttp"
 	"go.opencensus.io/plugin/ocgrpc"
 	"go.opencensus.io/trace"
@@ -63,8 +64,14 @@ func main() {
 
 	// Start grpc server
 	go func() {
-		grpcServer := grpc.NewServer(grpc.StatsHandler(&ocgrpc.ServerHandler{}))
+		grpc_prometheus.EnableHandlingTimeHistogram()
+		grpcServer := grpc.NewServer(
+			grpc.StatsHandler(&ocgrpc.ServerHandler{}),
+			grpc.StreamInterceptor(grpc_prometheus.StreamServerInterceptor),
+			grpc.UnaryInterceptor(grpc_prometheus.UnaryServerInterceptor),
+		)
 		api.NewGrpServer(grpcServer, poll)
+		grpc_prometheus.Register(grpcServer)
 		log.Printf("Starting grpc server on GRPC_PORT=[%s]", grpcPort)
 		err := grpcServer.Serve(lis)
 		errs <- err

go.mod

@@ -5,6 +5,7 @@ go 1.13
 require (
 	contrib.go.opencensus.io/exporter/ocagent v0.6.0
 	github.com/golang/protobuf v1.3.2
+	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0
 	github.com/prometheus/client_golang v1.2.1
 	go.opencensus.io v0.22.1
 	google.golang.org/grpc v1.22.0

go.sum

@@ -45,6 +45,8 @@ github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0 h1:Ovs26xHkKqVztRpIrF/92BcuyuQ/YW4NSIpoGtfXNho=
+github.com/grpc-ecosystem/go-grpc-prometheus v1.2.0/go.mod h1:8NvIoxWQoOIhqOTXgfV/d3M/q6VIi02HzZEHgUlZvzk=
 github.com/grpc-ecosystem/grpc-gateway v1.9.4 h1:5xLhQjsk4zqPf9EHCrja2qFZMx+yBqkO3XgJ14bNnU0=
 github.com/grpc-ecosystem/grpc-gateway v1.9.4/go.mod h1:vNeuVxBJEsws4ogUvrchl83t/GYV9WGTSLVdBhOQFDY=
 github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8=

kustomize/deployment/emoji.yml

@@ -24,11 +24,15 @@ spec:
       - env:
         - name: GRPC_PORT
           value: "8080"
+        - name: PROM_PORT
+          value: "8801"
         image: buoyantio/emojivoto-emoji-svc:v9
         name: emoji-svc
         ports:
         - containerPort: 8080
           name: grpc
+        - containerPort: 8801
+          name: prom
         resources:
           requests:
             cpu: 100m
@@ -45,3 +49,6 @@ spec:
   - name: grpc
     port: 8080
     targetPort: 8080
+  - name: prom
+    port: 8801
+    targetPort: 8801