diff --git a/README.md b/README.md index 77c9bb16..449e4b6b 100644 --- a/README.md +++ b/README.md @@ -44,12 +44,18 @@ ci/scripts/install-just.sh The Justfile is a simple text file that contains a list of tasks. Each task is a shell command. For example: -To run build and run the services locally: +To run build and run the services locally without the monitoring: ```bash just up test ``` +To run build and run the services locally with the monitoring: + +```bash +just local test +``` + To run everything including client and do a cleanup of the database afterward: ```bash diff --git a/datastore/database/extra.conf b/datastore/database/extra.conf new file mode 100644 index 00000000..9afe745a --- /dev/null +++ b/datastore/database/extra.conf @@ -0,0 +1,6 @@ +# postgres-exporter +shared_preload_libraries = 'pg_cron,pg_stat_statements' +track_activity_query_size = 2048 +pg_stat_statements.track = 'all' +pg_stat_statements.max = 10000 +pg_stat_statements.save = 'on' diff --git a/datastore/datastore/.dockerignore b/datastore/datastore/.dockerignore index 3a6d4f56..72c7bba9 100644 --- a/datastore/datastore/.dockerignore +++ b/datastore/datastore/.dockerignore @@ -5,3 +5,4 @@ !common !dsimpl !storagebackend +!metrics diff --git a/datastore/datastore/go.mod b/datastore/datastore/go.mod index a78c29b9..f4a37f3b 100644 --- a/datastore/datastore/go.mod +++ b/datastore/datastore/go.mod @@ -5,14 +5,27 @@ go 1.21 require google.golang.org/grpc v1.64.0 require ( - github.com/cridenour/go-postgis v1.0.0 - google.golang.org/protobuf v1.34.1 + github.com/cridenour/go-postgis v1.0.0 + github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 + github.com/prometheus/client_golang v1.20.4 + google.golang.org/protobuf v1.34.2 ) require ( - github.com/lib/pq v1.10.9 - golang.org/x/net v0.26.0 // indirect - golang.org/x/sys v0.21.0 // indirect - golang.org/x/text v0.16.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect +) + +require ( + github.com/lib/pq v1.10.9 + golang.org/x/net v0.26.0 // indirect + golang.org/x/sys v0.22.0 // indirect + golang.org/x/text v0.16.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect ) diff --git a/datastore/datastore/main/main.go b/datastore/datastore/main/main.go index 5bf3a654..1f78da59 100644 --- a/datastore/datastore/main/main.go +++ b/datastore/datastore/main/main.go @@ -8,15 +8,23 @@ import ( "datastore/storagebackend" "datastore/storagebackend/postgresql" "fmt" + "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" + "github.com/prometheus/client_golang/prometheus/promhttp" "log" "net" "time" + // gRPC "google.golang.org/grpc" "google.golang.org/grpc/health" "google.golang.org/grpc/health/grpc_health_v1" "google.golang.org/grpc/peer" + // Monitoring + "datastore/metrics" + grpcprometheus "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" + _ "expvar" "net/http" _ "net/http/pprof" @@ -52,8 +60,34 @@ func main() { return resp, err } - // create gRPC server - server := grpc.NewServer(grpc.UnaryInterceptor(reqTimeLogger)) + grpcMetrics := grpcprometheus.NewServerMetrics( + grpcprometheus.WithServerHandlingTimeHistogram( + grpcprometheus.WithHistogramBuckets([]float64{0.001, 0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120}), + ), + ) + reg := prometheus.NewRegistry() + reg.MustRegister( + grpcMetrics, + promservermetrics.InFlightRequests, + promservermetrics.UptimeCounter, + promservermetrics.ResponseSizeSummary, + collectors.NewGoCollector(), + collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), + ) + + go promservermetrics.TrackUptime() + + // create gRPC server with middleware + server := grpc.NewServer( + grpc.ChainUnaryInterceptor( + reqTimeLogger, + promservermetrics.InFlightRequestInterceptor, + promservermetrics.ResponseSizeUnaryInterceptor, + grpcMetrics.UnaryServerInterceptor(), + ), + ) + + grpcMetrics.InitializeMetrics(server) grpc_health_v1.RegisterHealthServer(server, health.NewServer()) // create storage backend @@ -81,6 +115,22 @@ func main() { http.ListenAndServe("0.0.0.0:6060", nil) }() + // serve go metrics for monitoring + go func() { + httpSrv := &http.Server{Addr: "0.0.0.0:8081"} + m := http.NewServeMux() + // Create HTTP handler for Prometheus metrics. + m.Handle("/metrics", promhttp.HandlerFor( + reg, + promhttp.HandlerOpts{ + EnableOpenMetrics: true, + }, + )) + httpSrv.Handler = m + log.Println("Starting HTTP server for Prometheus metrics on :8081") + log.Fatal(httpSrv.ListenAndServe()) + }() + // serve incoming requests log.Printf("starting server\n") if err := server.Serve(listener); err != nil { diff --git a/datastore/datastore/metrics/promservermetrics.go b/datastore/datastore/metrics/promservermetrics.go new file mode 100644 index 00000000..c7d2cf99 --- /dev/null +++ b/datastore/datastore/metrics/promservermetrics.go @@ -0,0 +1,67 @@ +package promservermetrics + +import ( + "context" + "github.com/prometheus/client_golang/prometheus" + "google.golang.org/grpc" + "sync" + "time" +) + +var ( + UptimeCounter = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "grpc_server_uptime_seconds", + Help: "Total uptime of the gRPC server in seconds", + }) + + InFlightRequests = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "grpc_in_flight_requests", + Help: "Current number of in-flight gRPC requests", + }) + + ResponseSizeSummary = prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Name: "grpc_response_size_summary_bytes", + Help: "Summary of response sizes in bytes for each gRPC method, with mean, min, and max", + Objectives: map[float64]float64{0.0: 0.001, 1.0: 0.001}, // Track min (0.0 quantile) and max (1.0 quantile) + }, + []string{"method"}, + ) + + responseSizeMu sync.Mutex + responseSizeSum = make(map[string]float64) + responseSizeCount = make(map[string]float64) +) + +func TrackUptime() { + // Increment the uptime every second + for { + UptimeCounter.Inc() + time.Sleep(1 * time.Second) + } +} + +func InFlightRequestInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + InFlightRequests.Inc() // Increment at the start of the request + defer InFlightRequests.Dec() // Decrement at the end of the request + return handler(ctx, req) +} + +func ResponseSizeUnaryInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) { + resp, err = handler(ctx, req) + + if resp != nil { + responseSize := float64(len(resp.(interface{ String() string }).String())) + + ResponseSizeSummary.WithLabelValues(info.FullMethod).Observe(responseSize) + + // Used a mutex to synchronise the access for the responseSizeSum and responseSizeCount. + // To prevent race conditions and multiple goroutines accessing the variables at the same time. + responseSizeMu.Lock() + responseSizeSum[info.FullMethod] += responseSize + responseSizeCount[info.FullMethod]++ + responseSizeMu.Unlock() + } + + return resp, err +} diff --git a/datastore/migrate/data/migrations/1723033622_setup_prometheus_extension.down.sql b/datastore/migrate/data/migrations/1723033622_setup_prometheus_extension.down.sql new file mode 100644 index 00000000..36bec35c --- /dev/null +++ b/datastore/migrate/data/migrations/1723033622_setup_prometheus_extension.down.sql @@ -0,0 +1 @@ +DROP EXTENSION pg_stat_statements; diff --git a/datastore/migrate/data/migrations/1723033622_setup_prometheus_extension.up.sql b/datastore/migrate/data/migrations/1723033622_setup_prometheus_extension.up.sql new file mode 100644 index 00000000..841ff0c4 --- /dev/null +++ b/datastore/migrate/data/migrations/1723033622_setup_prometheus_extension.up.sql @@ -0,0 +1 @@ +CREATE EXTENSION IF NOT EXISTS pg_stat_statements; diff --git a/datastore/migrate/data/migrations/1730887695_create_pg_stat_statements.down.sql b/datastore/migrate/data/migrations/1730887695_create_pg_stat_statements.down.sql new file mode 100644 index 00000000..be80e5c7 --- /dev/null +++ b/datastore/migrate/data/migrations/1730887695_create_pg_stat_statements.down.sql @@ -0,0 +1 @@ +DROP EXTENSION IF EXISTS pg_stat_statements; diff --git a/datastore/migrate/data/migrations/1730887695_create_pg_stat_statements.up.sql b/datastore/migrate/data/migrations/1730887695_create_pg_stat_statements.up.sql new file mode 100644 index 00000000..841ff0c4 --- /dev/null +++ b/datastore/migrate/data/migrations/1730887695_create_pg_stat_statements.up.sql @@ -0,0 +1 @@ +CREATE EXTENSION IF NOT EXISTS pg_stat_statements; diff --git a/docker-compose.yml b/docker-compose.yml index 29333ea1..b6ac982d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,3 @@ -version: "3.8" name: datastore services: @@ -10,8 +9,10 @@ services: volumes: # - ts-data:/home/postgres/pgdata/data # for timescale image - ts-data:/var/lib/postgresql # for postgres image - - ./datastore/database/healthcheck_postgis_uptime.sh:/healthcheck_postgis_uptime.sh # for the healthcheck + - ./datastore/database/extra.conf:/etc/conf_settings/extra.conf:ro # Extra Postgres configuration + - ./datastore/database/healthcheck_postgis_uptime.sh:/healthcheck_postgis_uptime.sh:ro # for the healthcheck environment: + - EXTRA_CONF_DIR=/etc/conf_settings - POSTGRES_USER=postgres - POSTGRES_PASSWORD=mysecretpassword - POSTGRES_DB=data @@ -26,7 +27,7 @@ services: ] interval: 5s timeout: 1s - retries: 3 + retries: 30 start_period: 30s # Failures in 30 seconds do not mark container as unhealthy migrate: @@ -46,6 +47,7 @@ services: ports: - "50050:50050" - "6060:6060" # for flame graphing + - "8081:8081" environment: - PGHOST=db - PGPORT=5432 @@ -156,10 +158,52 @@ services: - DSHOST=store - DSPORT=50050 volumes: - - ./datastore/load-test/output:/load-test/output + - ./datastore/load-test/output:/load-test/output:rw depends_on: store: condition: service_healthy + prometheus: + profiles: ["monitoring"] + image: prom/prometheus + ports: + - "9090:9090" + volumes: + - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - prometheus-data:/prometheus + + prometheus-postgres-exporter: + profiles: ["monitoring"] + image: quay.io/prometheuscommunity/postgres-exporter + environment: + - DATA_SOURCE_URI=db:5432/data + - DATA_SOURCE_USER=postgres + - DATA_SOURCE_PASS=mysecretpassword + ports: + - "9187:9187" + volumes: + - ./prometheus/postgres_exporter.yml:/postgres_exporter.yml:ro + depends_on: + db: + condition: service_healthy + command: ["--collector.stat_statements", "--collector.stat_user_tables", "--collector.stat_activity_autovacuum"] + + grafana: + profiles: ["monitoring"] + image: grafana/grafana-oss:11.2.0 + ports: + - "3000:3000" + environment: + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=mysecretpassword + volumes: + - grafana-storage:/var/lib/grafana + - ./grafana/provisioning:/etc/grafana/provisioning:rw + - ./grafana/dashboards:/var/lib/grafana/dashboards:rw + depends_on: + - prometheus + volumes: ts-data: + prometheus-data: + grafana-storage: diff --git a/grafana/dashboards/database-performance.json b/grafana/dashboards/database-performance.json new file mode 100755 index 00000000..6898cfdd --- /dev/null +++ b/grafana/dashboards/database-performance.json @@ -0,0 +1,1758 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 12, + "panels": [], + "title": "General", + "type": "row" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "green", + "value": 2 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "round((time() - process_start_time_seconds{instance=~\"$INSTANCE\"}), 0.01)", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Uptime", + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 70 + }, + { + "color": "green", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "background_solid", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "round(100*sum(pg_stat_database_blks_hit{instance=~\"$INSTANCE\"}) / (sum(pg_stat_database_blks_hit{instance=~\"$INSTANCE\"}) + sum(pg_stat_database_blks_read{instance=~\"$INSTANCE\"})),0.1)", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Buffer cache hit rate", + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + }, + { + "color": "#EAB839", + "value": 75 + }, + { + "color": "red", + "value": 85 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "(sum(pg_stat_activity_count) / sum(pg_settings_max_connections)) * 100", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Percentage of connections out of the total.", + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "#EAB839", + "value": 0 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (state) (pg_stat_activity_count)", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{state}}", + "range": true, + "refId": "A" + } + ], + "title": "Number of active connections", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 17, + "panels": [], + "title": "Query Performance", + "type": "row" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 13 + }, + "id": 1, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(pg_stat_statements_seconds_total{instance=~\"$INSTANCE\", datname=~\"$DATNAME\"}[$INTERVAL])) / sum(rate(pg_stat_statements_calls_total{instance=~\"$INSTANCE\", datname=~\"$DATNAME\"}[$INTERVAL]))", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "Query time", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Average query time", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 13 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(rate(pg_stat_statements_calls_total{instance=~\"$INSTANCE\"}[$INTERVAL]))", + "instant": false, + "legendFormat": "Number of queries", + "range": true, + "refId": "A" + } + ], + "title": "Number of queries", + "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "P5C9DBE12543779E4" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "query" + }, + "properties": [ + { + "id": "custom.minWidth", + "value": 400 + }, + { + "id": "custom.inspect", + "value": true + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 19 + }, + "id": 7, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "P5C9DBE12543779E4" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT queryid, query, calls\nFROM pg_stat_statements\nWHERE query NOT LIKE '%CREATE%'\nORDER BY calls DESC\nLIMIT 10;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "pg_stat_statements" + } + ], + "title": "Top 10 most called queries", + "type": "table" + }, + { + "datasource": { + "default": false, + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "mean_exec_time" + }, + "properties": [ + { + "id": "unit", + "value": "ms" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "query" + }, + "properties": [ + { + "id": "custom.minWidth", + "value": 400 + }, + { + "id": "custom.inspect", + "value": true + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 19 + }, + "id": 6, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "mean_exec_time" + } + ] + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "P5C9DBE12543779E4" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT queryid,query, mean_exec_time\nFROM pg_stat_statements\nWHERE query NOT LIKE '%CREATE%'\nORDER BY mean_exec_time DESC\nLIMIT 10;", + "refId": "B", + "sql": { + "columns": [ + { + "parameters": [ + { + "name": "queryid", + "type": "functionParameter" + } + ], + "type": "function" + }, + { + "parameters": [ + { + "name": "query", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "pg_stat_statements" + } + ], + "title": "Top 10 slowest queries", + "type": "table" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(pg_stat_database_tup_fetched[$INTERVAL]) > 1000", + "instant": false, + "legendFormat": "{{datname}} SELECT (index scan)", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(pg_stat_database_tup_returned[$INTERVAL]) > 1000", + "hide": false, + "instant": false, + "legendFormat": "{{datname}} SELECT (table scan)", + "range": true, + "refId": "B" + } + ], + "title": "Read statistics", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 15, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(pg_stat_database_tup_inserted[$INTERVAL]) > 1", + "instant": false, + "legendFormat": "{{datname}} INSERT", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(pg_stat_database_tup_updated[5m]) > 1", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{datname}} UPDATE", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(pg_stat_database_tup_deleted[5m]) > 1", + "hide": false, + "instant": false, + "legendFormat": "{{datname}} DELETE", + "range": true, + "refId": "C" + } + ], + "title": "Write statistics", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 44 + }, + "id": 18, + "panels": [], + "title": "Errors", + "type": "row" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 45 + }, + "id": 19, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(pg_stat_database_deadlocks[$INTERVAL])", + "instant": false, + "interval": "1m", + "legendFormat": "{{datname}}", + "range": true, + "refId": "A" + } + ], + "title": "Deadlocks", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 45 + }, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(rate(pg_stat_database_conflicts_confl_deadlock[$INTERVAL]))", + "instant": false, + "legendFormat": "Deadlock", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(rate(pg_stat_database_conflicts_confl_bufferpin[5m]))", + "hide": false, + "instant": false, + "legendFormat": "Bufferpin", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(rate(pg_stat_database_conflicts_confl_lock[5m]))", + "hide": false, + "instant": false, + "legendFormat": "Lock", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(rate(pg_stat_database_conflicts_confl_snapshot[5m]))", + "hide": false, + "instant": false, + "legendFormat": "Snapshot", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(rate(pg_stat_database_conflicts_confl_tablespace[5m]))", + "hide": false, + "instant": false, + "legendFormat": "Tablespace", + "range": true, + "refId": "E" + } + ], + "title": "Conflicts", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 53 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (datname, mode) (pg_locks_count{instance=~\"$INSTANCE\", datname=~\"$DATNAME\"})", + "instant": false, + "interval": "1m", + "legendFormat": "{{datname}} {{mode}}", + "range": true, + "refId": "A" + } + ], + "title": "Table locks", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 61 + }, + "id": 10, + "panels": [], + "title": "Database Size", + "type": "row" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 62 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(pg_database_size_bytes)", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Total database size", + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 62 + }, + "id": 8, + "options": { + "displayLabels": [ + "name", + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "pg_database_size_bytes", + "instant": false, + "interval": "", + "legendFormat": "{{ datname }}", + "range": true, + "refId": "A" + } + ], + "title": "Size per database", + "type": "piechart" + } + ], + "refresh": "1m", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "allValue": "", + "current": { + "selected": true, + "text": [ + "data" + ], + "value": [ + "data" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(datname)", + "hide": 0, + "includeAll": true, + "label": "Database name", + "multi": true, + "name": "DATNAME", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(datname)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": true, + "text": "prometheus-postgres-exporter:9187", + "value": "prometheus-postgres-exporter:9187" + }, + "hide": 0, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "INSTANCE", + "options": [ + { + "selected": true, + "text": "prometheus-postgres-exporter:9187", + "value": "prometheus-postgres-exporter:9187" + } + ], + "query": "prometheus-postgres-exporter:9187", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": true, + "text": "1m", + "value": "1m" + }, + "hide": 0, + "label": "Interval", + "name": "INTERVAL", + "options": [ + { + "selected": true, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "1m,5m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "queryValue": "", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Database Performance", + "uid": "adxl34wpc3if4a", + "version": 9, + "weekStart": "" +} diff --git a/grafana/dashboards/grpc-server.json b/grafana/dashboards/grpc-server.json new file mode 100755 index 00000000..6490fc83 --- /dev/null +++ b/grafana/dashboards/grpc-server.json @@ -0,0 +1,2908 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 1, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 5, + "panels": [], + "title": "gRPC General", + "type": "row" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 60 + }, + { + "color": "green", + "value": 3600 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "grpc_server_uptime_seconds", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Uptime", + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "#EAB839", + "value": 0.8 + }, + { + "color": "green", + "value": 0.95 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "1 - (sum(rate(grpc_server_handled_total{job=\"store\",grpc_type=\"unary\",grpc_code!=\"OK\"}[5m])) OR vector(0) / sum(rate(grpc_server_started_total{job=\"store\",grpc_type=\"unary\"}[5m])))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Succes Rate", + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "yellow", + "value": 0.1 + }, + { + "color": "red", + "value": 1 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"$JOB\",grpc_type=\"unary\"}[$INTERVAL])) by (le)\n)", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "dark-red", + "value": null + }, + { + "color": "yellow", + "value": 100 + }, + { + "color": "green", + "value": 500 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(rate(grpc_server_started_total{job=\"$JOB\"}[$INTERVAL]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Requests per interval", + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "percent" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 5 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(rate(grpc_server_handled_total{job=\"$JOB\",grpc_type=\"unary\",grpc_code=\"OK\"}[$INTERVAL]))\n/ sum(rate(grpc_server_handled_total{job=\"$JOB\",grpc_type=\"unary\"}[$INTERVAL]))", + "instant": false, + "legendFormat": "OK", + "range": true, + "refId": "OK" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "(sum(rate(grpc_server_handled_total{job=\"$JOB\",grpc_type=\"unary\",grpc_code=\"InvalidArgument\"}[$INTERVAL])) +\nsum(rate(grpc_server_handled_total{job=\"$JOB\",grpc_type=\"unary\",grpc_code=\"NotFound\"}[$INTERVAL])) + \nsum(rate(grpc_server_handled_total{job=\"$JOB\",grpc_type=\"unary\",grpc_code=\"AlreadyExists\"}[$INTERVAL])) + \nsum(rate(grpc_server_handled_total{job=\"$JOB\",grpc_type=\"unary\",grpc_code=\"Unauthenticated\"}[$INTERVAL])) +\nsum(rate(grpc_server_handled_total{job=\"$JOB\",grpc_type=\"unary\",grpc_code=\"PermissionDenied\"}[$INTERVAL])) + \nsum(rate(grpc_server_handled_total{job=\"$JOB\",grpc_type=\"unary\",grpc_code=\"FailedPrecondition\"}[$INTERVAL]))\n)/ sum(rate(grpc_server_handled_total{job=\"$JOB\",grpc_type=\"unary\"}[$INTERVAL]))", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "ClientError", + "range": true, + "refId": "ClientError" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "(sum(rate(grpc_server_handled_total{job=\"$JOB\",grpc_type=\"unary\",grpc_code=\"Internal\"}[$INTERVAL])) +\nsum(rate(grpc_server_handled_total{job=\"$JOB\",grpc_type=\"unary\",grpc_code=\"Unknown\"}[$INTERVAL])) + \nsum(rate(grpc_server_handled_total{job=\"$JOB\",grpc_type=\"unary\",grpc_code=\"Unavailable\"}[$INTERVAL])) +\nsum(rate(grpc_server_handled_total{job=\"$JOB\",grpc_type=\"unary\",grpc_code=\"Unimplemented\"}[$INTERVAL]))\n)/ sum(rate(grpc_server_handled_total{job=\"$JOB\",grpc_type=\"unary\"}[$INTERVAL]))", + "hide": false, + "instant": false, + "legendFormat": "ServerError", + "range": true, + "refId": "ServerError" + } + ], + "title": "Status", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "percent" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 5 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(rate(grpc_server_handled_total{job=\"$JOB\",grpc_type=\"unary\"}[$INTERVAL])) by (grpc_code)\n/ ignoring(grpc_code) group_left sum(rate(grpc_server_handled_total{job=\"$JOB\",grpc_type=\"unary\"}[$INTERVAL]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Status distribution", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 13 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(rate(grpc_server_started_total{job=\"$JOB\"}[$INTERVAL])) by (grpc_service)", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Requests per second", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "percent" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 13 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(rate(grpc_server_started_total{job=\"$JOB\"}[$INTERVAL])) by (grpc_service) \n/ ignoring(grpc_service) group_left sum(rate(grpc_server_started_total{job=\"$JOB\"}[$INTERVAL]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Request distribution per service", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 21 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"$JOB\",grpc_type=\"unary\"}[$INTERVAL])) by (grpc_service,le)\n)", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "99%" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 21 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"$JOB\",grpc_type=\"unary\"}[$INTERVAL])) by (le)\n)", + "instant": false, + "interval": "", + "legendFormat": "99%", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"$JOB\",grpc_type=\"unary\"}[$INTERVAL])) by (le)\n)", + "hide": false, + "instant": false, + "legendFormat": "95%", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"$JOB\",grpc_type=\"unary\"}[$INTERVAL])) by (le)\n)", + "hide": false, + "instant": false, + "legendFormat": "90%", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.75, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"$JOB\",grpc_type=\"unary\"}[$INTERVAL])) by (le)\n)", + "hide": false, + "instant": false, + "legendFormat": "75%", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"$JOB\",grpc_type=\"unary\"}[$INTERVAL])) by (le)\n)", + "hide": false, + "instant": false, + "legendFormat": "50%", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.25, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"$JOB\",grpc_type=\"unary\"}[$INTERVAL])) by (le)\n)", + "hide": false, + "instant": false, + "legendFormat": "25%", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.10, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"$JOB\",grpc_type=\"unary\"}[$INTERVAL])) by (le)\n)", + "hide": false, + "instant": false, + "legendFormat": "10%", + "range": true, + "refId": "G" + } + ], + "title": "Latency distribution", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 29 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "grpc_in_flight_requests", + "instant": false, + "legendFormat": "In flight requests", + "range": true, + "refId": "A" + } + ], + "title": "In flight requests", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 37 + }, + "id": 14, + "panels": [], + "title": "gRPC Methods", + "type": "row" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(grpc_response_size_summary_bytes_count{job=\"$JOB\"}[$INTERVAL])", + "instant": false, + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Number of calls over time", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 16, + "options": { + "displayLabels": [], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "percent", + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "grpc_response_size_summary_bytes_count{job=\"$JOB\"}", + "instant": false, + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Call distribution", + "type": "piechart" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 46 + }, + "id": 15, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(grpc_response_size_summary_bytes_sum{job=\"$JOB\"}[$INTERVAL]) / rate(grpc_response_size_summary_bytes_count{job=\"$JOB\"}[$INTERVAL])", + "instant": false, + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Mean response size", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 46 + }, + "id": 17, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "grpc_response_size_summary_bytes{quantile=\"1.0\", job=\"$JOB\"}", + "instant": false, + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Maximum response size", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 54 + }, + "id": 18, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "grpc_response_size_summary_bytes{quantile=\"0.0\", job=\"$JOB\"}", + "instant": false, + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Minimum response size", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 62 + }, + "id": 21, + "panels": [], + "title": "Go", + "type": "row" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 63 + }, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "go_memstats_mspan_inuse_bytes{job=\"$JOB\"}", + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "go_memstats_mspan_sys_bytes{job=\"$JOB\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "go_memstats_mcache_inuse_bytes{job=\"$JOB\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "go_memstats_mcache_sys_bytes{job=\"$JOB\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "go_memstats_buck_hash_sys_bytes{job=\"$JOB\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "go_memstats_gc_sys_bytes{job=\"$JOB\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "go_memstats_other_sys_bytes{job=\"$JOB\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "go_memstats_next_gc_bytes{job=\"$JOB\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "H" + } + ], + "title": "Memory in off-heap", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 63 + }, + "id": 22, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "go_memstats_heap_alloc_bytes{job=\"$JOB\"}", + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "go_memstats_heap_sys_bytes{job=\"$JOB\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "go_memstats_heap_idle_bytes{job=\"$JOB\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "go_memstats_heap_inuse_bytes{job=\"$JOB\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "go_memstats_heap_released_bytes{job=\"$JOB\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "E" + } + ], + "title": "Memory in heap", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 71 + }, + "id": 24, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "go_memstats_stack_inuse_bytes{job=\"$JOB\"}", + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "go_memstats_stack_sys_bytes{job=\"$JOB\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "Memory in stack", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 71 + }, + "id": 23, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "go_memstats_sys_bytes{job=\"$JOB\"}", + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A" + } + ], + "title": "Total used memory", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 79 + }, + "id": 25, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "go_memstats_mallocs_total{job=\"$JOB\"} - go_memstats_frees_total{job=\"$JOB\"}", + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A" + } + ], + "title": "Number of live objects", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 79 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(go_memstats_mallocs_total{job=\"$JOB\"}[$INTERVAL])", + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A" + } + ], + "title": "Rate of objects allocated", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 87 + }, + "id": 29, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(go_memstats_lookups_total{job=\"$JOB\"}[$INTERVAL])", + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A" + } + ], + "title": "Rate of a pointer dereferences", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 87 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "go_goroutines{job=\"$JOB\"}", + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A" + } + ], + "title": "Goroutines", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 95 + }, + "id": 28, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(go_memstats_alloc_bytes_total{job=\"$JOB\"}[$INTERVAL])", + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A" + } + ], + "title": "Rates of allocation", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 95 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "go_gc_duration_seconds{job=\"$JOB\"}", + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A" + } + ], + "title": "GC duration quantile", + "type": "timeseries" + } + ], + "refresh": "1m", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": "store", + "value": "store" + }, + "description": "Name of the database", + "hide": 0, + "includeAll": false, + "label": "Job", + "multi": false, + "name": "JOB", + "options": [ + { + "selected": true, + "text": "store", + "value": "store" + } + ], + "query": "store", + "queryValue": "", + "skipUrlSync": false, + "type": "custom" + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": false, + "text": "5m", + "value": "5m" + }, + "hide": 0, + "label": "Interval", + "name": "INTERVAL", + "options": [ + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "1m,5m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "queryValue": "", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "gRPC Server", + "uid": "cdydeqdbz124gf", + "version": 6, + "weekStart": "" +} diff --git a/grafana/provisioning/dashboards/dashboards.yaml b/grafana/provisioning/dashboards/dashboards.yaml new file mode 100755 index 00000000..009a1dbe --- /dev/null +++ b/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,13 @@ +apiVersion: 1 + +providers: + - name: 'E-SOH' + orgId: 1 + folder: '' + type: file + updateIntervalSeconds: 10 + disableDeletion: false + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: true # Does not support nested folders (max 1 level) diff --git a/grafana/provisioning/datasources/datasource.yaml b/grafana/provisioning/datasources/datasource.yaml new file mode 100755 index 00000000..9c7764d1 --- /dev/null +++ b/grafana/provisioning/datasources/datasource.yaml @@ -0,0 +1,29 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: true + jsonData: + httpMethod: POST + + - name: Postgres Datastore DB + type: grafana-postgresql-datasource + access: proxy + url: db:5432 + user: postgres + basicAuth: false + isDefault: false + editable: true + secureJsonData: + password: mysecretpassword + jsonData: + connMaxLifetime: 14400 + database: data + maxIdleConns: 3 + maxIdleConnsAuto: true + maxOpenConns: 3 + sslmode: disable diff --git a/justfile b/justfile index 4f8b880f..18fe6a40 100644 --- a/justfile +++ b/justfile @@ -5,9 +5,11 @@ default: set positional-arguments # Run all docker services. After running the database needs cleanup, run just destroy -all: lint build unit services load integration performance client +all: lint build unit services monitoring load integration performance client # Build and run the default docker services up: build services +# Build and run the default docker services and start up monitoring +local: up monitoring # Build and run the unit, load and integration tests test: build unit load integration @@ -41,11 +43,18 @@ _check-python-version: set -euxo pipefail # Get Python version - python_version=$(python --version 2>&1 | cut -d' ' -f2) + if command -v python &>/dev/null; then + python_version=$(python --version 2>&1 | cut -d ' ' -f2) + elif command -v python3 &>/dev/null; then + python_version=$(python3 --version 2>&1 | cut -d ' ' -f2) + else + echo "Python not found. Failing..." + exit 1 + fi # Extract major and minor version numbers - major_version=$(echo "$python_version" | cut -d'.' -f1) - minor_version=$(echo "$python_version" | cut -d'.' -f2) + major_version=$(echo "$python_version" | cut -d '.' -f1) + minor_version=$(echo "$python_version" | cut -d '.' -f2) # Check if Python version is greater than or equal to 3.11 if [[ "$major_version" -lt 3 || ( "$major_version" -eq 3 && "$minor_version" -lt 11 ) ]]; then @@ -93,6 +102,11 @@ unit: docker compose run --rm api-unit +# Start the monitoring +monitoring: + docker compose up prometheus prometheus-postgres-exporter grafana -d + + # Run the performance tests performance: docker compose --env-file ./ci/config/env.list run --rm performance @@ -108,7 +122,7 @@ client: # ---------------------------------------------------------------------------- # # Build the docker images build: copy-proto - docker compose --env-file ./ci/config/env.list --profile test build + docker compose --env-file ./ci/config/env.list --profile monitoring --profile test build # # ---------------------------------------------------------------------------- # @@ -126,9 +140,9 @@ load: # Stop all E-SOH containers down: - docker compose --profile test down + docker compose --profile monitoring --profile test down # Stop all E-SOH containers and remove their volumes destroy: - docker compose --profile test down --volumes + docker compose --profile monitoring --profile test down --volumes diff --git a/monitoring/prometheus.yml b/monitoring/prometheus.yml new file mode 100644 index 00000000..864488aa --- /dev/null +++ b/monitoring/prometheus.yml @@ -0,0 +1,15 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + query_log_file: /prometheus/query.log + +scrape_configs: + - job_name: monitoring + static_configs: + - targets: ["monitoring:9090"] + - job_name: export-db-metrics + static_configs: + - targets: ["export-db-metrics:9187"] + - job_name: store + static_configs: + - targets: [ 'store:8080' ] diff --git a/prometheus/postgres_exporter.yml b/prometheus/postgres_exporter.yml new file mode 100644 index 00000000..373a0743 --- /dev/null +++ b/prometheus/postgres_exporter.yml @@ -0,0 +1,8 @@ +auth_modules: + datastore-database-config: + type: userpass + userpass: + username: postgres + password: mysecretpassword + options: + sslmode: disable diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml new file mode 100644 index 00000000..d016f1a3 --- /dev/null +++ b/prometheus/prometheus.yml @@ -0,0 +1,15 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + query_log_file: /prometheus/query.log + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ["prometheus:9090"] + - job_name: prometheus-postgres-exporter + static_configs: + - targets: ["prometheus-postgres-exporter:9187"] + - job_name: store + static_configs: + - targets: [ 'store:8081' ]