From 9425ef139896e678a200b31c940a07e5fe40ef3a Mon Sep 17 00:00:00 2001 From: Jeffrey Vervoort Date: Fri, 9 Aug 2024 16:37:41 +0200 Subject: [PATCH 01/19] WIP: GO gRPC --- datastore/datastore/go.mod | 30 +++++++++++++--- datastore/datastore/main/main.go | 59 ++++++++++++++++++++++++++++++-- docker-compose.yml | 33 ++++++++++++++++-- monitoring/prometheus.yml | 15 ++++++++ 4 files changed, 128 insertions(+), 9 deletions(-) create mode 100644 monitoring/prometheus.yml diff --git a/datastore/datastore/go.mod b/datastore/datastore/go.mod index a78c29b9..0ee288c5 100644 --- a/datastore/datastore/go.mod +++ b/datastore/datastore/go.mod @@ -2,17 +2,39 @@ module datastore go 1.21 -require google.golang.org/grpc v1.64.0 +toolchain go1.21.12 + +require google.golang.org/grpc v1.65.0 require ( github.com/cridenour/go-postgis v1.0.0 - google.golang.org/protobuf v1.34.1 + github.com/golang/protobuf v1.5.4 + github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 + github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 + github.com/prometheus/client_golang v1.19.1 + go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 + go.opentelemetry.io/otel/trace v1.28.0 + google.golang.org/protobuf v1.34.2 +) + +require ( + cloud.google.com/go/compute/metadata v0.5.0 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/go-logr/logr v1.4.2 // indirect + github.com/go-logr/stdr v1.2.2 // indirect + github.com/prometheus/client_model v0.5.0 // indirect + github.com/prometheus/common v0.48.0 // indirect + github.com/prometheus/procfs v0.12.0 // indirect + go.opentelemetry.io/otel v1.28.0 // indirect + go.opentelemetry.io/otel/metric v1.28.0 // indirect ) require ( github.com/lib/pq v1.10.9 golang.org/x/net v0.26.0 // indirect - golang.org/x/sys v0.21.0 // indirect + golang.org/x/sys v0.22.0 // indirect golang.org/x/text v0.16.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect ) diff --git a/datastore/datastore/main/main.go b/datastore/datastore/main/main.go index 5bf3a654..90b8c1cb 100644 --- a/datastore/datastore/main/main.go +++ b/datastore/datastore/main/main.go @@ -8,15 +8,31 @@ import ( "datastore/storagebackend" "datastore/storagebackend/postgresql" "fmt" + "github.com/prometheus/client_golang/prometheus/promhttp" "log" "net" "time" + // gRPC "google.golang.org/grpc" "google.golang.org/grpc/health" "google.golang.org/grpc/health/grpc_health_v1" "google.golang.org/grpc/peer" + // Monitoring + grpc_prometheus "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" + //"github.com/grpc-ecosystem/go-grpc-middleware" + //"google.golang.org/grpc/ChainUnaryInterceptor" + //"github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors" + //"github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/auth" + //"github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/logging" + //"github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/selector" + //"github.com/prometheus/client_golang/prometheus" + //"github.com/prometheus/client_golang/prometheus/promhttp" + //"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" + //"github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/recovery" + //"go.opentelemetry.io/otel/trace" + _ "expvar" "net/http" _ "net/http/pprof" @@ -52,8 +68,41 @@ func main() { return resp, err } - // create gRPC server - server := grpc.NewServer(grpc.UnaryInterceptor(reqTimeLogger)) + //srvMetrics := grpcprom.NewServerMetrics() + //reg := prometheus.NewRegistry() + //reg.MustRegister(srvMetrics) + //exemplarFromContext := func(ctx context.Context) prometheus.Labels { + // if span := trace.SpanContextFromContext(ctx); span.IsSampled() { + // return prometheus.Labels{"traceID": span.TraceID().String()} + // } + // return nil + //} + grpcMetrics := grpc_prometheus.NewServerMetrics() + + // create gRPC server with middleware + server := grpc.NewServer( + grpc.ChainUnaryInterceptor(reqTimeLogger), + grpc.ChainUnaryInterceptor( + grpcMetrics.UnaryServerInterceptor(), + ), + ) + //server := grpc.NewServer( + // grpc.ChainUnaryInterceptor(reqTimeLogger), + // grpc.ChainUnaryInterceptor( + // // Order matters e.g. tracing interceptor have to create span first for the later exemplars to work. + // otelgrpc.UnaryServerInterceptor(), + // srvMetrics.UnaryServerInterceptor(grpcprom.WithExemplarFromContext(exemplarFromContext)), + // logging.UnaryServerInterceptor(interceptorLogger(rpcLogger), logging.WithFieldsFromContext(logTraceID)), + // recovery.UnaryServerInterceptor(recovery.WithRecoveryHandler(grpcPanicRecoveryHandler)), + // ), + // grpc.ChainStreamInterceptor( + // otelgrpc.StreamServerInterceptor(), + // srvMetrics.StreamServerInterceptor(grpcprom.WithExemplarFromContext(exemplarFromContext)), + // logging.StreamServerInterceptor(interceptorLogger(rpcLogger), logging.WithFieldsFromContext(logTraceID)), + // recovery.StreamServerInterceptor(recovery.WithRecoveryHandler(grpcPanicRecoveryHandler)), + // ), + //) + grpcMetrics.InitializeMetrics(server) grpc_health_v1.RegisterHealthServer(server, health.NewServer()) // create storage backend @@ -75,6 +124,12 @@ func main() { log.Fatalf("net.Listen() failed: %v", err) } + go func() { + log.Println("Starting HTTP server for Prometheus metrics on :8080") + http.Handle("/metrics", promhttp.Handler()) + log.Fatal(http.ListenAndServe(":8080", nil)) + }() + // serve profiling info log.Printf("serving profiling info\n") go func() { diff --git a/docker-compose.yml b/docker-compose.yml index 29333ea1..d4c67460 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,8 +10,11 @@ services: volumes: # - ts-data:/home/postgres/pgdata/data # for timescale image - ts-data:/var/lib/postgresql # for postgres image - - ./datastore/database/healthcheck_postgis_uptime.sh:/healthcheck_postgis_uptime.sh # for the healthcheck + - ./datastore/database/db.conf:/etc/conf_settings/db.conf:ro + - pg_logs:/logs + - ./datastore/database/healthcheck_postgis_uptime.sh:/healthcheck_postgis_uptime.sh:ro # for the healthcheck environment: + - EXTRA_CONF_DIR=/etc/conf_settings - POSTGRES_USER=postgres - POSTGRES_PASSWORD=mysecretpassword - POSTGRES_DB=data @@ -26,7 +29,7 @@ services: ] interval: 5s timeout: 1s - retries: 3 + retries: 30 start_period: 30s # Failures in 30 seconds do not mark container as unhealthy migrate: @@ -156,10 +159,34 @@ services: - DSHOST=store - DSPORT=50050 volumes: - - ./datastore/load-test/output:/load-test/output + - ./datastore/load-test/output:/load-test/output:rw depends_on: store: condition: service_healthy + monitoring: + image: prom/prometheus + ports: + - "9090:9090" + volumes: + - ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - prometheus-data:/prometheus:rw + + export-db-metrics: + image: quay.io/prometheuscommunity/postgres-exporter + environment: + - DATA_SOURCE_URI=db:5432/data + - DATA_SOURCE_USER=postgres + - DATA_SOURCE_PASS=mysecretpassword + ports: + - "9187:9187" + volumes: + - ./monitoring/postgres_exporter.yml:/postgres_exporter.yml:ro + depends_on: + db: + condition: service_healthy + volumes: ts-data: + pg_logs: + prometheus-data: diff --git a/monitoring/prometheus.yml b/monitoring/prometheus.yml new file mode 100644 index 00000000..864488aa --- /dev/null +++ b/monitoring/prometheus.yml @@ -0,0 +1,15 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + query_log_file: /prometheus/query.log + +scrape_configs: + - job_name: monitoring + static_configs: + - targets: ["monitoring:9090"] + - job_name: export-db-metrics + static_configs: + - targets: ["export-db-metrics:9187"] + - job_name: store + static_configs: + - targets: [ 'store:8080' ] From 6d2f5d31c2c9dbce535ac2deb36351a4436123bb Mon Sep 17 00:00:00 2001 From: Jeffrey Vervoort Date: Tue, 10 Sep 2024 17:08:04 +0200 Subject: [PATCH 02/19] Setup database logging. --- datastore/database/extra.conf | 15 +++++++++++++++ docker-compose.yml | 11 ++++++++--- 2 files changed, 23 insertions(+), 3 deletions(-) create mode 100644 datastore/database/extra.conf diff --git a/datastore/database/extra.conf b/datastore/database/extra.conf new file mode 100644 index 00000000..c928ec2c --- /dev/null +++ b/datastore/database/extra.conf @@ -0,0 +1,15 @@ +# Log config +listen_addresses = '*' +logging_collector = 'on' +log_destination = 'stderr' +log_statement = 'all' +log_directory = 'log' +log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log' +log_duration = 'on' + +# postgres-exporter +shared_preload_libraries = 'pg_cron,pg_stat_statements' +track_activity_query_size = 2048 +pg_stat_statements.track = 'all' +pg_stat_statements.max = 10000 +pg_stat_statements.save = 'on' diff --git a/docker-compose.yml b/docker-compose.yml index d4c67460..5724994c 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,3 @@ -version: "3.8" name: datastore services: @@ -10,8 +9,10 @@ services: volumes: # - ts-data:/home/postgres/pgdata/data # for timescale image - ts-data:/var/lib/postgresql # for postgres image - - ./datastore/database/db.conf:/etc/conf_settings/db.conf:ro - - pg_logs:/logs +# - ./datastore/database/custom.conf:/etc/postgresql/15/main/custom.conf:ro +# - ./datastore/database/extra.conf:/settings/extra.conf:ro + - ./datastore/database/extra.conf:/etc/conf_settings/extra.conf:ro + - pg_logs:/log - ./datastore/database/healthcheck_postgis_uptime.sh:/healthcheck_postgis_uptime.sh:ro # for the healthcheck environment: - EXTRA_CONF_DIR=/etc/conf_settings @@ -31,6 +32,10 @@ services: timeout: 1s retries: 30 start_period: 30s # Failures in 30 seconds do not mark container as unhealthy +# command: ["-c", "config_file=/etc/postgresql/15/main/custom.conf && postgres restart"] +# command: > +# -c "echo 'include ''custom.conf''' >> /etc/postgresql/15/main/postgresql.conf +# && postgres" migrate: build: From d2b65d0918c508440d1bac841224587446051420 Mon Sep 17 00:00:00 2001 From: Jeffrey Vervoort Date: Tue, 10 Sep 2024 18:08:11 +0200 Subject: [PATCH 03/19] Enable collector on postgres exporter to gather pg_stat_statements from the database to prometheus. --- docker-compose.yml | 1 + 1 file changed, 1 insertion(+) diff --git a/docker-compose.yml b/docker-compose.yml index 5724994c..fdae8ef4 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -190,6 +190,7 @@ services: depends_on: db: condition: service_healthy + command: "--collector.stat_statements" volumes: ts-data: From 654f7480bb563de8a761a06f1f7f4a7a2b34c778 Mon Sep 17 00:00:00 2001 From: Jeffrey Vervoort Date: Wed, 11 Sep 2024 08:28:58 +0200 Subject: [PATCH 04/19] Add migration for to setup the pg_stat_statements extension. --- .../migrations/1723033622_setup_prometheus_extension.down.sql | 1 + .../data/migrations/1723033622_setup_prometheus_extension.up.sql | 1 + 2 files changed, 2 insertions(+) create mode 100644 datastore/migrate/data/migrations/1723033622_setup_prometheus_extension.down.sql create mode 100644 datastore/migrate/data/migrations/1723033622_setup_prometheus_extension.up.sql diff --git a/datastore/migrate/data/migrations/1723033622_setup_prometheus_extension.down.sql b/datastore/migrate/data/migrations/1723033622_setup_prometheus_extension.down.sql new file mode 100644 index 00000000..36bec35c --- /dev/null +++ b/datastore/migrate/data/migrations/1723033622_setup_prometheus_extension.down.sql @@ -0,0 +1 @@ +DROP EXTENSION pg_stat_statements; diff --git a/datastore/migrate/data/migrations/1723033622_setup_prometheus_extension.up.sql b/datastore/migrate/data/migrations/1723033622_setup_prometheus_extension.up.sql new file mode 100644 index 00000000..841ff0c4 --- /dev/null +++ b/datastore/migrate/data/migrations/1723033622_setup_prometheus_extension.up.sql @@ -0,0 +1 @@ +CREATE EXTENSION IF NOT EXISTS pg_stat_statements; From ab1d0b345ecb5d20f013195a8261c86897825050 Mon Sep 17 00:00:00 2001 From: Jeffrey Vervoort Date: Mon, 16 Sep 2024 13:51:07 +0200 Subject: [PATCH 05/19] Setup Grafana dashboard for the database performance based on Prometheus and Postgres datasource. --- docker-compose.yml | 35 +- grafana/dashboards/database-performance.json | 1674 +++++++++++++++++ .../provisioning/dashboards/dashboards.yaml | 13 + .../provisioning/datasources/datasource.yaml | 29 + prometheus/postgres_exporter.yml | 8 + prometheus/prometheus.yml | 15 + 6 files changed, 1761 insertions(+), 13 deletions(-) create mode 100644 grafana/dashboards/database-performance.json create mode 100644 grafana/provisioning/dashboards/dashboards.yaml create mode 100644 grafana/provisioning/datasources/datasource.yaml create mode 100644 prometheus/postgres_exporter.yml create mode 100644 prometheus/prometheus.yml diff --git a/docker-compose.yml b/docker-compose.yml index fdae8ef4..1d51b7f5 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -9,9 +9,7 @@ services: volumes: # - ts-data:/home/postgres/pgdata/data # for timescale image - ts-data:/var/lib/postgresql # for postgres image -# - ./datastore/database/custom.conf:/etc/postgresql/15/main/custom.conf:ro -# - ./datastore/database/extra.conf:/settings/extra.conf:ro - - ./datastore/database/extra.conf:/etc/conf_settings/extra.conf:ro + - ./datastore/database/extra.conf:/etc/conf_settings/extra.conf:ro # Extra Postgres configuration - pg_logs:/log - ./datastore/database/healthcheck_postgis_uptime.sh:/healthcheck_postgis_uptime.sh:ro # for the healthcheck environment: @@ -32,10 +30,6 @@ services: timeout: 1s retries: 30 start_period: 30s # Failures in 30 seconds do not mark container as unhealthy -# command: ["-c", "config_file=/etc/postgresql/15/main/custom.conf && postgres restart"] -# command: > -# -c "echo 'include ''custom.conf''' >> /etc/postgresql/15/main/postgresql.conf -# && postgres" migrate: build: @@ -169,15 +163,15 @@ services: store: condition: service_healthy - monitoring: + prometheus: image: prom/prometheus ports: - "9090:9090" volumes: - - ./monitoring/prometheus.yml:/etc/prometheus/prometheus.yml:ro - - prometheus-data:/prometheus:rw + - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml:ro + - prometheus-data:/prometheus - export-db-metrics: + prometheus-postgres-exporter: image: quay.io/prometheuscommunity/postgres-exporter environment: - DATA_SOURCE_URI=db:5432/data @@ -186,13 +180,28 @@ services: ports: - "9187:9187" volumes: - - ./monitoring/postgres_exporter.yml:/postgres_exporter.yml:ro + - ./prometheus/postgres_exporter.yml:/postgres_exporter.yml:ro depends_on: db: condition: service_healthy - command: "--collector.stat_statements" + command: ["--collector.stat_statements", "--collector.stat_user_tables", "--collector.stat_activity_autovacuum"] + + grafana: + image: grafana/grafana-oss:11.2.0 + ports: + - "3000:3000" + environment: + - GF_SECURITY_ADMIN_USER=admin + - GF_SECURITY_ADMIN_PASSWORD=mysecretpassword + volumes: + - grafana-storage:/var/lib/grafana + - ./grafana/provisioning:/etc/grafana/provisioning + - ./grafana/dashboards:/var/lib/grafana/dashboards + depends_on: + - prometheus volumes: ts-data: pg_logs: prometheus-data: + grafana-storage: diff --git a/grafana/dashboards/database-performance.json b/grafana/dashboards/database-performance.json new file mode 100644 index 00000000..31967ffc --- /dev/null +++ b/grafana/dashboards/database-performance.json @@ -0,0 +1,1674 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 12, + "panels": [], + "title": "General", + "type": "row" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 1 + }, + { + "color": "green", + "value": 2 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 0, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "round((time() - process_start_time_seconds{instance=~\"$INSTANCE\"}), 0.01)", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Uptime", + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "orange", + "value": 70 + }, + { + "color": "green", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 8, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "background_solid", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "round(100*sum(pg_stat_database_blks_hit{instance=~\"$INSTANCE\"}) / (sum(pg_stat_database_blks_hit{instance=~\"$INSTANCE\"}) + sum(pg_stat_database_blks_read{instance=~\"$INSTANCE\"})),0.1)", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Buffer cache hit rate", + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "green", + "value": 1 + }, + { + "color": "#EAB839", + "value": 75 + }, + { + "color": "red", + "value": 85 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 8, + "x": 16, + "y": 1 + }, + "id": 5, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "(sum(pg_stat_activity_count) / sum(pg_settings_max_connections)) * 100", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Percentage of connections out of the total.", + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "decimals": 0, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "#EAB839", + "value": 0 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 7, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 13, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum by (state) (pg_stat_activity_count)", + "format": "time_series", + "instant": false, + "interval": "", + "legendFormat": "{{state}}", + "range": true, + "refId": "A" + } + ], + "title": "Number of active connections", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 12 + }, + "id": 17, + "panels": [], + "title": "Query Performance", + "type": "row" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 13 + }, + "id": 1, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "disableTextWrap": false, + "editorMode": "code", + "exemplar": false, + "expr": "sum(rate(pg_stat_statements_seconds_total{instance=~\"$INSTANCE\", datname=~\"$DATNAME\"}[1m])) / sum(rate(pg_stat_statements_calls_total{instance=~\"$INSTANCE\", datname=~\"$DATNAME\"}[1m]))", + "format": "time_series", + "fullMetaSearch": false, + "includeNullMetadata": true, + "instant": false, + "interval": "", + "legendFormat": "Query time", + "range": true, + "refId": "A", + "useBackend": false + } + ], + "title": "Average query time", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 13 + }, + "id": 3, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(rate(pg_stat_statements_calls_total{instance=~\"$INSTANCE\"}[1m]))", + "instant": false, + "legendFormat": "Queries per second", + "range": true, + "refId": "A" + } + ], + "title": "Number of queries per second", + "type": "timeseries" + }, + { + "datasource": { + "default": false, + "type": "grafana-postgresql-datasource", + "uid": "P5C9DBE12543779E4" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "query" + }, + "properties": [ + { + "id": "custom.minWidth", + "value": 400 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 19 + }, + "id": 7, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [] + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "P5C9DBE12543779E4" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT queryid, query, calls\nFROM pg_stat_statements\nWHERE query NOT LIKE '%CREATE%'\nORDER BY calls DESC\nLIMIT 10;", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "pg_stat_statements" + } + ], + "title": "Top 10 most called queries", + "type": "table" + }, + { + "datasource": { + "default": false, + "type": "datasource", + "uid": "-- Mixed --" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "custom": { + "align": "auto", + "cellOptions": { + "type": "auto" + }, + "inspect": false + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "mean_exec_time" + }, + "properties": [ + { + "id": "unit", + "value": "ms" + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "query" + }, + "properties": [ + { + "id": "custom.minWidth", + "value": 400 + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 19 + }, + "id": 6, + "options": { + "cellHeight": "sm", + "footer": { + "countRows": false, + "fields": "", + "reducer": [ + "sum" + ], + "show": false + }, + "showHeader": true, + "sortBy": [ + { + "desc": true, + "displayName": "mean_exec_time" + } + ] + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "P5C9DBE12543779E4" + }, + "editorMode": "code", + "format": "table", + "hide": false, + "rawQuery": true, + "rawSql": "SELECT queryid,query, mean_exec_time\nFROM pg_stat_statements\nWHERE query NOT LIKE '%CREATE%'\nORDER BY mean_exec_time DESC\nLIMIT 10;", + "refId": "B", + "sql": { + "columns": [ + { + "parameters": [ + { + "name": "queryid", + "type": "functionParameter" + } + ], + "type": "function" + }, + { + "parameters": [ + { + "name": "query", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "pg_stat_statements" + } + ], + "title": "Top 10 slowest queries in micro seconds", + "type": "table" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 24, + "x": 0, + "y": 27 + }, + "id": 14, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(pg_stat_database_tup_fetched[5m]) > 1000", + "instant": false, + "legendFormat": "{{datname}} SELECT (index scan)", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(pg_stat_database_tup_returned[5m]) > 1000", + "hide": false, + "instant": false, + "legendFormat": "{{datname}} SELECT (table scan)", + "range": true, + "refId": "B" + } + ], + "title": "Read statistics", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 10, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 36 + }, + "id": 15, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(pg_stat_database_tup_inserted[5m]) > 1", + "instant": false, + "legendFormat": "{{datname}} INSERT", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(pg_stat_database_tup_updated[5m]) > 1", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "{{datname}} UPDATE", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(pg_stat_database_tup_deleted[5m]) > 1", + "hide": false, + "instant": false, + "legendFormat": "{{datname}} DELETE", + "range": true, + "refId": "C" + } + ], + "title": "Write statistics", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 44 + }, + "id": 18, + "panels": [], + "title": "Errors", + "type": "row" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 45 + }, + "id": 19, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(pg_stat_database_deadlocks[1m])", + "instant": false, + "interval": "1m", + "legendFormat": "{{datname}}", + "range": true, + "refId": "A" + } + ], + "title": "Deadlocks", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 45 + }, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(rate(pg_stat_database_conflicts_confl_deadlock[1m]))", + "instant": false, + "legendFormat": "Deadlock", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(rate(pg_stat_database_conflicts_confl_bufferpin[5m]))", + "hide": false, + "instant": false, + "legendFormat": "Bufferpin", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(rate(pg_stat_database_conflicts_confl_lock[5m]))", + "hide": false, + "instant": false, + "legendFormat": "Lock", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(rate(pg_stat_database_conflicts_confl_snapshot[5m]))", + "hide": false, + "instant": false, + "legendFormat": "Snapshot", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(rate(pg_stat_database_conflicts_confl_tablespace[5m]))", + "hide": false, + "instant": false, + "legendFormat": "Tablespace", + "range": true, + "refId": "E" + } + ], + "title": "Conflicts", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 53 + }, + "id": 16, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "sum by (datname, mode) (pg_locks_count{instance=~\"$INSTANCE\", datname=~\"$DATNAME\"})", + "instant": false, + "interval": "1m", + "legendFormat": "{{datname}} {{mode}}", + "range": true, + "refId": "A" + } + ], + "title": "Table locks", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 61 + }, + "id": 10, + "panels": [], + "title": "Database Size", + "type": "row" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green" + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 62 + }, + "id": 9, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(pg_database_size_bytes)", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Total database size", + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 62 + }, + "id": 8, + "options": { + "displayLabels": [ + "name", + "percent" + ], + "legend": { + "displayMode": "list", + "placement": "right", + "showLegend": true + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "exemplar": false, + "expr": "pg_database_size_bytes", + "instant": false, + "interval": "", + "legendFormat": "{{ datname }}", + "range": true, + "refId": "A" + } + ], + "title": "Size per database", + "type": "piechart" + } + ], + "refresh": "1m", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "allValue": "", + "current": { + "selected": true, + "text": [ + "data" + ], + "value": [ + "data" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(datname)", + "hide": 0, + "includeAll": true, + "label": "Database name", + "multi": true, + "name": "DATNAME", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(datname)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "current": { + "selected": false, + "text": "prometheus-postgres-exporter:9187", + "value": "prometheus-postgres-exporter:9187" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(instance)", + "hide": 0, + "includeAll": false, + "label": "Instance", + "multi": false, + "name": "INSTANCE", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(instance)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "Database Performance", + "uid": "adxl34wpc3if4a", + "version": 10, + "weekStart": "" +} diff --git a/grafana/provisioning/dashboards/dashboards.yaml b/grafana/provisioning/dashboards/dashboards.yaml new file mode 100644 index 00000000..009a1dbe --- /dev/null +++ b/grafana/provisioning/dashboards/dashboards.yaml @@ -0,0 +1,13 @@ +apiVersion: 1 + +providers: + - name: 'E-SOH' + orgId: 1 + folder: '' + type: file + updateIntervalSeconds: 10 + disableDeletion: false + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: true # Does not support nested folders (max 1 level) diff --git a/grafana/provisioning/datasources/datasource.yaml b/grafana/provisioning/datasources/datasource.yaml new file mode 100644 index 00000000..9c7764d1 --- /dev/null +++ b/grafana/provisioning/datasources/datasource.yaml @@ -0,0 +1,29 @@ +apiVersion: 1 + +datasources: + - name: Prometheus + type: prometheus + access: proxy + url: http://prometheus:9090 + isDefault: true + editable: true + jsonData: + httpMethod: POST + + - name: Postgres Datastore DB + type: grafana-postgresql-datasource + access: proxy + url: db:5432 + user: postgres + basicAuth: false + isDefault: false + editable: true + secureJsonData: + password: mysecretpassword + jsonData: + connMaxLifetime: 14400 + database: data + maxIdleConns: 3 + maxIdleConnsAuto: true + maxOpenConns: 3 + sslmode: disable diff --git a/prometheus/postgres_exporter.yml b/prometheus/postgres_exporter.yml new file mode 100644 index 00000000..373a0743 --- /dev/null +++ b/prometheus/postgres_exporter.yml @@ -0,0 +1,8 @@ +auth_modules: + datastore-database-config: + type: userpass + userpass: + username: postgres + password: mysecretpassword + options: + sslmode: disable diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml new file mode 100644 index 00000000..c44803fb --- /dev/null +++ b/prometheus/prometheus.yml @@ -0,0 +1,15 @@ +global: + scrape_interval: 15s + evaluation_interval: 15s + query_log_file: /prometheus/query.log + +scrape_configs: + - job_name: prometheus + static_configs: + - targets: ["prometheus:9090"] + - job_name: prometheus-postgres-exporter + static_configs: + - targets: ["prometheus-postgres-exporter:9187"] + - job_name: store + static_configs: + - targets: [ 'store:8080' ] From 0903bd36539c48ba0490b4ca113f99d0e45b340c Mon Sep 17 00:00:00 2001 From: Jeffrey Vervoort Date: Wed, 18 Sep 2024 12:55:25 +0200 Subject: [PATCH 06/19] Expose logging via local volume, document watch logs command and set log files to daily. --- .gitignore | 3 +++ datastore/database/README.md | 10 ++++++++++ datastore/database/extra.conf | 8 ++++++-- datastore/database/logs/.gitkeep | 0 docker-compose.yml | 7 +++---- 5 files changed, 22 insertions(+), 6 deletions(-) create mode 100644 datastore/database/README.md create mode 100755 datastore/database/logs/.gitkeep diff --git a/.gitignore b/.gitignore index d6a7061c..1b91787a 100644 --- a/.gitignore +++ b/.gitignore @@ -212,3 +212,6 @@ cf_standard_names_v84.txt # API api/test/output/ datastore/load-test/output/ + +# Database +datastore/database/logs/*.log diff --git a/datastore/database/README.md b/datastore/database/README.md new file mode 100644 index 00000000..91aa41f4 --- /dev/null +++ b/datastore/database/README.md @@ -0,0 +1,10 @@ +# Database logs +Set UID and GID of the postgres user +`sudo chown -R 101:103 ./logs` + + +To watch the database logs: + +`sudo tail -f ./logs/postgresql-%Y-%m-%d.log` + +`sudo tail -f ./logs/postgresql-2024-09-18.log` diff --git a/datastore/database/extra.conf b/datastore/database/extra.conf index c928ec2c..3df96940 100644 --- a/datastore/database/extra.conf +++ b/datastore/database/extra.conf @@ -3,9 +3,13 @@ listen_addresses = '*' logging_collector = 'on' log_destination = 'stderr' log_statement = 'all' -log_directory = 'log' -log_filename = 'postgresql-%Y-%m-%d_%H%M%S.log' +log_directory = '/logs' log_duration = 'on' +# TODO: Consider changing log rotation to reduce file size +log_filename = 'postgresql-%Y-%m-%d.log' +log_rotation_age = 1d +log_rotation_size = 0 + # postgres-exporter shared_preload_libraries = 'pg_cron,pg_stat_statements' diff --git a/datastore/database/logs/.gitkeep b/datastore/database/logs/.gitkeep new file mode 100755 index 00000000..e69de29b diff --git a/docker-compose.yml b/docker-compose.yml index 1d51b7f5..dc20e4c1 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,7 @@ services: # - ts-data:/home/postgres/pgdata/data # for timescale image - ts-data:/var/lib/postgresql # for postgres image - ./datastore/database/extra.conf:/etc/conf_settings/extra.conf:ro # Extra Postgres configuration - - pg_logs:/log + - ./datastore/database/logs:/logs:rw - ./datastore/database/healthcheck_postgis_uptime.sh:/healthcheck_postgis_uptime.sh:ro # for the healthcheck environment: - EXTRA_CONF_DIR=/etc/conf_settings @@ -195,13 +195,12 @@ services: - GF_SECURITY_ADMIN_PASSWORD=mysecretpassword volumes: - grafana-storage:/var/lib/grafana - - ./grafana/provisioning:/etc/grafana/provisioning - - ./grafana/dashboards:/var/lib/grafana/dashboards + - ./grafana/provisioning:/etc/grafana/provisioning:rw + - ./grafana/dashboards:/var/lib/grafana/dashboards:rw depends_on: - prometheus volumes: ts-data: - pg_logs: prometheus-data: grafana-storage: From b2c462e6b1ad216dc465acfd951dbff3efd95ed0 Mon Sep 17 00:00:00 2001 From: Jeffrey Vervoort Date: Thu, 19 Sep 2024 15:08:14 +0200 Subject: [PATCH 07/19] Add gRPC Go metrics. --- datastore/datastore/go.mod | 12 +++--- datastore/datastore/main/main.go | 72 +++++++++++++++++--------------- docker-compose.yml | 1 + prometheus/prometheus.yml | 2 +- 4 files changed, 46 insertions(+), 41 deletions(-) diff --git a/datastore/datastore/go.mod b/datastore/datastore/go.mod index 0ee288c5..e755bdef 100644 --- a/datastore/datastore/go.mod +++ b/datastore/datastore/go.mod @@ -9,26 +9,26 @@ require google.golang.org/grpc v1.65.0 require ( github.com/cridenour/go-postgis v1.0.0 github.com/golang/protobuf v1.5.4 - github.com/grpc-ecosystem/go-grpc-middleware v1.4.0 github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 + github.com/oklog/run v1.1.0 github.com/prometheus/client_golang v1.19.1 - go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.53.0 go.opentelemetry.io/otel/trace v1.28.0 google.golang.org/protobuf v1.34.2 ) require ( - cloud.google.com/go/compute/metadata v0.5.0 // indirect github.com/beorn7/perks v1.0.1 // indirect github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/go-logr/logr v1.4.2 // indirect - github.com/go-logr/stdr v1.2.2 // indirect + github.com/davecgh/go-spew v1.1.1 // indirect + github.com/kr/text v0.1.0 // indirect + github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/client_model v0.5.0 // indirect github.com/prometheus/common v0.48.0 // indirect github.com/prometheus/procfs v0.12.0 // indirect + github.com/stretchr/testify v1.9.0 // indirect go.opentelemetry.io/otel v1.28.0 // indirect - go.opentelemetry.io/otel/metric v1.28.0 // indirect + gopkg.in/yaml.v3 v3.0.1 // indirect ) require ( diff --git a/datastore/datastore/main/main.go b/datastore/datastore/main/main.go index 90b8c1cb..a6dfee21 100644 --- a/datastore/datastore/main/main.go +++ b/datastore/datastore/main/main.go @@ -8,7 +8,9 @@ import ( "datastore/storagebackend" "datastore/storagebackend/postgresql" "fmt" + "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" + "go.opentelemetry.io/otel/trace" "log" "net" "time" @@ -68,40 +70,31 @@ func main() { return resp, err } - //srvMetrics := grpcprom.NewServerMetrics() - //reg := prometheus.NewRegistry() - //reg.MustRegister(srvMetrics) - //exemplarFromContext := func(ctx context.Context) prometheus.Labels { - // if span := trace.SpanContextFromContext(ctx); span.IsSampled() { - // return prometheus.Labels{"traceID": span.TraceID().String()} - // } - // return nil - //} - grpcMetrics := grpc_prometheus.NewServerMetrics() + grpcMetrics := grpc_prometheus.NewServerMetrics( + grpc_prometheus.WithServerHandlingTimeHistogram( + grpc_prometheus.WithHistogramBuckets([]float64{0.001, 0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120}), + ), + ) + reg := prometheus.NewRegistry() + reg.MustRegister(grpcMetrics) + exemplarFromContext := func(ctx context.Context) prometheus.Labels { + if span := trace.SpanContextFromContext(ctx); span.IsSampled() { + return prometheus.Labels{"traceID": span.TraceID().String()} + } + return nil + } // create gRPC server with middleware server := grpc.NewServer( - grpc.ChainUnaryInterceptor(reqTimeLogger), grpc.ChainUnaryInterceptor( - grpcMetrics.UnaryServerInterceptor(), + reqTimeLogger, + grpcMetrics.UnaryServerInterceptor(grpc_prometheus.WithExemplarFromContext(exemplarFromContext)), + ), + grpc.ChainStreamInterceptor( + grpcMetrics.StreamServerInterceptor(grpc_prometheus.WithExemplarFromContext(exemplarFromContext)), ), ) - //server := grpc.NewServer( - // grpc.ChainUnaryInterceptor(reqTimeLogger), - // grpc.ChainUnaryInterceptor( - // // Order matters e.g. tracing interceptor have to create span first for the later exemplars to work. - // otelgrpc.UnaryServerInterceptor(), - // srvMetrics.UnaryServerInterceptor(grpcprom.WithExemplarFromContext(exemplarFromContext)), - // logging.UnaryServerInterceptor(interceptorLogger(rpcLogger), logging.WithFieldsFromContext(logTraceID)), - // recovery.UnaryServerInterceptor(recovery.WithRecoveryHandler(grpcPanicRecoveryHandler)), - // ), - // grpc.ChainStreamInterceptor( - // otelgrpc.StreamServerInterceptor(), - // srvMetrics.StreamServerInterceptor(grpcprom.WithExemplarFromContext(exemplarFromContext)), - // logging.StreamServerInterceptor(interceptorLogger(rpcLogger), logging.WithFieldsFromContext(logTraceID)), - // recovery.StreamServerInterceptor(recovery.WithRecoveryHandler(grpcPanicRecoveryHandler)), - // ), - //) + grpcMetrics.InitializeMetrics(server) grpc_health_v1.RegisterHealthServer(server, health.NewServer()) @@ -124,18 +117,29 @@ func main() { log.Fatalf("net.Listen() failed: %v", err) } - go func() { - log.Println("Starting HTTP server for Prometheus metrics on :8080") - http.Handle("/metrics", promhttp.Handler()) - log.Fatal(http.ListenAndServe(":8080", nil)) - }() - // serve profiling info log.Printf("serving profiling info\n") go func() { http.ListenAndServe("0.0.0.0:6060", nil) }() + go func() { + httpSrv := &http.Server{Addr: "0.0.0.0:8081"} + m := http.NewServeMux() + //log.Println("Starting HTTP server for Prometheus metrics on :8081") + // Create HTTP handler for Prometheus metrics. + m.Handle("/metrics", promhttp.HandlerFor( + reg, + promhttp.HandlerOpts{ + // Opt into OpenMetrics e.g. to support exemplars. + EnableOpenMetrics: true, + }, + )) + httpSrv.Handler = m + log.Println("Starting HTTP server for Prometheus metrics on :8081") + log.Fatal(httpSrv.ListenAndServe()) + }() + // serve incoming requests log.Printf("starting server\n") if err := server.Serve(listener); err != nil { diff --git a/docker-compose.yml b/docker-compose.yml index dc20e4c1..81cdeb80 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -48,6 +48,7 @@ services: ports: - "50050:50050" - "6060:6060" # for flame graphing + - "8081:8081" environment: - PGHOST=db - PGPORT=5432 diff --git a/prometheus/prometheus.yml b/prometheus/prometheus.yml index c44803fb..d016f1a3 100644 --- a/prometheus/prometheus.yml +++ b/prometheus/prometheus.yml @@ -12,4 +12,4 @@ scrape_configs: - targets: ["prometheus-postgres-exporter:9187"] - job_name: store static_configs: - - targets: [ 'store:8080' ] + - targets: [ 'store:8081' ] From 3eeb83bbba9bf6f29abef1942feb85acbc4162f8 Mon Sep 17 00:00:00 2001 From: Jeffrey Vervoort Date: Fri, 20 Sep 2024 15:47:08 +0200 Subject: [PATCH 08/19] Add inspect to query top 10. --- grafana/dashboards/database-performance.json | 30 ++++++++++++++------ 1 file changed, 22 insertions(+), 8 deletions(-) diff --git a/grafana/dashboards/database-performance.json b/grafana/dashboards/database-performance.json index 31967ffc..d1b8f416 100644 --- a/grafana/dashboards/database-performance.json +++ b/grafana/dashboards/database-performance.json @@ -18,7 +18,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 2, + "id": 3, "links": [], "panels": [ { @@ -641,6 +641,10 @@ { "id": "custom.minWidth", "value": 400 + }, + { + "id": "custom.inspect", + "value": true } ] } @@ -756,6 +760,10 @@ { "id": "custom.minWidth", "value": 400 + }, + { + "id": "custom.inspect", + "value": true } ] } @@ -886,7 +894,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1000,7 +1009,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1141,7 +1151,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1243,7 +1254,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1391,7 +1403,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null }, { "color": "red", @@ -1469,7 +1482,8 @@ "mode": "absolute", "steps": [ { - "color": "green" + "color": "green", + "value": null } ] }, @@ -1669,6 +1683,6 @@ "timezone": "browser", "title": "Database Performance", "uid": "adxl34wpc3if4a", - "version": 10, + "version": 2, "weekStart": "" } From d1b50a500d32bde31f507f36e7983c2782f1080d Mon Sep 17 00:00:00 2001 From: Jeffrey Vervoort Date: Fri, 20 Sep 2024 16:54:33 +0200 Subject: [PATCH 09/19] Add gRPC store dashboard. --- datastore/datastore/.dockerignore | 1 + datastore/datastore/go.mod | 45 +- datastore/datastore/main/main.go | 38 +- .../datastore/metrics/promservermetrics.go | 67 + grafana/dashboards/grpc-server.json | 1763 +++++++++++++++++ 5 files changed, 1862 insertions(+), 52 deletions(-) create mode 100644 datastore/datastore/metrics/promservermetrics.go create mode 100644 grafana/dashboards/grpc-server.json diff --git a/datastore/datastore/.dockerignore b/datastore/datastore/.dockerignore index 3a6d4f56..72c7bba9 100644 --- a/datastore/datastore/.dockerignore +++ b/datastore/datastore/.dockerignore @@ -5,3 +5,4 @@ !common !dsimpl !storagebackend +!metrics diff --git a/datastore/datastore/go.mod b/datastore/datastore/go.mod index e755bdef..f4a37f3b 100644 --- a/datastore/datastore/go.mod +++ b/datastore/datastore/go.mod @@ -2,39 +2,30 @@ module datastore go 1.21 -toolchain go1.21.12 - -require google.golang.org/grpc v1.65.0 +require google.golang.org/grpc v1.64.0 require ( - github.com/cridenour/go-postgis v1.0.0 - github.com/golang/protobuf v1.5.4 - github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 - github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 - github.com/oklog/run v1.1.0 - github.com/prometheus/client_golang v1.19.1 - go.opentelemetry.io/otel/trace v1.28.0 - google.golang.org/protobuf v1.34.2 + github.com/cridenour/go-postgis v1.0.0 + github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus v1.0.1 + github.com/prometheus/client_golang v1.20.4 + google.golang.org/protobuf v1.34.2 ) require ( - github.com/beorn7/perks v1.0.1 // indirect - github.com/cespare/xxhash/v2 v2.3.0 // indirect - github.com/davecgh/go-spew v1.1.1 // indirect - github.com/kr/text v0.1.0 // indirect - github.com/pmezard/go-difflib v1.0.0 // indirect - github.com/prometheus/client_model v0.5.0 // indirect - github.com/prometheus/common v0.48.0 // indirect - github.com/prometheus/procfs v0.12.0 // indirect - github.com/stretchr/testify v1.9.0 // indirect - go.opentelemetry.io/otel v1.28.0 // indirect - gopkg.in/yaml.v3 v3.0.1 // indirect + github.com/beorn7/perks v1.0.1 // indirect + github.com/cespare/xxhash/v2 v2.3.0 // indirect + github.com/grpc-ecosystem/go-grpc-middleware/v2 v2.1.0 // indirect + github.com/klauspost/compress v1.17.9 // indirect + github.com/munnerz/goautoneg v0.0.0-20191010083416-a7dc8b61c822 // indirect + github.com/prometheus/client_model v0.6.1 // indirect + github.com/prometheus/common v0.55.0 // indirect + github.com/prometheus/procfs v0.15.1 // indirect ) require ( - github.com/lib/pq v1.10.9 - golang.org/x/net v0.26.0 // indirect - golang.org/x/sys v0.22.0 // indirect - golang.org/x/text v0.16.0 // indirect - google.golang.org/genproto/googleapis/rpc v0.0.0-20240701130421-f6361c86f094 // indirect + github.com/lib/pq v1.10.9 + golang.org/x/net v0.26.0 // indirect + golang.org/x/sys v0.22.0 // indirect + golang.org/x/text v0.16.0 // indirect + google.golang.org/genproto/googleapis/rpc v0.0.0-20240604185151-ef581f913117 // indirect ) diff --git a/datastore/datastore/main/main.go b/datastore/datastore/main/main.go index a6dfee21..8d29b23b 100644 --- a/datastore/datastore/main/main.go +++ b/datastore/datastore/main/main.go @@ -10,7 +10,6 @@ import ( "fmt" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/client_golang/prometheus/promhttp" - "go.opentelemetry.io/otel/trace" "log" "net" "time" @@ -22,18 +21,8 @@ import ( "google.golang.org/grpc/peer" // Monitoring + "datastore/metrics" grpc_prometheus "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" - //"github.com/grpc-ecosystem/go-grpc-middleware" - //"google.golang.org/grpc/ChainUnaryInterceptor" - //"github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors" - //"github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/auth" - //"github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/logging" - //"github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/selector" - //"github.com/prometheus/client_golang/prometheus" - //"github.com/prometheus/client_golang/prometheus/promhttp" - //"go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc" - //"github.com/grpc-ecosystem/go-grpc-middleware/v2/interceptors/recovery" - //"go.opentelemetry.io/otel/trace" _ "expvar" "net/http" @@ -76,22 +65,22 @@ func main() { ), ) reg := prometheus.NewRegistry() - reg.MustRegister(grpcMetrics) - exemplarFromContext := func(ctx context.Context) prometheus.Labels { - if span := trace.SpanContextFromContext(ctx); span.IsSampled() { - return prometheus.Labels{"traceID": span.TraceID().String()} - } - return nil - } + reg.MustRegister( + grpcMetrics, + promservermetrics.ActiveConnections, + promservermetrics.UptimeCounter, + promservermetrics.ResponseSizeSummary, + ) + + go promservermetrics.TrackUptime() // create gRPC server with middleware server := grpc.NewServer( grpc.ChainUnaryInterceptor( reqTimeLogger, - grpcMetrics.UnaryServerInterceptor(grpc_prometheus.WithExemplarFromContext(exemplarFromContext)), - ), - grpc.ChainStreamInterceptor( - grpcMetrics.StreamServerInterceptor(grpc_prometheus.WithExemplarFromContext(exemplarFromContext)), + promservermetrics.ConnectionUnaryInterceptor, + promservermetrics.ResponseSizeUnaryInterceptor, + grpcMetrics.UnaryServerInterceptor(), ), ) @@ -123,15 +112,14 @@ func main() { http.ListenAndServe("0.0.0.0:6060", nil) }() + // serve go metrics for monitoring go func() { httpSrv := &http.Server{Addr: "0.0.0.0:8081"} m := http.NewServeMux() - //log.Println("Starting HTTP server for Prometheus metrics on :8081") // Create HTTP handler for Prometheus metrics. m.Handle("/metrics", promhttp.HandlerFor( reg, promhttp.HandlerOpts{ - // Opt into OpenMetrics e.g. to support exemplars. EnableOpenMetrics: true, }, )) diff --git a/datastore/datastore/metrics/promservermetrics.go b/datastore/datastore/metrics/promservermetrics.go new file mode 100644 index 00000000..f4000431 --- /dev/null +++ b/datastore/datastore/metrics/promservermetrics.go @@ -0,0 +1,67 @@ +package promservermetrics + +import ( + "context" + "github.com/prometheus/client_golang/prometheus" + "google.golang.org/grpc" + "sync" + "time" +) + +var ( + UptimeCounter = prometheus.NewCounter(prometheus.CounterOpts{ + Name: "grpc_server_uptime_seconds", + Help: "Total uptime of the gRPC server in seconds", + }) + + ActiveConnections = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "grpc_active_connections", + Help: "Current number of active gRPC connections", + }) + + ResponseSizeSummary = prometheus.NewSummaryVec( + prometheus.SummaryOpts{ + Name: "grpc_response_size_summary_bytes", + Help: "Summary of response sizes in bytes for each gRPC method, with mean, min, and max", + Objectives: map[float64]float64{0.0: 0.001, 1.0: 0.001}, // Track min (0.0 quantile) and max (1.0 quantile) + }, + []string{"method"}, + ) + + responseSizeMu sync.Mutex + responseSizeSum = make(map[string]float64) + responseSizeCount = make(map[string]float64) +) + +func TrackUptime() { + // Increment the uptime every second + for { + UptimeCounter.Inc() + time.Sleep(1 * time.Second) + } +} + +func ConnectionUnaryInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + ActiveConnections.Inc() // Increment when a new unary request (connection) is opened + defer ActiveConnections.Dec() // Decrement when the unary request (connection) is completed + return handler(ctx, req) +} + +func ResponseSizeUnaryInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (resp interface{}, err error) { + resp, err = handler(ctx, req) + + if resp != nil { + responseSize := float64(len(resp.(interface{ String() string }).String())) + + ResponseSizeSummary.WithLabelValues(info.FullMethod).Observe(responseSize) + + // Used a mutex to synchronise the access for the responseSizeSum and responseSizeCount. + // To prevent race conditions and multiple goroutines accessing the variables at the same time. + responseSizeMu.Lock() + responseSizeSum[info.FullMethod] += responseSize + responseSizeCount[info.FullMethod]++ + responseSizeMu.Unlock() + } + + return resp, err +} diff --git a/grafana/dashboards/grpc-server.json b/grafana/dashboards/grpc-server.json new file mode 100644 index 00000000..d98165b0 --- /dev/null +++ b/grafana/dashboards/grpc-server.json @@ -0,0 +1,1763 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 4, + "links": [], + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 5, + "panels": [], + "title": "General", + "type": "row" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "red", + "value": null + }, + { + "color": "yellow", + "value": 60 + }, + { + "color": "green", + "value": 3600 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 0, + "y": 1 + }, + "id": 1, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "grpc_server_uptime_seconds", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Uptime", + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 6, + "y": 1 + }, + "id": 2, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "1 - (sum(rate(grpc_server_handled_total{job=\"store\",grpc_type=\"unary\",grpc_code!=\"OK\"}[5m])) OR vector(0) / sum(rate(grpc_server_started_total{job=\"store\",grpc_type=\"unary\"}[5m])))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Succes Rate", + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 12, + "y": 1 + }, + "id": 4, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"$JOB\",grpc_type=\"unary\"}[$INTERVAL])) by (le)\n)", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "type": "stat" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 6, + "x": 18, + "y": 1 + }, + "id": 3, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "percentChangeColorMode": "standard", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showPercentChange": false, + "textMode": "auto", + "wideLayout": true + }, + "pluginVersion": "11.2.0", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(rate(grpc_server_started_total{job=\"$JOB\"}[$INTERVAL]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Requests per second", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 8, + "panels": [], + "title": "gRPC", + "type": "row" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "percent" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 6 + }, + "id": 6, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(rate(grpc_server_handled_total{job=\"$JOB\",grpc_type=\"unary\",grpc_code=\"OK\"}[$INTERVAL]))\n/ sum(rate(grpc_server_handled_total{job=\"$JOB\",grpc_type=\"unary\"}[$INTERVAL]))", + "instant": false, + "legendFormat": "OK", + "range": true, + "refId": "OK" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "(sum(rate(grpc_server_handled_total{job=\"$JOB\",grpc_type=\"unary\",grpc_code=\"InvalidArgument\"}[$INTERVAL])) +\nsum(rate(grpc_server_handled_total{job=\"$JOB\",grpc_type=\"unary\",grpc_code=\"NotFound\"}[$INTERVAL])) + \nsum(rate(grpc_server_handled_total{job=\"$JOB\",grpc_type=\"unary\",grpc_code=\"AlreadyExists\"}[$INTERVAL])) + \nsum(rate(grpc_server_handled_total{job=\"$JOB\",grpc_type=\"unary\",grpc_code=\"Unauthenticated\"}[$INTERVAL])) +\nsum(rate(grpc_server_handled_total{job=\"$JOB\",grpc_type=\"unary\",grpc_code=\"PermissionDenied\"}[$INTERVAL])) + \nsum(rate(grpc_server_handled_total{job=\"$JOB\",grpc_type=\"unary\",grpc_code=\"FailedPrecondition\"}[$INTERVAL]))\n)/ sum(rate(grpc_server_handled_total{job=\"$JOB\",grpc_type=\"unary\"}[$INTERVAL]))", + "hide": false, + "instant": false, + "interval": "", + "legendFormat": "ClientError", + "range": true, + "refId": "ClientError" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "(sum(rate(grpc_server_handled_total{job=\"$JOB\",grpc_type=\"unary\",grpc_code=\"Internal\"}[$INTERVAL])) +\nsum(rate(grpc_server_handled_total{job=\"$JOB\",grpc_type=\"unary\",grpc_code=\"Unknown\"}[$INTERVAL])) + \nsum(rate(grpc_server_handled_total{job=\"$JOB\",grpc_type=\"unary\",grpc_code=\"Unavailable\"}[$INTERVAL])) +\nsum(rate(grpc_server_handled_total{job=\"$JOB\",grpc_type=\"unary\",grpc_code=\"Unimplemented\"}[$INTERVAL]))\n)/ sum(rate(grpc_server_handled_total{job=\"$JOB\",grpc_type=\"unary\"}[$INTERVAL]))", + "hide": false, + "instant": false, + "legendFormat": "ServerError", + "range": true, + "refId": "ServerError" + } + ], + "title": "Status", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "percent" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 6 + }, + "id": 7, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(rate(grpc_server_handled_total{job=\"$JOB\",grpc_type=\"unary\"}[$INTERVAL])) by (grpc_code)\n/ ignoring(grpc_code) group_left sum(rate(grpc_server_handled_total{job=\"$JOB\",grpc_type=\"unary\"}[$INTERVAL]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Status distribution", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "reqps" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 14 + }, + "id": 9, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(rate(grpc_server_started_total{job=\"$JOB\"}[$INTERVAL])) by (grpc_service)", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Requests per second", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "bars", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "percent" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 14 + }, + "id": 10, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "sum(rate(grpc_server_started_total{job=\"$JOB\"}[$INTERVAL])) by (grpc_service) \n/ ignoring(grpc_service) group_left sum(rate(grpc_server_started_total{job=\"$JOB\"}[$INTERVAL]))", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Request distribution", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 22 + }, + "id": 11, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"$JOB\",grpc_type=\"unary\"}[$INTERVAL])) by (grpc_service,le)\n)", + "instant": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Latency", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "s" + }, + "overrides": [ + { + "__systemRef": "hideSeriesFrom", + "matcher": { + "id": "byNames", + "options": { + "mode": "exclude", + "names": [ + "99%" + ], + "prefix": "All except:", + "readOnly": true + } + }, + "properties": [ + { + "id": "custom.hideFrom", + "value": { + "legend": false, + "tooltip": false, + "viz": true + } + } + ] + } + ] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 22 + }, + "id": 12, + "options": { + "legend": { + "calcs": [ + "mean", + "lastNotNull", + "max", + "min" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.99, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"$JOB\",grpc_type=\"unary\"}[$INTERVAL])) by (le)\n)", + "instant": false, + "interval": "", + "legendFormat": "99%", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.95, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"$JOB\",grpc_type=\"unary\"}[$INTERVAL])) by (le)\n)", + "hide": false, + "instant": false, + "legendFormat": "95%", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.90, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"$JOB\",grpc_type=\"unary\"}[$INTERVAL])) by (le)\n)", + "hide": false, + "instant": false, + "legendFormat": "90%", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.75, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"$JOB\",grpc_type=\"unary\"}[$INTERVAL])) by (le)\n)", + "hide": false, + "instant": false, + "legendFormat": "75%", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.50, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"$JOB\",grpc_type=\"unary\"}[$INTERVAL])) by (le)\n)", + "hide": false, + "instant": false, + "legendFormat": "50%", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.25, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"$JOB\",grpc_type=\"unary\"}[$INTERVAL])) by (le)\n)", + "hide": false, + "instant": false, + "legendFormat": "25%", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "histogram_quantile(0.10, \n sum(rate(grpc_server_handling_seconds_bucket{job=\"$JOB\",grpc_type=\"unary\"}[$INTERVAL])) by (le)\n)", + "hide": false, + "instant": false, + "legendFormat": "10%", + "range": true, + "refId": "G" + } + ], + "title": "Latency distribution", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 30 + }, + "id": 13, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "grpc_active_connections", + "instant": false, + "legendFormat": "Active gRPC connections", + "range": true, + "refId": "A" + } + ], + "title": "Active gRPC connections", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 38 + }, + "id": 14, + "panels": [], + "title": "gRPC Methods", + "type": "row" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 39 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(grpc_response_size_summary_bytes_count{job=\"$JOB\"}[$INTERVAL])", + "instant": false, + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Number of calls over time", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 39 + }, + "id": 16, + "options": { + "displayLabels": [], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "percent", + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "grpc_response_size_summary_bytes_count{job=\"$JOB\"}", + "instant": false, + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Call distribution", + "type": "piechart" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 47 + }, + "id": 15, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(grpc_response_size_summary_bytes_sum{job=\"$JOB\"}[$INTERVAL]) / rate(grpc_response_size_summary_bytes_count{job=\"$JOB\"}[$INTERVAL])", + "instant": false, + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Mean response size", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 47 + }, + "id": 17, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "grpc_response_size_summary_bytes{quantile=\"1.0\", job=\"$JOB\"}", + "instant": false, + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Maximum response size", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 55 + }, + "id": 18, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "grpc_response_size_summary_bytes{quantile=\"0.0\", job=\"$JOB\"}", + "instant": false, + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Minimum response size", + "type": "timeseries" + } + ], + "refresh": "1m", + "schemaVersion": 39, + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "store", + "value": "store" + }, + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "definition": "label_values(job)", + "hide": 0, + "includeAll": false, + "label": "Job", + "multi": false, + "name": "JOB", + "options": [], + "query": { + "qryType": 1, + "query": "label_values(job)", + "refId": "PrometheusVariableQueryEditor-VariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 1, + "type": "query" + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": false, + "text": "5m", + "value": "5m" + }, + "hide": 0, + "label": "Interval", + "name": "INTERVAL", + "options": [ + { + "selected": false, + "text": "1m", + "value": "1m" + }, + { + "selected": true, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "1m,5m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "queryValue": "", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" + } + ] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": {}, + "timezone": "browser", + "title": "gRPC Server", + "uid": "cdydeqdbz124gf", + "version": 20, + "weekStart": "" +} From a6fae5008c4d39533692aee70ee3ad2893d5144f Mon Sep 17 00:00:00 2001 From: Jeffrey Vervoort Date: Fri, 18 Oct 2024 15:21:19 +0200 Subject: [PATCH 10/19] Improve colours for grpc server dashboard. --- grafana/dashboards/grpc-server.json | 875 ++++++++++++++-------------- 1 file changed, 444 insertions(+), 431 deletions(-) diff --git a/grafana/dashboards/grpc-server.json b/grafana/dashboards/grpc-server.json index d98165b0..1eef030e 100644 --- a/grafana/dashboards/grpc-server.json +++ b/grafana/dashboards/grpc-server.json @@ -18,7 +18,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 4, + "id": 3, "links": [], "panels": [ { @@ -125,12 +125,16 @@ "mode": "absolute", "steps": [ { - "color": "green", + "color": "red", "value": null }, { - "color": "red", - "value": 80 + "color": "#EAB839", + "value": 0.8 + }, + { + "color": "green", + "value": 0.95 } ] }, @@ -199,9 +203,13 @@ "color": "green", "value": null }, + { + "color": "yellow", + "value": 0.1 + }, { "color": "red", - "value": 80 + "value": 1 } ] }, @@ -267,12 +275,16 @@ "mode": "absolute", "steps": [ { - "color": "green", + "color": "red", "value": null }, { - "color": "red", - "value": 80 + "color": "yellow", + "value": 100 + }, + { + "color": "green", + "value": 500 } ] } @@ -1157,7 +1169,7 @@ "type": "timeseries" }, { - "collapsed": false, + "collapsed": true, "gridPos": { "h": 1, "w": 24, @@ -1165,479 +1177,480 @@ "y": 38 }, "id": 14, - "panels": [], - "title": "gRPC Methods", - "type": "row" - }, - { - "datasource": { - "default": true, - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 39 - }, - "id": 19, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ + "panels": [ { "datasource": { + "default": true, "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "editorMode": "code", - "expr": "rate(grpc_response_size_summary_bytes_count{job=\"$JOB\"}[$INTERVAL])", - "instant": false, - "legendFormat": "{{method}}", - "range": true, - "refId": "A" - } - ], - "title": "Number of calls over time", - "type": "timeseries" - }, - { - "datasource": { - "default": true, - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 7 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" } }, - "mappings": [], - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 39 - }, - "id": 16, - "options": { - "displayLabels": [], - "legend": { - "displayMode": "table", - "placement": "right", - "showLegend": true, - "values": [ - "percent", - "value" - ] - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "lastNotNull" + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(grpc_response_size_summary_bytes_count{job=\"$JOB\"}[$INTERVAL])", + "instant": false, + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } ], - "fields": "", - "values": false + "title": "Number of calls over time", + "type": "timeseries" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ { "datasource": { + "default": true, "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "editorMode": "code", - "expr": "grpc_response_size_summary_bytes_count{job=\"$JOB\"}", - "instant": false, - "legendFormat": "{{method}}", - "range": true, - "refId": "A" - } - ], - "title": "Call distribution", - "type": "piechart" - }, - { - "datasource": { - "default": true, - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "none" }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 7 + }, + "id": 16, + "options": { + "displayLabels": [], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "percent", + "value" + ] }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "single", + "sort": "none" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 47 - }, - "id": 15, - "options": { - "legend": { - "calcs": [ - "lastNotNull" + "editorMode": "code", + "expr": "grpc_response_size_summary_bytes_count{job=\"$JOB\"}", + "instant": false, + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } ], - "displayMode": "table", - "placement": "right", - "showLegend": true + "title": "Call distribution", + "type": "piechart" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ { "datasource": { + "default": true, "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "editorMode": "code", - "expr": "rate(grpc_response_size_summary_bytes_sum{job=\"$JOB\"}[$INTERVAL]) / rate(grpc_response_size_summary_bytes_count{job=\"$JOB\"}[$INTERVAL])", - "instant": false, - "legendFormat": "{{method}}", - "range": true, - "refId": "A" - } - ], - "title": "Mean response size", - "type": "timeseries" - }, - { - "datasource": { - "default": true, - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 15 + }, + "id": 15, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "single", + "sort": "none" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 47 - }, - "id": 17, - "options": { - "legend": { - "calcs": [ - "lastNotNull" + "editorMode": "code", + "expr": "rate(grpc_response_size_summary_bytes_sum{job=\"$JOB\"}[$INTERVAL]) / rate(grpc_response_size_summary_bytes_count{job=\"$JOB\"}[$INTERVAL])", + "instant": false, + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } ], - "displayMode": "table", - "placement": "right", - "showLegend": true + "title": "Mean response size", + "type": "timeseries" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ { "datasource": { + "default": true, "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "editorMode": "code", - "expr": "grpc_response_size_summary_bytes{quantile=\"1.0\", job=\"$JOB\"}", - "instant": false, - "legendFormat": "{{method}}", - "range": true, - "refId": "A" - } - ], - "title": "Maximum response size", - "type": "timeseries" - }, - { - "datasource": { - "default": true, - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 15 + }, + "id": 17, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true }, - "thresholdsStyle": { - "mode": "off" + "tooltip": { + "mode": "single", + "sort": "none" } }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 55 - }, - "id": 18, - "options": { - "legend": { - "calcs": [ - "lastNotNull" + "editorMode": "code", + "expr": "grpc_response_size_summary_bytes{quantile=\"1.0\", job=\"$JOB\"}", + "instant": false, + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } ], - "displayMode": "table", - "placement": "right", - "showLegend": true + "title": "Maximum response size", + "type": "timeseries" }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "targets": [ { "datasource": { + "default": true, "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "editorMode": "code", - "expr": "grpc_response_size_summary_bytes{quantile=\"0.0\", job=\"$JOB\"}", - "instant": false, - "legendFormat": "{{method}}", - "range": true, - "refId": "A" + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 23 + }, + "id": 18, + "options": { + "legend": { + "calcs": [ + "lastNotNull" + ], + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "grpc_response_size_summary_bytes{quantile=\"0.0\", job=\"$JOB\"}", + "instant": false, + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Minimum response size", + "type": "timeseries" } ], - "title": "Minimum response size", - "type": "timeseries" + "title": "gRPC Methods", + "type": "row" } ], "refresh": "1m", @@ -1647,7 +1660,7 @@ "list": [ { "current": { - "selected": false, + "selected": true, "text": "store", "value": "store" }, @@ -1758,6 +1771,6 @@ "timezone": "browser", "title": "gRPC Server", "uid": "cdydeqdbz124gf", - "version": 20, + "version": 2, "weekStart": "" } From 81333f77a3017376c8b1e25d110a8c1412ae5521 Mon Sep 17 00:00:00 2001 From: Jeffrey Vervoort Date: Fri, 18 Oct 2024 16:24:24 +0200 Subject: [PATCH 11/19] Add GoLang metrics for the Store. --- datastore/datastore/main/main.go | 11 +- grafana/dashboards/database-performance.json | 0 grafana/dashboards/grpc-server.json | 2043 +++++++++++++---- .../provisioning/dashboards/dashboards.yaml | 0 .../provisioning/datasources/datasource.yaml | 0 5 files changed, 1596 insertions(+), 458 deletions(-) mode change 100644 => 100755 grafana/dashboards/database-performance.json mode change 100644 => 100755 grafana/dashboards/grpc-server.json mode change 100644 => 100755 grafana/provisioning/dashboards/dashboards.yaml mode change 100644 => 100755 grafana/provisioning/datasources/datasource.yaml diff --git a/datastore/datastore/main/main.go b/datastore/datastore/main/main.go index 8d29b23b..ae19695f 100644 --- a/datastore/datastore/main/main.go +++ b/datastore/datastore/main/main.go @@ -9,6 +9,7 @@ import ( "datastore/storagebackend/postgresql" "fmt" "github.com/prometheus/client_golang/prometheus" + "github.com/prometheus/client_golang/prometheus/collectors" "github.com/prometheus/client_golang/prometheus/promhttp" "log" "net" @@ -22,7 +23,7 @@ import ( // Monitoring "datastore/metrics" - grpc_prometheus "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" + grpcprometheus "github.com/grpc-ecosystem/go-grpc-middleware/providers/prometheus" _ "expvar" "net/http" @@ -59,9 +60,9 @@ func main() { return resp, err } - grpcMetrics := grpc_prometheus.NewServerMetrics( - grpc_prometheus.WithServerHandlingTimeHistogram( - grpc_prometheus.WithHistogramBuckets([]float64{0.001, 0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120}), + grpcMetrics := grpcprometheus.NewServerMetrics( + grpcprometheus.WithServerHandlingTimeHistogram( + grpcprometheus.WithHistogramBuckets([]float64{0.001, 0.01, 0.1, 0.3, 0.6, 1, 3, 6, 9, 20, 30, 60, 90, 120}), ), ) reg := prometheus.NewRegistry() @@ -70,6 +71,8 @@ func main() { promservermetrics.ActiveConnections, promservermetrics.UptimeCounter, promservermetrics.ResponseSizeSummary, + collectors.NewGoCollector(), + collectors.NewProcessCollector(collectors.ProcessCollectorOpts{}), ) go promservermetrics.TrackUptime() diff --git a/grafana/dashboards/database-performance.json b/grafana/dashboards/database-performance.json old mode 100644 new mode 100755 diff --git a/grafana/dashboards/grpc-server.json b/grafana/dashboards/grpc-server.json old mode 100644 new mode 100755 index 1eef030e..7a0feea9 --- a/grafana/dashboards/grpc-server.json +++ b/grafana/dashboards/grpc-server.json @@ -31,7 +31,7 @@ }, "id": 5, "panels": [], - "title": "General", + "title": "gRPC General", "type": "row" }, { @@ -275,7 +275,7 @@ "mode": "absolute", "steps": [ { - "color": "red", + "color": "dark-red", "value": null }, { @@ -333,19 +333,6 @@ "title": "Requests per second", "type": "stat" }, - { - "collapsed": false, - "gridPos": { - "h": 1, - "w": 24, - "x": 0, - "y": 5 - }, - "id": 8, - "panels": [], - "title": "gRPC", - "type": "row" - }, { "datasource": { "default": true, @@ -412,7 +399,7 @@ "h": 8, "w": 12, "x": 0, - "y": 6 + "y": 5 }, "id": 6, "options": { @@ -537,7 +524,7 @@ "h": 8, "w": 12, "x": 12, - "y": 6 + "y": 5 }, "id": 7, "options": { @@ -635,7 +622,7 @@ "h": 8, "w": 12, "x": 0, - "y": 14 + "y": 13 }, "id": 9, "options": { @@ -733,7 +720,7 @@ "h": 8, "w": 12, "x": 12, - "y": 14 + "y": 13 }, "id": 10, "options": { @@ -762,7 +749,7 @@ "refId": "A" } ], - "title": "Request distribution", + "title": "Request distribution per service", "type": "timeseries" }, { @@ -831,7 +818,7 @@ "h": 8, "w": 12, "x": 0, - "y": 22 + "y": 21 }, "id": 11, "options": { @@ -954,7 +941,7 @@ "h": 8, "w": 12, "x": 12, - "y": 22 + "y": 21 }, "id": 12, "options": { @@ -1136,7 +1123,7 @@ "h": 8, "w": 12, "x": 0, - "y": 30 + "y": 29 }, "id": 13, "options": { @@ -1169,488 +1156,1636 @@ "type": "timeseries" }, { - "collapsed": true, + "collapsed": false, "gridPos": { "h": 1, "w": 24, "x": 0, - "y": 38 + "y": 37 }, "id": 14, - "panels": [ - { - "datasource": { - "default": true, - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "panels": [], + "title": "gRPC Methods", + "type": "row" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 7 - }, - "id": 19, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "editorMode": "code", - "expr": "rate(grpc_response_size_summary_bytes_count{job=\"$JOB\"}[$INTERVAL])", - "instant": false, - "legendFormat": "{{method}}", - "range": true, - "refId": "A" - } - ], - "title": "Number of calls over time", - "type": "timeseries" + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 38 + }, + "id": 19, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ { "datasource": { - "default": true, "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "description": "", - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - } - }, - "mappings": [], - "unit": "none" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 7 - }, - "id": 16, - "options": { - "displayLabels": [], - "legend": { - "displayMode": "table", - "placement": "right", - "showLegend": true, - "values": [ - "percent", - "value" - ] - }, - "pieType": "pie", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "editorMode": "code", + "expr": "rate(grpc_response_size_summary_bytes_count{job=\"$JOB\"}[$INTERVAL])", + "instant": false, + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Number of calls over time", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "editorMode": "code", - "expr": "grpc_response_size_summary_bytes_count{job=\"$JOB\"}", - "instant": false, - "legendFormat": "{{method}}", - "range": true, - "refId": "A" + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false } + }, + "mappings": [], + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 38 + }, + "id": 16, + "options": { + "displayLabels": [], + "legend": { + "displayMode": "table", + "placement": "right", + "showLegend": true, + "values": [ + "percent", + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" ], - "title": "Call distribution", - "type": "piechart" + "fields": "", + "values": false }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ { "datasource": { - "default": true, "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 15 - }, - "id": 15, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" + "editorMode": "code", + "expr": "grpc_response_size_summary_bytes_count{job=\"$JOB\"}", + "instant": false, + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Call distribution", + "type": "piechart" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "editorMode": "code", - "expr": "rate(grpc_response_size_summary_bytes_sum{job=\"$JOB\"}[$INTERVAL]) / rate(grpc_response_size_summary_bytes_count{job=\"$JOB\"}[$INTERVAL])", - "instant": false, - "legendFormat": "{{method}}", - "range": true, - "refId": "A" - } + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 46 + }, + "id": 15, + "options": { + "legend": { + "calcs": [ + "lastNotNull" ], - "title": "Mean response size", - "type": "timeseries" + "displayMode": "table", + "placement": "right", + "showLegend": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ { "datasource": { - "default": true, "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 12, - "y": 15 - }, - "id": 17, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" + "editorMode": "code", + "expr": "rate(grpc_response_size_summary_bytes_sum{job=\"$JOB\"}[$INTERVAL]) / rate(grpc_response_size_summary_bytes_count{job=\"$JOB\"}[$INTERVAL])", + "instant": false, + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Mean response size", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "editorMode": "code", - "expr": "grpc_response_size_summary_bytes{quantile=\"1.0\", job=\"$JOB\"}", - "instant": false, - "legendFormat": "{{method}}", - "range": true, - "refId": "A" - } + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 46 + }, + "id": 17, + "options": { + "legend": { + "calcs": [ + "lastNotNull" ], - "title": "Maximum response size", - "type": "timeseries" + "displayMode": "table", + "placement": "right", + "showLegend": true }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ { "datasource": { - "default": true, "type": "prometheus", "uid": "PBFA97CFB590B2093" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "barWidthFactor": 0.6, - "drawStyle": "line", - "fillOpacity": 0, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "auto", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "bytes" - }, - "overrides": [] - }, - "gridPos": { - "h": 8, - "w": 12, - "x": 0, - "y": 23 - }, - "id": 18, - "options": { - "legend": { - "calcs": [ - "lastNotNull" - ], - "displayMode": "table", - "placement": "right", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" + "editorMode": "code", + "expr": "grpc_response_size_summary_bytes{quantile=\"1.0\", job=\"$JOB\"}", + "instant": false, + "legendFormat": "{{method}}", + "range": true, + "refId": "A" + } + ], + "title": "Maximum response size", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" } }, - "targets": [ - { - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null }, - "editorMode": "code", - "expr": "grpc_response_size_summary_bytes{quantile=\"0.0\", job=\"$JOB\"}", - "instant": false, - "legendFormat": "{{method}}", - "range": true, - "refId": "A" - } + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "bytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 54 + }, + "id": 18, + "options": { + "legend": { + "calcs": [ + "lastNotNull" ], - "title": "Minimum response size", - "type": "timeseries" + "displayMode": "table", + "placement": "right", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "grpc_response_size_summary_bytes{quantile=\"0.0\", job=\"$JOB\"}", + "instant": false, + "legendFormat": "{{method}}", + "range": true, + "refId": "A" } ], - "title": "gRPC Methods", + "title": "Minimum response size", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 62 + }, + "id": 21, + "panels": [], + "title": "Go", "type": "row" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 63 + }, + "id": 20, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "go_memstats_mspan_inuse_bytes{job=\"$JOB\"}", + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "go_memstats_mspan_sys_bytes{job=\"$JOB\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "go_memstats_mcache_inuse_bytes{job=\"$JOB\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "go_memstats_mcache_sys_bytes{job=\"$JOB\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "go_memstats_buck_hash_sys_bytes{job=\"$JOB\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "E" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "go_memstats_gc_sys_bytes{job=\"$JOB\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "F" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "go_memstats_other_sys_bytes{job=\"$JOB\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "G" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "go_memstats_next_gc_bytes{job=\"$JOB\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "H" + } + ], + "title": "Memory in Off-Heap", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 63 + }, + "id": 22, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "go_memstats_heap_alloc_bytes{job=\"$JOB\"}", + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "go_memstats_heap_sys_bytes{job=\"$JOB\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "go_memstats_heap_idle_bytes{job=\"$JOB\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "go_memstats_heap_inuse_bytes{job=\"$JOB\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "D" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "go_memstats_heap_released_bytes{job=\"$JOB\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "E" + } + ], + "title": "Memory in Heap", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 71 + }, + "id": 24, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "go_memstats_stack_inuse_bytes{job=\"$JOB\"}", + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "go_memstats_stack_sys_bytes{job=\"$JOB\"}", + "hide": false, + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "B" + } + ], + "title": "Memory in Stack", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 71 + }, + "id": 23, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "go_memstats_sys_bytes{job=\"$JOB\"}", + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A" + } + ], + "title": "Total Used Memory", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 79 + }, + "id": 25, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "go_memstats_mallocs_total{job=\"$JOB\"} - go_memstats_frees_total{job=\"$JOB\"}", + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A" + } + ], + "title": "Number of Live Objects", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 79 + }, + "id": 26, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(go_memstats_mallocs_total{job=\"$JOB\"}[$INTERVAL])", + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A" + } + ], + "title": "Rate of Objects Allocated", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ops" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 87 + }, + "id": 29, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(go_memstats_lookups_total{job=\"$JOB\"}[$INTERVAL])", + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A" + } + ], + "title": "Rate of a Pointer Dereferences", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "short" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 87 + }, + "id": 27, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "go_goroutines{job=\"$JOB\"}", + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A" + } + ], + "title": "Goroutines", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 95 + }, + "id": 28, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "rate(go_memstats_alloc_bytes_total{job=\"$JOB\"}[$INTERVAL])", + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A" + } + ], + "title": "Rates of Allocation", + "type": "timeseries" + }, + { + "datasource": { + "default": true, + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "barWidthFactor": 0.6, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "ms" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 95 + }, + "id": 30, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "code", + "expr": "go_gc_duration_seconds{job=\"$JOB\"}", + "instant": false, + "legendFormat": "{{__name__}}", + "range": true, + "refId": "A" + } + ], + "title": "GC duration quantile", + "type": "timeseries" } ], "refresh": "1m", @@ -1771,6 +2906,6 @@ "timezone": "browser", "title": "gRPC Server", "uid": "cdydeqdbz124gf", - "version": 2, + "version": 10, "weekStart": "" } diff --git a/grafana/provisioning/dashboards/dashboards.yaml b/grafana/provisioning/dashboards/dashboards.yaml old mode 100644 new mode 100755 diff --git a/grafana/provisioning/datasources/datasource.yaml b/grafana/provisioning/datasources/datasource.yaml old mode 100644 new mode 100755 From a19f5ae06fa1d2d9f47f00463729a6bc273661c7 Mon Sep 17 00:00:00 2001 From: Jeffrey Vervoort Date: Fri, 18 Oct 2024 17:38:52 +0200 Subject: [PATCH 12/19] Grafana and Prometheus do not need to run in CI CD. Created a seperate profile for them. --- docker-compose.yml | 3 +++ justfile | 13 ++++++++++--- 2 files changed, 13 insertions(+), 3 deletions(-) diff --git a/docker-compose.yml b/docker-compose.yml index 81cdeb80..8e219bd8 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -165,6 +165,7 @@ services: condition: service_healthy prometheus: + profiles: ["monitoring"] image: prom/prometheus ports: - "9090:9090" @@ -173,6 +174,7 @@ services: - prometheus-data:/prometheus prometheus-postgres-exporter: + profiles: ["monitoring"] image: quay.io/prometheuscommunity/postgres-exporter environment: - DATA_SOURCE_URI=db:5432/data @@ -188,6 +190,7 @@ services: command: ["--collector.stat_statements", "--collector.stat_user_tables", "--collector.stat_activity_autovacuum"] grafana: + profiles: ["monitoring"] image: grafana/grafana-oss:11.2.0 ports: - "3000:3000" diff --git a/justfile b/justfile index 4f8b880f..0abf5e49 100644 --- a/justfile +++ b/justfile @@ -8,6 +8,8 @@ set positional-arguments all: lint build unit services load integration performance client # Build and run the default docker services up: build services +# Build and run the default docker services and start up monitoring +local: up monitoring # Build and run the unit, load and integration tests test: build unit load integration @@ -93,6 +95,11 @@ unit: docker compose run --rm api-unit +# Start the monitoring +monitoring: + docker compose up prometheus prometheus-postgres-exporter grafana -d + + # Run the performance tests performance: docker compose --env-file ./ci/config/env.list run --rm performance @@ -108,7 +115,7 @@ client: # ---------------------------------------------------------------------------- # # Build the docker images build: copy-proto - docker compose --env-file ./ci/config/env.list --profile test build + docker compose --env-file ./ci/config/env.list --profile monitoring --profile test build # # ---------------------------------------------------------------------------- # @@ -126,9 +133,9 @@ load: # Stop all E-SOH containers down: - docker compose --profile test down + docker compose --profile monitoring --profile test down # Stop all E-SOH containers and remove their volumes destroy: - docker compose --profile test down --volumes + docker compose --profile monitoring --profile test down --volumes From 6dec4285f1deecf1a7277cba88ec89d566041f62 Mon Sep 17 00:00:00 2001 From: Jeffrey Vervoort Date: Wed, 23 Oct 2024 12:45:35 +0200 Subject: [PATCH 13/19] Remove logging as it is not necessary and makes the compose more complex. Update check python version in case python or python3 is not installed. --- datastore/database/extra.conf | 13 ------------- datastore/database/logs/.gitkeep | 0 docker-compose.yml | 1 - justfile | 15 +++++++++++---- 4 files changed, 11 insertions(+), 18 deletions(-) delete mode 100755 datastore/database/logs/.gitkeep diff --git a/datastore/database/extra.conf b/datastore/database/extra.conf index 3df96940..9afe745a 100644 --- a/datastore/database/extra.conf +++ b/datastore/database/extra.conf @@ -1,16 +1,3 @@ -# Log config -listen_addresses = '*' -logging_collector = 'on' -log_destination = 'stderr' -log_statement = 'all' -log_directory = '/logs' -log_duration = 'on' -# TODO: Consider changing log rotation to reduce file size -log_filename = 'postgresql-%Y-%m-%d.log' -log_rotation_age = 1d -log_rotation_size = 0 - - # postgres-exporter shared_preload_libraries = 'pg_cron,pg_stat_statements' track_activity_query_size = 2048 diff --git a/datastore/database/logs/.gitkeep b/datastore/database/logs/.gitkeep deleted file mode 100755 index e69de29b..00000000 diff --git a/docker-compose.yml b/docker-compose.yml index 8e219bd8..b6ac982d 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -10,7 +10,6 @@ services: # - ts-data:/home/postgres/pgdata/data # for timescale image - ts-data:/var/lib/postgresql # for postgres image - ./datastore/database/extra.conf:/etc/conf_settings/extra.conf:ro # Extra Postgres configuration - - ./datastore/database/logs:/logs:rw - ./datastore/database/healthcheck_postgis_uptime.sh:/healthcheck_postgis_uptime.sh:ro # for the healthcheck environment: - EXTRA_CONF_DIR=/etc/conf_settings diff --git a/justfile b/justfile index 0abf5e49..18fe6a40 100644 --- a/justfile +++ b/justfile @@ -5,7 +5,7 @@ default: set positional-arguments # Run all docker services. After running the database needs cleanup, run just destroy -all: lint build unit services load integration performance client +all: lint build unit services monitoring load integration performance client # Build and run the default docker services up: build services # Build and run the default docker services and start up monitoring @@ -43,11 +43,18 @@ _check-python-version: set -euxo pipefail # Get Python version - python_version=$(python --version 2>&1 | cut -d' ' -f2) + if command -v python &>/dev/null; then + python_version=$(python --version 2>&1 | cut -d ' ' -f2) + elif command -v python3 &>/dev/null; then + python_version=$(python3 --version 2>&1 | cut -d ' ' -f2) + else + echo "Python not found. Failing..." + exit 1 + fi # Extract major and minor version numbers - major_version=$(echo "$python_version" | cut -d'.' -f1) - minor_version=$(echo "$python_version" | cut -d'.' -f2) + major_version=$(echo "$python_version" | cut -d '.' -f1) + minor_version=$(echo "$python_version" | cut -d '.' -f2) # Check if Python version is greater than or equal to 3.11 if [[ "$major_version" -lt 3 || ( "$major_version" -eq 3 && "$minor_version" -lt 11 ) ]]; then From f5d8ff60f71f0a963acdc0e3cfa5d6c27004e04a Mon Sep 17 00:00:00 2001 From: Jeffrey Vervoort Date: Wed, 23 Oct 2024 15:15:06 +0200 Subject: [PATCH 14/19] Update readme with just local command. --- README.md | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/README.md b/README.md index 77c9bb16..449e4b6b 100644 --- a/README.md +++ b/README.md @@ -44,12 +44,18 @@ ci/scripts/install-just.sh The Justfile is a simple text file that contains a list of tasks. Each task is a shell command. For example: -To run build and run the services locally: +To run build and run the services locally without the monitoring: ```bash just up test ``` +To run build and run the services locally with the monitoring: + +```bash +just local test +``` + To run everything including client and do a cleanup of the database afterward: ```bash From 5b2af46b23d31d809c7663ea4f63fb7032ade695 Mon Sep 17 00:00:00 2001 From: Jeffrey Vervoort Date: Fri, 25 Oct 2024 13:42:05 +0200 Subject: [PATCH 15/19] Removed units from all titles. Made every rate based on interval drop down. --- grafana/dashboards/database-performance.json | 99 +++++++++++++++++--- grafana/dashboards/grpc-server.json | 25 ++--- 2 files changed, 100 insertions(+), 24 deletions(-) diff --git a/grafana/dashboards/database-performance.json b/grafana/dashboards/database-performance.json index d1b8f416..24fb3ebc 100755 --- a/grafana/dashboards/database-performance.json +++ b/grafana/dashboards/database-performance.json @@ -18,7 +18,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 3, + "id": 2, "links": [], "panels": [ { @@ -478,7 +478,7 @@ "disableTextWrap": false, "editorMode": "code", "exemplar": false, - "expr": "sum(rate(pg_stat_statements_seconds_total{instance=~\"$INSTANCE\", datname=~\"$DATNAME\"}[1m])) / sum(rate(pg_stat_statements_calls_total{instance=~\"$INSTANCE\", datname=~\"$DATNAME\"}[1m]))", + "expr": "sum(rate(pg_stat_statements_seconds_total{instance=~\"$INSTANCE\", datname=~\"$DATNAME\"}[$INTERVAL])) / sum(rate(pg_stat_statements_calls_total{instance=~\"$INSTANCE\", datname=~\"$DATNAME\"}[$INTERVAL]))", "format": "time_series", "fullMetaSearch": false, "includeNullMetadata": true, @@ -587,14 +587,14 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum(rate(pg_stat_statements_calls_total{instance=~\"$INSTANCE\"}[1m]))", + "expr": "sum(rate(pg_stat_statements_calls_total{instance=~\"$INSTANCE\"}[$INTERVAL]))", "instant": false, - "legendFormat": "Queries per second", + "legendFormat": "Number of queries", "range": true, "refId": "A" } ], - "title": "Number of queries per second", + "title": "Number of queries", "type": "timeseries" }, { @@ -841,7 +841,7 @@ "table": "pg_stat_statements" } ], - "title": "Top 10 slowest queries in micro seconds", + "title": "Top 10 slowest queries", "type": "table" }, { @@ -937,7 +937,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "rate(pg_stat_database_tup_fetched[5m]) > 1000", + "expr": "rate(pg_stat_database_tup_fetched[$INTERVAL]) > 1000", "instant": false, "legendFormat": "{{datname}} SELECT (index scan)", "range": true, @@ -949,7 +949,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "rate(pg_stat_database_tup_returned[5m]) > 1000", + "expr": "rate(pg_stat_database_tup_returned[$INTERVAL]) > 1000", "hide": false, "instant": false, "legendFormat": "{{datname}} SELECT (table scan)", @@ -1052,7 +1052,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "rate(pg_stat_database_tup_inserted[5m]) > 1", + "expr": "rate(pg_stat_database_tup_inserted[$INTERVAL]) > 1", "instant": false, "legendFormat": "{{datname}} INSERT", "range": true, @@ -1194,7 +1194,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "rate(pg_stat_database_deadlocks[1m])", + "expr": "rate(pg_stat_database_deadlocks[$INTERVAL])", "instant": false, "interval": "1m", "legendFormat": "{{datname}}", @@ -1292,7 +1292,7 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "sum(rate(pg_stat_database_conflicts_confl_deadlock[1m]))", + "expr": "sum(rate(pg_stat_database_conflicts_confl_deadlock[$INTERVAL]))", "instant": false, "legendFormat": "Deadlock", "range": true, @@ -1672,6 +1672,81 @@ "skipUrlSync": false, "sort": 1, "type": "query" + }, + { + "auto": false, + "auto_count": 30, + "auto_min": "10s", + "current": { + "selected": false, + "text": "1m", + "value": "1m" + }, + "hide": 0, + "label": "Interval", + "name": "INTERVAL", + "options": [ + { + "selected": true, + "text": "1m", + "value": "1m" + }, + { + "selected": false, + "text": "5m", + "value": "5m" + }, + { + "selected": false, + "text": "10m", + "value": "10m" + }, + { + "selected": false, + "text": "30m", + "value": "30m" + }, + { + "selected": false, + "text": "1h", + "value": "1h" + }, + { + "selected": false, + "text": "6h", + "value": "6h" + }, + { + "selected": false, + "text": "12h", + "value": "12h" + }, + { + "selected": false, + "text": "1d", + "value": "1d" + }, + { + "selected": false, + "text": "7d", + "value": "7d" + }, + { + "selected": false, + "text": "14d", + "value": "14d" + }, + { + "selected": false, + "text": "30d", + "value": "30d" + } + ], + "query": "1m,5m,10m,30m,1h,6h,12h,1d,7d,14d,30d", + "queryValue": "", + "refresh": 2, + "skipUrlSync": false, + "type": "interval" } ] }, @@ -1683,6 +1758,6 @@ "timezone": "browser", "title": "Database Performance", "uid": "adxl34wpc3if4a", - "version": 2, + "version": 5, "weekStart": "" } diff --git a/grafana/dashboards/grpc-server.json b/grafana/dashboards/grpc-server.json index 7a0feea9..78c0fde7 100755 --- a/grafana/dashboards/grpc-server.json +++ b/grafana/dashboards/grpc-server.json @@ -18,7 +18,7 @@ "editable": true, "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 3, + "id": 1, "links": [], "panels": [ { @@ -265,6 +265,7 @@ "type": "prometheus", "uid": "PBFA97CFB590B2093" }, + "description": "", "fieldConfig": { "defaults": { "color": { @@ -330,7 +331,7 @@ "refId": "A" } ], - "title": "Requests per second", + "title": "Requests per interval", "type": "stat" }, { @@ -1837,7 +1838,7 @@ "refId": "H" } ], - "title": "Memory in Off-Heap", + "title": "Memory in off-heap", "type": "timeseries" }, { @@ -1987,7 +1988,7 @@ "refId": "E" } ], - "title": "Memory in Heap", + "title": "Memory in heap", "type": "timeseries" }, { @@ -2098,7 +2099,7 @@ "refId": "B" } ], - "title": "Memory in Stack", + "title": "Memory in stack", "type": "timeseries" }, { @@ -2196,7 +2197,7 @@ "refId": "A" } ], - "title": "Total Used Memory", + "title": "Total used memory", "type": "timeseries" }, { @@ -2294,7 +2295,7 @@ "refId": "A" } ], - "title": "Number of Live Objects", + "title": "Number of live objects", "type": "timeseries" }, { @@ -2392,7 +2393,7 @@ "refId": "A" } ], - "title": "Rate of Objects Allocated", + "title": "Rate of objects allocated", "type": "timeseries" }, { @@ -2490,7 +2491,7 @@ "refId": "A" } ], - "title": "Rate of a Pointer Dereferences", + "title": "Rate of a pointer dereferences", "type": "timeseries" }, { @@ -2686,7 +2687,7 @@ "refId": "A" } ], - "title": "Rates of Allocation", + "title": "Rates of allocation", "type": "timeseries" }, { @@ -2795,7 +2796,7 @@ "list": [ { "current": { - "selected": true, + "selected": false, "text": "store", "value": "store" }, @@ -2906,6 +2907,6 @@ "timezone": "browser", "title": "gRPC Server", "uid": "cdydeqdbz124gf", - "version": 10, + "version": 3, "weekStart": "" } From cfca1461806c5df2be81d7652b6259fb9b01ea5a Mon Sep 17 00:00:00 2001 From: Jeffrey Vervoort Date: Mon, 28 Oct 2024 15:37:52 +0100 Subject: [PATCH 16/19] Delete old readme. --- .gitignore | 3 --- datastore/database/README.md | 10 ---------- 2 files changed, 13 deletions(-) delete mode 100644 datastore/database/README.md diff --git a/.gitignore b/.gitignore index 1b91787a..d6a7061c 100644 --- a/.gitignore +++ b/.gitignore @@ -212,6 +212,3 @@ cf_standard_names_v84.txt # API api/test/output/ datastore/load-test/output/ - -# Database -datastore/database/logs/*.log diff --git a/datastore/database/README.md b/datastore/database/README.md deleted file mode 100644 index 91aa41f4..00000000 --- a/datastore/database/README.md +++ /dev/null @@ -1,10 +0,0 @@ -# Database logs -Set UID and GID of the postgres user -`sudo chown -R 101:103 ./logs` - - -To watch the database logs: - -`sudo tail -f ./logs/postgresql-%Y-%m-%d.log` - -`sudo tail -f ./logs/postgresql-2024-09-18.log` From b33c15091546e6ff39bcd75f97ce05838c984c5e Mon Sep 17 00:00:00 2001 From: Jeffrey Vervoort Date: Tue, 29 Oct 2024 08:16:17 +0100 Subject: [PATCH 17/19] Change variables which are list of values to a single value. As in 99% of the time we won't look at the others. --- grafana/dashboards/database-performance.json | 31 ++++++++------------ grafana/dashboards/grpc-server.json | 30 ++++++++----------- 2 files changed, 26 insertions(+), 35 deletions(-) diff --git a/grafana/dashboards/database-performance.json b/grafana/dashboards/database-performance.json index 24fb3ebc..6898cfdd 100755 --- a/grafana/dashboards/database-performance.json +++ b/grafana/dashboards/database-performance.json @@ -1647,38 +1647,33 @@ }, { "current": { - "selected": false, + "selected": true, "text": "prometheus-postgres-exporter:9187", "value": "prometheus-postgres-exporter:9187" }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "definition": "label_values(instance)", "hide": 0, "includeAll": false, "label": "Instance", "multi": false, "name": "INSTANCE", - "options": [], - "query": { - "qryType": 1, - "query": "label_values(instance)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", + "options": [ + { + "selected": true, + "text": "prometheus-postgres-exporter:9187", + "value": "prometheus-postgres-exporter:9187" + } + ], + "query": "prometheus-postgres-exporter:9187", + "queryValue": "", "skipUrlSync": false, - "sort": 1, - "type": "query" + "type": "custom" }, { "auto": false, "auto_count": 30, "auto_min": "10s", "current": { - "selected": false, + "selected": true, "text": "1m", "value": "1m" }, @@ -1758,6 +1753,6 @@ "timezone": "browser", "title": "Database Performance", "uid": "adxl34wpc3if4a", - "version": 5, + "version": 9, "weekStart": "" } diff --git a/grafana/dashboards/grpc-server.json b/grafana/dashboards/grpc-server.json index 78c0fde7..22158e5c 100755 --- a/grafana/dashboards/grpc-server.json +++ b/grafana/dashboards/grpc-server.json @@ -2796,31 +2796,27 @@ "list": [ { "current": { - "selected": false, + "selected": true, "text": "store", "value": "store" }, - "datasource": { - "type": "prometheus", - "uid": "PBFA97CFB590B2093" - }, - "definition": "label_values(job)", + "description": "Name of the database", "hide": 0, "includeAll": false, "label": "Job", "multi": false, "name": "JOB", - "options": [], - "query": { - "qryType": 1, - "query": "label_values(job)", - "refId": "PrometheusVariableQueryEditor-VariableQuery" - }, - "refresh": 1, - "regex": "", + "options": [ + { + "selected": true, + "text": "store", + "value": "store" + } + ], + "query": "store", + "queryValue": "", "skipUrlSync": false, - "sort": 1, - "type": "query" + "type": "custom" }, { "auto": false, @@ -2907,6 +2903,6 @@ "timezone": "browser", "title": "gRPC Server", "uid": "cdydeqdbz124gf", - "version": 3, + "version": 6, "weekStart": "" } From e989c33896a53cceb37455ce453487995ba7c2e7 Mon Sep 17 00:00:00 2001 From: Jeffrey Vervoort Date: Tue, 5 Nov 2024 15:46:25 +0100 Subject: [PATCH 18/19] Rename to in flight grpc requests. --- datastore/datastore/main/main.go | 4 ++-- datastore/datastore/metrics/promservermetrics.go | 12 ++++++------ grafana/dashboards/grpc-server.json | 6 +++--- 3 files changed, 11 insertions(+), 11 deletions(-) diff --git a/datastore/datastore/main/main.go b/datastore/datastore/main/main.go index ae19695f..1f78da59 100644 --- a/datastore/datastore/main/main.go +++ b/datastore/datastore/main/main.go @@ -68,7 +68,7 @@ func main() { reg := prometheus.NewRegistry() reg.MustRegister( grpcMetrics, - promservermetrics.ActiveConnections, + promservermetrics.InFlightRequests, promservermetrics.UptimeCounter, promservermetrics.ResponseSizeSummary, collectors.NewGoCollector(), @@ -81,7 +81,7 @@ func main() { server := grpc.NewServer( grpc.ChainUnaryInterceptor( reqTimeLogger, - promservermetrics.ConnectionUnaryInterceptor, + promservermetrics.InFlightRequestInterceptor, promservermetrics.ResponseSizeUnaryInterceptor, grpcMetrics.UnaryServerInterceptor(), ), diff --git a/datastore/datastore/metrics/promservermetrics.go b/datastore/datastore/metrics/promservermetrics.go index f4000431..c7d2cf99 100644 --- a/datastore/datastore/metrics/promservermetrics.go +++ b/datastore/datastore/metrics/promservermetrics.go @@ -14,9 +14,9 @@ var ( Help: "Total uptime of the gRPC server in seconds", }) - ActiveConnections = prometheus.NewGauge(prometheus.GaugeOpts{ - Name: "grpc_active_connections", - Help: "Current number of active gRPC connections", + InFlightRequests = prometheus.NewGauge(prometheus.GaugeOpts{ + Name: "grpc_in_flight_requests", + Help: "Current number of in-flight gRPC requests", }) ResponseSizeSummary = prometheus.NewSummaryVec( @@ -41,9 +41,9 @@ func TrackUptime() { } } -func ConnectionUnaryInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { - ActiveConnections.Inc() // Increment when a new unary request (connection) is opened - defer ActiveConnections.Dec() // Decrement when the unary request (connection) is completed +func InFlightRequestInterceptor(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, handler grpc.UnaryHandler) (interface{}, error) { + InFlightRequests.Inc() // Increment at the start of the request + defer InFlightRequests.Dec() // Decrement at the end of the request return handler(ctx, req) } diff --git a/grafana/dashboards/grpc-server.json b/grafana/dashboards/grpc-server.json index 22158e5c..6490fc83 100755 --- a/grafana/dashboards/grpc-server.json +++ b/grafana/dashboards/grpc-server.json @@ -1146,14 +1146,14 @@ "uid": "PBFA97CFB590B2093" }, "editorMode": "code", - "expr": "grpc_active_connections", + "expr": "grpc_in_flight_requests", "instant": false, - "legendFormat": "Active gRPC connections", + "legendFormat": "In flight requests", "range": true, "refId": "A" } ], - "title": "Active gRPC connections", + "title": "In flight requests", "type": "timeseries" }, { From 6d45f4807a9249240de7fa339ce0da85984cef06 Mon Sep 17 00:00:00 2001 From: Jeffrey Vervoort Date: Wed, 6 Nov 2024 11:19:25 +0100 Subject: [PATCH 19/19] When adding pg_stat_statements to the settings of the database with the extra.conf the extension is loaded on a new clean database (just destroy local). But it is not created for existing databases. Add a migration to create the extension. --- .../migrations/1730887695_create_pg_stat_statements.down.sql | 1 + .../data/migrations/1730887695_create_pg_stat_statements.up.sql | 1 + 2 files changed, 2 insertions(+) create mode 100644 datastore/migrate/data/migrations/1730887695_create_pg_stat_statements.down.sql create mode 100644 datastore/migrate/data/migrations/1730887695_create_pg_stat_statements.up.sql diff --git a/datastore/migrate/data/migrations/1730887695_create_pg_stat_statements.down.sql b/datastore/migrate/data/migrations/1730887695_create_pg_stat_statements.down.sql new file mode 100644 index 00000000..be80e5c7 --- /dev/null +++ b/datastore/migrate/data/migrations/1730887695_create_pg_stat_statements.down.sql @@ -0,0 +1 @@ +DROP EXTENSION IF EXISTS pg_stat_statements; diff --git a/datastore/migrate/data/migrations/1730887695_create_pg_stat_statements.up.sql b/datastore/migrate/data/migrations/1730887695_create_pg_stat_statements.up.sql new file mode 100644 index 00000000..841ff0c4 --- /dev/null +++ b/datastore/migrate/data/migrations/1730887695_create_pg_stat_statements.up.sql @@ -0,0 +1 @@ +CREATE EXTENSION IF NOT EXISTS pg_stat_statements;