From 35febac7230de1b79399e0c20db96e15dc7eeaa4 Mon Sep 17 00:00:00 2001 From: QuentinBisson Date: Thu, 11 Apr 2024 11:33:04 +0200 Subject: [PATCH] Use mixins Signed-off-by: QuentinBisson --- .../shared/private/loki-canary.json | 92 +- .../shared/private/loki-chunks.json | 1295 +++++------- .../shared/private/loki-deletion.json | 790 +++---- .../dashboards/shared/private/loki-logs.json | 30 +- .../private/loki-mixin-recording-rules.json | 6 +- .../shared/private/loki-operational.json | 186 +- .../shared/private/loki-reads-resources.json | 1085 +++++----- .../dashboards/shared/private/loki-reads.json | 836 +++++--- .../shared/private/loki-retention.json | 1853 +++++++---------- .../shared/private/loki-writes-resources.json | 786 +++---- .../shared/private/loki-writes.json | 827 +++++--- loki/.gitignore | 5 + loki/mixin.libsonnet | 17 + loki/update.sh | 39 +- 14 files changed, 3543 insertions(+), 4304 deletions(-) create mode 100644 loki/.gitignore create mode 100644 loki/mixin.libsonnet diff --git a/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-canary.json b/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-canary.json index 43a40254..9afbd2db 100644 --- a/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-canary.json +++ b/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-canary.json @@ -12,7 +12,7 @@ }, "editable": true, "gnetId": null, - "graphTooltip": 0, + "graphTooltip": 1, "hideControls": false, "links": [ { @@ -21,7 +21,9 @@ "includeVars": true, "keepTime": true, "tags": [ - "loki" + "owner:team-atlas", + "topic:observability", + "component:loki" ], "targetBlank": false, "title": "Loki Dashboards", @@ -81,7 +83,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(count(loki_canary_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}))", + "expr": "sum(count(loki_canary_entries_total{cluster_id=~\"$cluster\", namespace=~\"$namespace\"}))", "format": null, "instant": false, "interval": "", @@ -179,7 +181,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(increase(loki_canary_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))", + "expr": "sum(increase(loki_canary_entries_total{cluster_id=~\"$cluster\", namespace=~\"$namespace\"}[$__range]))", "format": null, "instant": false, "interval": "", @@ -277,7 +279,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(increase(loki_canary_missing_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))", + "expr": "sum(increase(loki_canary_missing_entries_total{cluster_id=~\"$cluster\", namespace=~\"$namespace\"}[$__range]))", "format": null, "instant": false, "interval": "", @@ -375,7 +377,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(increase(loki_canary_spot_check_missing_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))", + "expr": "sum(increase(loki_canary_spot_check_missing_entries_total{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))", "format": null, "instant": false, "interval": "", @@ -473,7 +475,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(increase(loki_canary_spot_check_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))", + "expr": "sum(increase(loki_canary_spot_check_entries_total{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))", "format": null, "instant": false, "interval": "", @@ -571,7 +573,7 @@ "steppedLine": false, "targets": [ { - "expr": "((sum(loki_canary_metric_test_expected{cluster=~\"$cluster\",namespace=~\"$namespace\"}) - sum(loki_canary_metric_test_actual{cluster=~\"$cluster\",namespace=~\"$namespace\"}))/(sum(loki_canary_metric_test_actual{cluster=~\"$cluster\",namespace=~\"$namespace\"}))) * 100", + "expr": "((sum(loki_canary_metric_test_expected{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}) - sum(loki_canary_metric_test_actual{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}))/(sum(loki_canary_metric_test_actual{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}))) * 100", "format": null, "instant": false, "interval": "", @@ -669,7 +671,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(increase(loki_canary_missing_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))/sum(increase(loki_canary_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__range])))*100", + "expr": "(sum(increase(loki_canary_missing_entries_total{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))/sum(increase(loki_canary_entries_total{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[$__range])))*100", "format": null, "instant": false, "interval": "", @@ -767,7 +769,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(increase(loki_canary_spot_check_missing_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))/sum(increase(loki_canary_spot_check_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))) * 100", + "expr": "(sum(increase(loki_canary_spot_check_missing_entries_total{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))/sum(increase(loki_canary_spot_check_entries_total{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))) * 100", "format": null, "instant": false, "interval": "", @@ -865,7 +867,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(loki_canary_metric_test_expected{cluster=~\"$cluster\",namespace=~\"$namespace\"})", + "expr": "sum(loki_canary_metric_test_expected{cluster_id=~\"$cluster\",namespace=~\"$namespace\"})", "format": null, "instant": false, "interval": "", @@ -963,7 +965,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(loki_canary_metric_test_actual{cluster=~\"$cluster\",namespace=~\"$namespace\"})", + "expr": "sum(loki_canary_metric_test_actual{cluster_id=~\"$cluster\",namespace=~\"$namespace\"})", "format": null, "instant": false, "interval": "", @@ -1061,7 +1063,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(increase(loki_canary_websocket_missing_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))", + "expr": "sum(increase(loki_canary_websocket_missing_entries_total{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))", "format": null, "instant": false, "interval": "", @@ -1159,7 +1161,7 @@ "steppedLine": false, "targets": [ { - "expr": "(sum(increase(loki_canary_websocket_missing_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))/sum(increase(loki_canary_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__range])))*100", + "expr": "(sum(increase(loki_canary_websocket_missing_entries_total{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))/sum(increase(loki_canary_entries_total{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[$__range])))*100", "format": null, "instant": false, "interval": "", @@ -1242,20 +1244,16 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.95, sum(rate(loki_canary_response_latency_seconds_bucket{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(loki_canary_response_latency_seconds_bucket{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) by (le))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "p95", - "legendLink": null, - "step": 10 + "legendLink": null }, { - "expr": "histogram_quantile(0.50, sum(rate(loki_canary_response_latency_seconds_bucket{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.50, sum(rate(loki_canary_response_latency_seconds_bucket{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) by (le))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "p50", - "legendLink": null, - "step": 10 + "legendLink": null } ], "thresholds": [], @@ -1322,7 +1320,7 @@ }, "targets": [ { - "expr": "sum(rate(loki_canary_response_latency_seconds_bucket{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) by (le)", + "expr": "sum(rate(loki_canary_response_latency_seconds_bucket{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) by (le)", "format": "heatmap", "intervalFactor": 2, "legendFormat": "{{le}}", @@ -1390,20 +1388,16 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(loki_canary_spot_check_request_duration_seconds_bucket{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(loki_canary_spot_check_request_duration_seconds_bucket{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) by (le))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "p99", - "legendLink": null, - "step": 10 + "legendLink": null }, { - "expr": "histogram_quantile(0.50, sum(rate(loki_canary_spot_check_request_duration_seconds_bucket{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.50, sum(rate(loki_canary_spot_check_request_duration_seconds_bucket{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) by (le))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "p95", - "legendLink": null, - "step": 10 + "legendLink": null } ], "thresholds": [], @@ -1479,20 +1473,16 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(loki_canary_metric_test_request_duration_seconds_bucket{cluster=~\"$cluster\",namespace=~\"$namespace\"}[15m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(loki_canary_metric_test_request_duration_seconds_bucket{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[15m])) by (le))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "p99", - "legendLink": null, - "step": 10 + "legendLink": null }, { - "expr": "histogram_quantile(0.50, sum(rate(loki_canary_metric_test_request_duration_seconds_bucket{cluster=~\"$cluster\",namespace=~\"$namespace\"}[15m])) by (le))", + "expr": "histogram_quantile(0.50, sum(rate(loki_canary_metric_test_request_duration_seconds_bucket{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[15m])) by (le))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "p95", - "legendLink": null, - "step": 10 + "legendLink": null } ], "thresholds": [], @@ -1568,12 +1558,10 @@ "steppedLine": false, "targets": [ { - "expr": "topk(20, (sum by (cluster, pod) (increase(loki_canary_spot_check_missing_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval]))/sum by (cluster, pod) (increase(loki_canary_spot_check_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) * 100)) > 0", + "expr": "topk(20, (sum by (cluster_id, pod) (increase(loki_canary_spot_check_missing_entries_total{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval]))/sum by (cluster_id, pod) (increase(loki_canary_spot_check_entries_total{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) * 100)) > 0", "format": "time_series", - "intervalFactor": 2, "legendFormat": "", - "legendLink": null, - "step": 10 + "legendLink": null } ], "thresholds": [], @@ -1649,12 +1637,10 @@ "steppedLine": false, "targets": [ { - "expr": "topk(20,(sum by (cluster, pod)(increase(loki_canary_missing_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval]))/sum by (cluster, pod)(increase(loki_canary_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])))*100) > 0", + "expr": "topk(20,(sum by (cluster_id, pod)(increase(loki_canary_missing_entries_total{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval]))/sum by (cluster_id, pod)(increase(loki_canary_entries_total{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])))*100) > 0", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Missing {{ cluster }} {{ pod }}", - "legendLink": null, - "step": 10 + "legendFormat": "Missing {{ cluster_id }} {{ pod }}", + "legendLink": null } ], "thresholds": [], @@ -1699,7 +1685,9 @@ "schemaVersion": 27, "style": "dark", "tags": [ - "loki" + "owner:team-atlas", + "topic:observability", + "component:loki" ], "templating": { "list": [ @@ -1709,7 +1697,7 @@ "value": "default" }, "hide": 0, - "label": "Data Source", + "label": "Data source", "name": "datasource", "options": [], "query": "prometheus", @@ -1730,7 +1718,7 @@ "multi": false, "name": "cluster", "options": [], - "query": "label_values(loki_build_info, cluster)", + "query": "label_values(loki_build_info, cluster_id)", "refresh": 1, "regex": "", "sort": 2, @@ -1753,7 +1741,7 @@ "multi": false, "name": "namespace", "options": [], - "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)", + "query": "label_values(loki_build_info{cluster_id=~\"$cluster\"}, namespace)", "refresh": 1, "regex": "", "sort": 2, diff --git a/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-chunks.json b/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-chunks.json index 2ef85f5d..712fbe7f 100644 --- a/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-chunks.json +++ b/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-chunks.json @@ -29,156 +29,98 @@ "height": "250px", "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [] }, - "lines": true, - "linewidth": 1, + "id": 1, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "sum(loki_ingester_memory_chunks{cluster_id=\"$cluster_id\", job=~\"$namespace/(loki|enterprise-logs)-write\"})", + "expr": "sum(loki_ingester_memory_chunks{cluster_id=~\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "series", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Series", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [] }, - "lines": true, - "linewidth": 1, + "id": 2, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "sum(loki_ingester_memory_chunks{cluster_id=\"$cluster_id\", job=~\"$namespace/(loki|enterprise-logs)-write\"}) / sum(loki_ingester_memory_streams{cluster_id=\"$cluster_id\", job=~\"$namespace/(loki|enterprise-logs)-write\"})", + "expr": "sum(loki_ingester_memory_chunks{cluster_id=~\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}) / sum(loki_ingester_memory_streams{cluster_id=~\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "chunks", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Chunks per series", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -193,81 +135,67 @@ "height": "250px", "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "percentunit" + }, + "overrides": [] }, - "lines": true, - "linewidth": 1, + "id": 3, "links": [], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_utilization_bucket{cluster_id=\"$cluster_id\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1", + "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_utilization_bucket{cluster_id=~\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1", "format": "time_series", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { - "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_utilization_bucket{cluster_id=\"$cluster_id\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1", + "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_utilization_bucket{cluster_id=~\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1", "format": "time_series", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { - "expr": "sum(rate(loki_ingester_chunk_utilization_sum{cluster_id=\"$cluster_id\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) * 1 / sum(rate(loki_ingester_chunk_utilization_count{cluster_id=\"$cluster_id\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]))", + "expr": "sum(rate(loki_ingester_chunk_utilization_sum{cluster_id=~\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) * 1 / sum(rate(loki_ingester_chunk_utilization_count{cluster_id=~\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Utilization", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, + "type": "timeseries", "yaxes": [ { - "format": "percentunit", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -285,78 +213,64 @@ ] }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [] }, - "lines": true, - "linewidth": 1, + "id": 4, "links": [], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_age_seconds_bucket{cluster_id=\"$cluster_id\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_age_seconds_bucket{cluster_id=~\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { - "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_age_seconds_bucket{cluster_id=\"$cluster_id\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_age_seconds_bucket{cluster_id=~\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { - "expr": "sum(rate(loki_ingester_chunk_age_seconds_sum{cluster_id=\"$cluster_id\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) * 1e3 / sum(rate(loki_ingester_chunk_age_seconds_count{cluster_id=\"$cluster_id\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]))", + "expr": "sum(rate(loki_ingester_chunk_age_seconds_sum{cluster_id=~\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) * 1e3 / sum(rate(loki_ingester_chunk_age_seconds_count{cluster_id=~\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Age", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -389,81 +303,67 @@ "height": "250px", "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [] }, - "lines": true, - "linewidth": 1, + "id": 5, "links": [], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_entries_bucket{cluster_id=\"$cluster_id\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1", + "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_entries_bucket{cluster_id=~\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1", "format": "time_series", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { - "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_entries_bucket{cluster_id=\"$cluster_id\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1", + "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_entries_bucket{cluster_id=~\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1", "format": "time_series", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { - "expr": "sum(rate(loki_ingester_chunk_entries_sum{cluster_id=\"$cluster_id\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) * 1 / sum(rate(loki_ingester_chunk_entries_count{cluster_id=\"$cluster_id\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]))", + "expr": "sum(rate(loki_ingester_chunk_entries_sum{cluster_id=~\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) * 1 / sum(rate(loki_ingester_chunk_entries_count{cluster_id=~\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Log Entries Per Chunk", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, + "type": "timeseries", "yaxes": [ { - "format": "short", + "format": "ms", "label": null, "logBase": 1, "max": null, @@ -481,80 +381,51 @@ ] }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [] }, - "lines": true, - "linewidth": 1, + "id": 6, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_chunk_store_index_entries_per_chunk_sum{cluster_id=\"$cluster_id\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[5m])) / sum(rate(loki_chunk_store_index_entries_per_chunk_count{cluster_id=\"$cluster_id\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[5m]))", + "expr": "sum(rate(loki_chunk_store_index_entries_per_chunk_sum{cluster_id=~\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[5m])) / sum(rate(loki_chunk_store_index_entries_per_chunk_count{cluster_id=~\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[5m]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Index Entries", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Index Entries Per Chunk", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -569,80 +440,51 @@ "height": "250px", "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [] }, - "lines": true, - "linewidth": 1, + "id": 7, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "loki_ingester_flush_queue_length{cluster_id=\"$cluster_id\", job=~\"$namespace/(loki|enterprise-logs)-write\"} or cortex_ingester_flush_queue_length{cluster_id=\"$cluster_id\", job=~\"$namespace/(loki|enterprise-logs)-write\"}", + "expr": "loki_ingester_flush_queue_length{cluster_id=~\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"} or cortex_ingester_flush_queue_length{cluster_id=~\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Queue Length", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { "aliasColors": { @@ -651,82 +493,196 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_ingester_chunk_age_seconds_count{cluster_id=\"$cluster_id\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_ingester_chunk_age_seconds_count{cluster_id=~\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Flush Rate", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -741,138 +697,99 @@ "height": "250px", "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [] }, - "lines": true, - "linewidth": 1, + "id": 9, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_ingester_chunks_flushed_total{cluster_id=\"$cluster_id\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]))", + "expr": "sum(rate(loki_ingester_chunks_flushed_total{cluster_id=~\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Chunks Flushed/Second", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [] }, - "lines": true, - "linewidth": 1, + "id": 10, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (reason) (rate(loki_ingester_chunks_flushed_total{cluster_id=\"$cluster_id\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) / ignoring(reason) group_left sum(rate(loki_ingester_chunks_flushed_total{cluster_id=\"$cluster_id\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]))", + "expr": "sum by (reason) (rate(loki_ingester_chunks_flushed_total{cluster_id=~\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) / ignoring(reason) group_left sum(rate(loki_ingester_chunks_flushed_total{cluster_id=~\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{reason}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Chunk Flush Reason", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, + "type": "timeseries", "yaxes": [ { "format": "short", @@ -928,7 +845,7 @@ "span": 12, "targets": [ { - "expr": "sum by (le) (rate(loki_ingester_chunk_utilization_bucket{cluster_id=\"$cluster_id\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]))", + "expr": "sum by (le) (rate(loki_ingester_chunk_utilization_bucket{cluster_id=~\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]))", "format": "heatmap", "intervalFactor": 2, "legendFormat": "{{le}}", @@ -990,7 +907,7 @@ "span": 12, "targets": [ { - "expr": "sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster_id=\"$cluster_id\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)", + "expr": "sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster_id=~\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)", "format": "heatmap", "intervalFactor": 2, "legendFormat": "{{le}}", @@ -1029,96 +946,63 @@ "height": "250px", "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 13, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "bytes" + }, + "overrides": [] }, - "lines": true, - "linewidth": 1, + "id": 13, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 12, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster_id=\"$cluster_id\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[1m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster_id=~\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[1m])) by (le))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "p99", - "legendLink": null, - "step": 10 + "legendLink": null }, { - "expr": "histogram_quantile(0.90, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster_id=\"$cluster_id\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[1m])) by (le))", + "expr": "histogram_quantile(0.90, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster_id=~\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[1m])) by (le))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "p90", - "legendLink": null, - "step": 10 + "legendLink": null }, { - "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster_id=\"$cluster_id\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[1m])) by (le))", + "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster_id=~\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[1m])) by (le))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "p50", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Chunk Size Quantiles", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -1133,96 +1017,63 @@ "height": "250px", "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 14, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [] }, - "lines": true, - "linewidth": 1, + "id": 14, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 12, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.5, sum(rate(loki_ingester_chunk_bounds_hours_bucket{cluster_id=\"$cluster_id\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[5m])) by (le))", + "expr": "histogram_quantile(0.5, sum(rate(loki_ingester_chunk_bounds_hours_bucket{cluster_id=~\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[5m])) by (le))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "p50", - "legendLink": null, - "step": 10 + "legendLink": null }, { - "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_bounds_hours_bucket{cluster_id=\"$cluster_id\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[5m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_bounds_hours_bucket{cluster_id=~\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[5m])) by (le))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "p99", - "legendLink": null, - "step": 10 + "legendLink": null }, { - "expr": "sum(rate(loki_ingester_chunk_bounds_hours_sum{cluster_id=\"$cluster_id\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[5m])) / sum(rate(loki_ingester_chunk_bounds_hours_count{cluster_id=\"$cluster_id\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[5m]))", + "expr": "sum(rate(loki_ingester_chunk_bounds_hours_sum{cluster_id=~\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[5m])) / sum(rate(loki_ingester_chunk_bounds_hours_count{cluster_id=~\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[5m]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "avg", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Chunk Duration hours (end-start)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -1248,7 +1099,7 @@ "value": "default" }, "hide": 0, - "label": "Data Source", + "label": "Data source", "name": "datasource", "options": [], "query": "prometheus", @@ -1292,7 +1143,7 @@ "multi": false, "name": "namespace", "options": [], - "query": "label_values(loki_build_info{cluster_id=~\"$cluster_id\"}, namespace)", + "query": "label_values(loki_build_info{cluster_id=~\"$cluster\"}, namespace)", "refresh": 1, "regex": "", "sort": 2, diff --git a/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-deletion.json b/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-deletion.json index 5e94f769..be85db07 100644 --- a/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-deletion.json +++ b/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-deletion.json @@ -61,10 +61,9 @@ "steppedLine": false, "targets": [ { - "expr": "sum(loki_compactor_pending_delete_requests_count{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\"})", + "expr": "sum(loki_compactor_pending_delete_requests_count{cluster_id=~\"$cluster\", namespace=~\"$namespace\"})", "format": "time_series", "instant": true, - "intervalFactor": 2, "refId": "A" } ], @@ -137,10 +136,9 @@ "steppedLine": false, "targets": [ { - "expr": "max(loki_compactor_oldest_pending_delete_request_age_seconds{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\"})", + "expr": "max(loki_compactor_oldest_pending_delete_request_age_seconds{cluster_id=~\"$cluster\", namespace=~\"$namespace\"})", "format": "time_series", "instant": true, - "intervalFactor": 2, "refId": "A" } ], @@ -193,232 +191,145 @@ "height": "250px", "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [] }, - "lines": true, - "linewidth": 1, + "id": 3, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "(loki_compactor_delete_requests_received_total{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\"} or on() vector(0)) - on () (loki_compactor_delete_requests_processed_total{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\"} or on () vector(0))", + "expr": "(loki_compactor_delete_requests_received_total{cluster_id=~\"$cluster\", namespace=~\"$namespace\"} or on() vector(0)) - on () (loki_compactor_delete_requests_processed_total{cluster_id=~\"$cluster\", namespace=~\"$namespace\"} or on () vector(0))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "in progress", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "# of Delete Requests (received - processed) ", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [] }, - "lines": true, - "linewidth": 1, + "id": 4, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "sum(increase(loki_compactor_delete_requests_received_total{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\"}[1d]))", + "expr": "sum(increase(loki_compactor_delete_requests_received_total{cluster_id=~\"$cluster\", namespace=~\"$namespace\"}[1d]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "received", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Delete Requests Received / Day", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [] }, - "lines": true, - "linewidth": 1, + "id": 5, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "sum(increase(loki_compactor_delete_requests_processed_total{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\"}[1d]))", + "expr": "sum(increase(loki_compactor_delete_requests_processed_total{cluster_id=~\"$cluster\", namespace=~\"$namespace\"}[1d]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "processed", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Delete Requests Processed / Day", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -433,232 +344,145 @@ "height": "250px", "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [] }, - "lines": true, - "linewidth": 1, + "id": 6, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"compactor\"}", + "expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster_id=~\"$cluster\", namespace=~\"$namespace\", container=\"compactor\"}", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Compactor CPU usage", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [] }, - "lines": true, - "linewidth": 1, + "id": 7, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "go_memstats_heap_inuse_bytes{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"compactor\"} / 1024 / 1024 ", + "expr": "go_memstats_heap_inuse_bytes{cluster_id=~\"$cluster\", namespace=~\"$namespace\", container=\"compactor\"} / 1024 / 1024 ", "format": "time_series", - "intervalFactor": 2, "legendFormat": " {{pod}} ", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Compactor memory usage (MiB)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [] }, - "lines": true, - "linewidth": 1, + "id": 8, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "loki_boltdb_shipper_compact_tables_operation_duration_seconds{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\"}", + "expr": "loki_boltdb_shipper_compact_tables_operation_duration_seconds{cluster_id=~\"$cluster\", namespace=~\"$namespace\"}", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Compaction run duration (seconds)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -673,156 +497,98 @@ "height": "250px", "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [] }, - "lines": true, - "linewidth": 1, + "id": 9, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "sum(increase(loki_compactor_load_pending_requests_attempts_total{status=\"fail\", cluster_id=~\"$cluster_id\", namespace=~\"$namespace\"}[1h]))", + "expr": "sum(increase(loki_compactor_load_pending_requests_attempts_total{status=\"fail\", cluster_id=~\"$cluster\", namespace=~\"$namespace\"}[1h]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "failures", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Failures in Loading Delete Requests / Hour", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [] }, - "lines": true, - "linewidth": 1, + "id": 10, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_compactor_deleted_lines{cluster_id=~\"$cluster_id\",job=~\"$namespace/(loki|enterprise-logs)-read\"}[$__rate_interval])) by (user)", + "expr": "sum(rate(loki_compactor_deleted_lines{cluster_id=~\"$cluster\",job=~\"$namespace/(loki|enterprise-logs)-read\"}[$__rate_interval])) by (user)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{user}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Lines Deleted / Sec", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -842,7 +608,7 @@ "span": 6, "targets": [ { - "expr": "{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"compactor\"} |~ \"Started processing delete request|delete request for user marked as processed\" | logfmt | line_format \"{{.ts}} user={{.user}} delete_request_id={{.delete_request_id}} msg={{.msg}}\" ", + "expr": "{cluster_id=~\"$cluster\", namespace=~\"$namespace\", container=\"compactor\"} |~ \"Started processing delete request|delete request for user marked as processed\" | logfmt | line_format \"{{.ts}} user={{.user}} delete_request_id={{.delete_request_id}} msg={{.msg}}\" ", "refId": "A" } ], @@ -855,7 +621,7 @@ "span": 6, "targets": [ { - "expr": "{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"compactor\"} |~ \"delete request for user added\" | logfmt | line_format \"{{.ts}} user={{.user}} query='{{.query}}'\"", + "expr": "{cluster_id=~\"$cluster\", namespace=~\"$namespace\", container=\"compactor\"} |~ \"delete request for user added\" | logfmt | line_format \"{{.ts}} user={{.user}} query='{{.query}}'\"", "refId": "A" } ], @@ -886,7 +652,7 @@ "value": "default" }, "hide": 0, - "label": "Data Source", + "label": "Data source", "name": "datasource", "options": [], "query": "prometheus", @@ -930,7 +696,7 @@ "multi": false, "name": "namespace", "options": [], - "query": "label_values(loki_build_info{cluster_id=~\"$cluster_id\"}, namespace)", + "query": "label_values(loki_build_info{cluster_id=~\"$cluster\"}, namespace)", "refresh": 1, "regex": "", "sort": 2, diff --git a/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-logs.json b/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-logs.json index 171491c6..233cc5d7 100644 --- a/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-logs.json +++ b/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-logs.json @@ -65,7 +65,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(go_goroutines{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"})", + "expr": "sum(go_goroutines{cluster=~\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"})", "refId": "A" } ], @@ -151,7 +151,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(go_gc_duration_seconds{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"}) by (quantile)", + "expr": "sum(go_gc_duration_seconds{cluster=~\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"}) by (quantile)", "legendFormat": "{{quantile}}", "refId": "A" } @@ -238,7 +238,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"}[5m]))", + "expr": "sum(rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"}[5m]))", "refId": "A" } ], @@ -324,7 +324,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"})", + "expr": "sum(container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"})", "refId": "A" } ], @@ -410,7 +410,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"}[5m]))", + "expr": "sum(rate(container_network_transmit_bytes_total{cluster=~\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"}[5m]))", "refId": "A" } ], @@ -496,7 +496,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"}[5m]))", + "expr": "sum(rate(container_network_receive_bytes_total{cluster=~\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"}[5m]))", "refId": "A" } ], @@ -582,7 +582,7 @@ "steppedLine": false, "targets": [ { - "expr": "increase(kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"}[30m]) > 0", + "expr": "increase(kube_pod_container_status_last_terminated_reason{cluster=~\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"}[30m]) > 0", "legendFormat": "{{reason}}", "refId": "A" } @@ -669,7 +669,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(promtail_custom_bad_words_total{cluster=\"$cluster\", exported_namespace=\"$namespace\", exported_pod=~\"$deployment.*\", exported_pod=~\"$pod\", container=~\"$container\"}[5m])) by (level)", + "expr": "sum(rate(promtail_custom_bad_words_total{cluster=~\"$cluster\", exported_namespace=\"$namespace\", exported_pod=~\"$deployment.*\", exported_pod=~\"$pod\", container=~\"$container\"}[5m])) by (level)", "legendFormat": "{{level}}", "refId": "A" } @@ -773,7 +773,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\" } |logfmt| level=\"$level\" |= \"$filter\" [5m])) by (level)", + "expr": "sum(rate({cluster=~\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\" } |logfmt| level=\"$level\" |= \"$filter\" [5m])) by (level)", "intervalFactor": 3, "legendFormat": "{{level}}", "refId": "A" @@ -838,7 +838,7 @@ }, "targets": [ { - "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"} | logfmt | level=\"$level\" |= \"$filter\"", + "expr": "{cluster=~\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"} | logfmt | level=\"$level\" |= \"$filter\"", "refId": "A" } ], @@ -865,7 +865,7 @@ "value": "default" }, "hide": 0, - "label": "Data Source", + "label": "Data source", "name": "datasource", "options": [], "query": "prometheus", @@ -909,7 +909,7 @@ "multi": false, "name": "namespace", "options": [], - "query": "label_values(loki_build_info{cluster_id=~\"$cluster_id\"}, namespace)", + "query": "label_values(loki_build_info{cluster_id=~\"$cluster\"}, namespace)", "refresh": 1, "regex": "", "sort": 2, @@ -939,7 +939,7 @@ "multi": false, "name": "deployment", "options": [], - "query": "label_values(kube_deployment_created{cluster_id=\"$cluster_id\", namespace=\"$namespace\"}, deployment)", + "query": "label_values(kube_deployment_created{cluster_id=~\"$cluster\", namespace=\"$namespace\"}, deployment)", "refresh": 0, "regex": "", "sort": 1, @@ -959,7 +959,7 @@ "multi": false, "name": "pod", "options": [], - "query": "label_values(kube_pod_container_info{cluster_id=\"$cluster_id\", namespace=\"$namespace\", pod=~\"$deployment.*\"}, pod)", + "query": "label_values(kube_pod_container_info{cluster_id=~\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\"}, pod)", "refresh": 0, "regex": "", "sort": 1, @@ -979,7 +979,7 @@ "multi": false, "name": "container", "options": [], - "query": "label_values(kube_pod_container_info{cluster_id=\"$cluster_id\", namespace=\"$namespace\", pod=~\"$pod\", pod=~\"$deployment.*\"}, container)", + "query": "label_values(kube_pod_container_info{cluster_id=~\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\", pod=~\"$deployment.*\"}, container)", "refresh": 0, "regex": "", "sort": 1, diff --git a/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-mixin-recording-rules.json b/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-mixin-recording-rules.json index 7c19b841..5ab6c442 100644 --- a/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-mixin-recording-rules.json +++ b/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-mixin-recording-rules.json @@ -604,7 +604,7 @@ "value": "default" }, "hide": 0, - "label": "Data Source", + "label": "Data source", "name": "datasource", "options": [], "query": "prometheus", @@ -648,7 +648,7 @@ "multi": false, "name": "namespace", "options": [], - "query": "label_values(loki_build_info{cluster_id=~\"$cluster_id\"}, namespace)", + "query": "label_values(loki_build_info{cluster_id=~\"$cluster\"}, namespace)", "refresh": 1, "regex": "", "sort": 2, @@ -678,7 +678,7 @@ "multi": false, "name": "tenant", "options": [], - "query": "query_result(sum by (id) (grafanacloud_logs_instance_info) and sum(label_replace(loki_tenant:active_streams{cluster_id=\"$cluster_id\",namespace=\"$namespace\"},\"id\",\"$1\",\"tenant\",\"(.*)\")) by(id))", + "query": "query_result(sum by (id) (grafanacloud_logs_instance_info) and sum(label_replace(loki_tenant:active_streams{cluster_id=~\"$cluster\",namespace=\"$namespace\"},\"id\",\"$1\",\"tenant\",\"(.*)\")) by(id))", "refresh": 0, "regex": "/\"([^\"]+)\"/", "sort": 1, diff --git a/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-operational.json b/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-operational.json index 9e75e84a..b5a84f31 100644 --- a/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-operational.json +++ b/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-operational.json @@ -89,7 +89,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\nlabel_replace(\n label_replace(\n rate(loki_request_duration_seconds_count{cluster=\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\"}[5m]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\")\n)", + "expr": "sum by (status) (\nlabel_replace(\n label_replace(\n rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\"}[5m]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\")\n)", "legendFormat": "{{status}}", "refId": "A" } @@ -185,7 +185,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\nlabel_replace(\n label_replace(\n rate(loki_request_duration_seconds_count{cluster=\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"api_prom_push|loki_api_v1_push\"}[5m]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))", + "expr": "sum by (status) (\nlabel_replace(\n label_replace(\n rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"api_prom_push|loki_api_v1_push\"}[5m]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))", "legendFormat": "{{status}}", "refId": "A" } @@ -280,7 +280,7 @@ "steppedLine": false, "targets": [ { - "expr": "topk(10, sum(rate(loki_distributor_lines_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (tenant))", + "expr": "topk(10, sum(rate(loki_distributor_lines_received_total{cluster=~\"$cluster\", namespace=\"$namespace\"}[5m])) by (tenant))", "legendFormat": "{{tenant}}", "refId": "A" } @@ -376,7 +376,7 @@ "steppedLine": false, "targets": [ { - "expr": "topk(10, sum(rate(loki_distributor_bytes_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (tenant)) / 1024 / 1024", + "expr": "topk(10, sum(rate(loki_distributor_bytes_received_total{cluster=~\"$cluster\", namespace=\"$namespace\"}[5m])) by (tenant)) / 1024 / 1024", "legendFormat": "{{tenant}}", "refId": "A" } @@ -470,7 +470,7 @@ "steppedLine": false, "targets": [ { - "expr": "increase(kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\"}[10m]) > 0", + "expr": "increase(kube_pod_container_status_restarts_total{cluster=~\"$cluster\", namespace=\"$namespace\"}[10m]) > 0", "hide": false, "interval": "", "legendFormat": "{{container}}-{{pod}}", @@ -780,7 +780,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (route) > 0", + "expr": "sum(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (route) > 0", "interval": "", "legendFormat": "{{route}}", "refId": "A" @@ -878,18 +878,18 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", route=\"/logproto.Pusher/Push\", cluster=~\"$cluster\"})) * 1e3", + "expr": "histogram_quantile(0.99, sum by (le) (cluster_id_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", route=\"/logproto.Pusher/Push\", cluster=~\"$cluster\"})) * 1e3", "legendFormat": ".99", "refId": "A" }, { - "expr": "histogram_quantile(0.9, sum by (le) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", route=\"/logproto.Pusher/Push\", cluster=~\"$cluster\"})) * 1e3", + "expr": "histogram_quantile(0.9, sum by (le) (cluster_id_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", route=\"/logproto.Pusher/Push\", cluster=~\"$cluster\"})) * 1e3", "hide": false, "legendFormat": ".9", "refId": "B" }, { - "expr": "histogram_quantile(0.5, sum by (le) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", route=\"/logproto.Pusher/Push\", cluster=~\"$cluster\"})) * 1e3", + "expr": "histogram_quantile(0.5, sum by (le) (cluster_id_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", route=\"/logproto.Pusher/Push\", cluster=~\"$cluster\"})) * 1e3", "hide": false, "legendFormat": ".5", "refId": "C" @@ -986,7 +986,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", status_code!~\"5[0-9]{2}\", route=\"/logproto.Pusher/Push\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", route=\"/logproto.Pusher/Push\"}[$__rate_interval])) by (route) > 0", + "expr": "sum(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", status_code!~\"5[0-9]{2}\", route=\"/logproto.Pusher/Push\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", route=\"/logproto.Pusher/Push\"}[$__rate_interval])) by (route) > 0", "interval": "", "legendFormat": "{{route}}", "refId": "A" @@ -1086,17 +1086,17 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"}))", + "expr": "histogram_quantile(0.99, sum by (le,route) (cluster_id_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=~\"$cluster\"}))", "legendFormat": "{{route}}-.99", "refId": "A" }, { - "expr": "histogram_quantile(0.9, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"}))", + "expr": "histogram_quantile(0.9, sum by (le,route) (cluster_id_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=~\"$cluster\"}))", "legendFormat": "{{route}}-.9", "refId": "B" }, { - "expr": "histogram_quantile(0.5, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"}))", + "expr": "histogram_quantile(0.5, sum by (le,route) (cluster_id_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=~\"$cluster\"}))", "legendFormat": "{{route}}-.5", "refId": "C" } @@ -1192,17 +1192,17 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"})) * 1e3", + "expr": "histogram_quantile(0.99, sum by (le,route) (cluster_id_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=~\"$cluster\"})) * 1e3", "legendFormat": ".99-{{route}}", "refId": "A" }, { - "expr": "histogram_quantile(0.9, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"})) * 1e3", + "expr": "histogram_quantile(0.9, sum by (le,route) (cluster_id_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=~\"$cluster\"})) * 1e3", "legendFormat": ".9-{{route}}", "refId": "B" }, { - "expr": "histogram_quantile(0.5, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=\"$cluster\"})) * 1e3", + "expr": "histogram_quantile(0.5, sum by (le,route) (cluster_id_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_labels|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\", cluster=~\"$cluster\"})) * 1e3", "legendFormat": ".5-{{route}}", "refId": "C" } @@ -1298,7 +1298,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-read\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-read\"}[$__rate_interval])) by (route) > 0", + "expr": "sum(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-read\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-read\"}[$__rate_interval])) by (route) > 0", "interval": "", "legendFormat": "{{route}}", "refId": "A" @@ -1397,17 +1397,17 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\", cluster=\"$cluster\"})) * 1e3", + "expr": "histogram_quantile(0.99, sum by (le,route) (cluster_id_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\", cluster=~\"$cluster\"})) * 1e3", "legendFormat": ".99-{{route}}", "refId": "A" }, { - "expr": "histogram_quantile(0.9, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\", cluster=\"$cluster\"})) * 1e3", + "expr": "histogram_quantile(0.9, sum by (le,route) (cluster_id_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\", cluster=~\"$cluster\"})) * 1e3", "legendFormat": ".9-{{route}}", "refId": "B" }, { - "expr": "histogram_quantile(0.5, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\", cluster=\"$cluster\"})) * 1e3", + "expr": "histogram_quantile(0.5, sum by (le,route) (cluster_id_job_route:loki_request_duration_seconds_bucket:sum_rate{job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\", cluster=~\"$cluster\"})) * 1e3", "legendFormat": ".5-{{route}}", "refId": "C" } @@ -1503,7 +1503,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", status_code!~\"5[0-9]{2}\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"}[$__rate_interval])) by (route) > 0", + "expr": "sum(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", status_code!~\"5[0-9]{2}\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"}[$__rate_interval])) by (route) > 0", "interval": "", "legendFormat": "{{route}}", "refId": "A" @@ -1604,7 +1604,7 @@ "steppedLine": false, "targets": [ { - "expr": "topk(10,sum by (tenant, reason) (rate(loki_discarded_samples_total{cluster=\"$cluster\",namespace=\"$namespace\"}[1m])))", + "expr": "topk(10,sum by (tenant, reason) (rate(loki_discarded_samples_total{cluster=~\"$cluster\",namespace=\"$namespace\"}[1m])))", "interval": "", "legendFormat": "{{ tenant }} - {{ reason }}", "refId": "A" @@ -1729,7 +1729,7 @@ ], "targets": [ { - "expr": "topk(10, sum by (tenant, reason) (sum_over_time(increase(loki_discarded_samples_total{cluster=\"$cluster\",namespace=\"$namespace\"}[1m])[$__range:1m])))", + "expr": "topk(10, sum by (tenant, reason) (sum_over_time(increase(loki_discarded_samples_total{cluster=~\"$cluster\",namespace=\"$namespace\"}[1m])[$__range:1m])))", "format": "table", "instant": true, "interval": "", @@ -1892,7 +1892,7 @@ "steppedLine": false, "targets": [ { - "expr": "go_memstats_heap_inuse_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"(loki|enterprise-logs)-write.*\"}", + "expr": "go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=\"$namespace\", pod=~\"(loki|enterprise-logs)-write.*\"}", "instant": false, "intervalFactor": 3, "legendFormat": "{{pod}}", @@ -1987,7 +1987,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\"} | logfmt | level=\"error\"[1m]))", + "expr": "sum(rate({cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\"} | logfmt | level=\"error\"[1m]))", "refId": "A" } ], @@ -2050,7 +2050,7 @@ "panels": [], "targets": [ { - "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\"} |= \"level=error\"", + "expr": "{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\"} |= \"level=error\"", "refId": "A" } ], @@ -2101,7 +2101,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (route) > 0", + "expr": "sum(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (route) > 0", "interval": "", "intervalFactor": 1, "legendFormat": "{{route}}", @@ -2191,7 +2191,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_distributor_ingester_append_failures_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (pod)", + "expr": "sum(rate(loki_distributor_ingester_append_failures_total{cluster=~\"$cluster\", namespace=\"$namespace\"}[5m])) by (pod)", "intervalFactor": 1, "legendFormat": "{{pod}}", "refId": "A" @@ -2280,7 +2280,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_distributor_bytes_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (pod)", + "expr": "sum(rate(loki_distributor_bytes_received_total{cluster=~\"$cluster\", namespace=\"$namespace\"}[5m])) by (pod)", "intervalFactor": 1, "legendFormat": "{{pod}}", "refId": "A" @@ -2369,7 +2369,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_distributor_lines_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (pod)", + "expr": "sum(rate(loki_distributor_lines_received_total{cluster=~\"$cluster\", namespace=\"$namespace\"}[5m])) by (pod)", "intervalFactor": 1, "legendFormat": "{{pod}}", "refId": "A" @@ -2476,7 +2476,7 @@ "steppedLine": false, "targets": [ { - "expr": "topk(10,sum by (tenant) (loki_ingester_memory_streams{cluster=\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\"}))", + "expr": "topk(10,sum by (tenant) (loki_ingester_memory_streams{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\"}))", "interval": "", "legendFormat": "{{ tenant }}", "refId": "A" @@ -2567,7 +2567,7 @@ "steppedLine": false, "targets": [ { - "expr": "topk(10, sum by (tenant) (rate(loki_ingester_streams_created_total{cluster=\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m]) > 0))", + "expr": "topk(10, sum by (tenant) (rate(loki_ingester_streams_created_total{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m]) > 0))", "interval": "", "legendFormat": "{{ tenant }}", "refId": "A" @@ -2677,13 +2677,13 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m]))", + "expr": "sum(rate(loki_ingester_chunks_flushed_total{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m]))", "interval": "", "legendFormat": "Chunks", "refId": "A" }, { - "expr": "sum(increase(loki_chunk_store_deduped_chunks_total{cluster=\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m]))/sum(increase(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m])) < 1", + "expr": "sum(increase(loki_chunk_store_deduped_chunks_total{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m]))/sum(increase(loki_ingester_chunks_flushed_total{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m])) < 1", "interval": "", "legendFormat": "De-Dupe Ratio", "refId": "B" @@ -2761,7 +2761,7 @@ "reverseYBuckets": false, "targets": [ { - "expr": "sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m])) by (le)", + "expr": "sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m])) by (le)", "format": "heatmap", "instant": false, "interval": "", @@ -2916,7 +2916,7 @@ "reverseYBuckets": false, "targets": [ { - "expr": "sum by (le) (rate(loki_ingester_chunk_utilization_bucket{cluster=\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m]))", + "expr": "sum by (le) (rate(loki_ingester_chunk_utilization_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m]))", "format": "heatmap", "instant": false, "interval": "", @@ -3099,7 +3099,7 @@ "steppedLine": false, "targets": [ { - "expr": "go_memstats_heap_inuse_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"(loki|enterprise-logs)-read.*\"}", + "expr": "go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=\"$namespace\", pod=~\"(loki|enterprise-logs)-read.*\"}", "instant": false, "intervalFactor": 3, "legendFormat": "{{pod}}", @@ -3194,7 +3194,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-read\"} | logfmt | level=\"error\"[1m]))", + "expr": "sum(rate({cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-read\"} | logfmt | level=\"error\"[1m]))", "refId": "A" } ], @@ -3257,7 +3257,7 @@ "panels": [], "targets": [ { - "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-read\"} |= \"level=error\"", + "expr": "{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-read\"} |= \"level=error\"", "refId": "A" } ], @@ -3308,7 +3308,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-read\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-read\"}[$__rate_interval])) by (route) > 0", + "expr": "sum(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-read\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-read\"}[$__rate_interval])) by (route) > 0", "interval": "", "intervalFactor": 1, "legendFormat": "{{route}}", @@ -3417,19 +3417,19 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (method, name, le, container))", + "expr": "histogram_quantile(.99, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[5m])) by (method, name, le, container))", "intervalFactor": 1, "legendFormat": "{{container}}: .99-{{method}}-{{name}}", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (method, name, le, container))", + "expr": "histogram_quantile(.9, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[5m])) by (method, name, le, container))", "hide": false, "legendFormat": "{{container}}: .9-{{method}}-{{name}}", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (method, name, le, container))", + "expr": "histogram_quantile(.5, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[5m])) by (method, name, le, container))", "hide": false, "legendFormat": "{{container}}: .5-{{method}}-{{name}}", "refId": "C" @@ -3521,7 +3521,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_memcache_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, method, name, container)", + "expr": "sum(rate(loki_memcache_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, method, name, container)", "intervalFactor": 1, "legendFormat": "{{container}}: {{status_code}}-{{method}}-{{name}}", "refId": "A" @@ -3629,19 +3629,19 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", "intervalFactor": 1, "legendFormat": ".99-{{operation}}", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", "hide": false, "legendFormat": ".9-{{operation}}", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", "hide": false, "legendFormat": ".5-{{operation}}", "refId": "C" @@ -3733,7 +3733,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, status_code, method)", + "expr": "sum(rate(loki_consul_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, status_code, method)", "intervalFactor": 1, "legendFormat": "{{status_code}}-{{operation}}", "refId": "A" @@ -3839,17 +3839,17 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (operation, le))", "intervalFactor": 1, "legendFormat": ".9", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (operation, le))", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (operation, le))", "refId": "C" } ], @@ -3937,20 +3937,20 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (operation, le))", "interval": "", "intervalFactor": 1, "legendFormat": "99%", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (operation, le))", "interval": "", "legendFormat": "90%", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (operation, le))", "interval": "", "legendFormat": "50%", "refId": "C" @@ -4040,20 +4040,20 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (operation, le))", "interval": "", "intervalFactor": 1, "legendFormat": "99%", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (operation, le))", "interval": "", "legendFormat": "90%", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (operation, le))", "interval": "", "legendFormat": "50%", "refId": "C" @@ -4143,17 +4143,17 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (operation, le))", "intervalFactor": 1, "legendFormat": ".9", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (operation, le))", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (operation, le))", "refId": "C" } ], @@ -4241,7 +4241,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (status_code)", + "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (status_code)", "intervalFactor": 1, "legendFormat": "{{status_code}}", "refId": "A" @@ -4331,7 +4331,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (status_code)", + "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (status_code)", "intervalFactor": 1, "legendFormat": "{{status_code}}", "refId": "A" @@ -4421,7 +4421,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (status_code)", + "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (status_code)", "intervalFactor": 1, "legendFormat": "{{status_code}}", "refId": "A" @@ -4511,7 +4511,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (status_code)", + "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (status_code)", "intervalFactor": 1, "legendFormat": "{{status_code}}", "refId": "A" @@ -4619,19 +4619,19 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", "intervalFactor": 1, "legendFormat": ".99-{{operation}}", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", "hide": false, "legendFormat": ".9-{{operation}}", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", "hide": false, "legendFormat": ".5-{{operation}}", "refId": "C" @@ -4723,7 +4723,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_gcs_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", + "expr": "sum(rate(loki_gcs_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", "intervalFactor": 1, "legendFormat": "{{status_code}}-{{operation}}", "refId": "A" @@ -4827,7 +4827,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_dynamo_failures_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))", + "expr": "sum(rate(loki_dynamo_failures_total{cluster=~\"$cluster\", namespace=\"$namespace\"}[5m]))", "refId": "A" } ], @@ -4913,7 +4913,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_dynamo_consumed_capacity_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))", + "expr": "sum(rate(loki_dynamo_consumed_capacity_total{cluster=~\"$cluster\", namespace=\"$namespace\"}[5m]))", "refId": "A" } ], @@ -4999,7 +4999,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_dynamo_throttled_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))", + "expr": "sum(rate(loki_dynamo_throttled_total{cluster=~\"$cluster\", namespace=\"$namespace\"}[5m]))", "refId": "A" } ], @@ -5085,7 +5085,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_dynamo_dropped_requests_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))", + "expr": "sum(rate(loki_dynamo_dropped_requests_total{cluster=~\"$cluster\", namespace=\"$namespace\"}[5m]))", "refId": "A" } ], @@ -5171,17 +5171,17 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])))", + "expr": "histogram_quantile(.99, sum(rate(loki_dynamo_query_pages_count{cluster=~\"$cluster\", namespace=\"$namespace\"}[5m])))", "legendFormat": ".99", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])))", + "expr": "histogram_quantile(.9, sum(rate(loki_dynamo_query_pages_count{cluster=~\"$cluster\", namespace=\"$namespace\"}[5m])))", "legendFormat": ".9", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])))", + "expr": "histogram_quantile(.5, sum(rate(loki_dynamo_query_pages_count{cluster=~\"$cluster\", namespace=\"$namespace\"}[5m])))", "legendFormat": ".5", "refId": "C" } @@ -5271,19 +5271,19 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(loki_dynamo_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", "intervalFactor": 1, "legendFormat": ".99-{{operation}}", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(loki_dynamo_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", "hide": false, "legendFormat": ".9-{{operation}}", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(loki_dynamo_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", "hide": false, "legendFormat": ".5-{{operation}}", "refId": "C" @@ -5374,7 +5374,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_dynamo_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", + "expr": "sum(rate(loki_dynamo_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", "intervalFactor": 1, "legendFormat": "{{status_code}}-{{operation}}", "refId": "A" @@ -5481,19 +5481,19 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", "intervalFactor": 1, "legendFormat": ".99-{{operation}}", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", "hide": false, "legendFormat": ".9-{{operation}}", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", "hide": false, "legendFormat": ".5-{{operation}}", "refId": "C" @@ -5584,7 +5584,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_s3_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", + "expr": "sum(rate(loki_s3_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", "intervalFactor": 1, "legendFormat": "{{status_code}}-{{operation}}", "refId": "A" @@ -5691,19 +5691,19 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", "intervalFactor": 1, "legendFormat": ".99-{{operation}}", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", "hide": false, "legendFormat": ".9-{{operation}}", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", "hide": false, "legendFormat": ".5-{{operation}}", "refId": "C" @@ -5794,7 +5794,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_azure_blob_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", + "expr": "sum(rate(loki_azure_blob_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", "intervalFactor": 1, "legendFormat": "{{status_code}}-{{operation}}", "refId": "A" @@ -5901,19 +5901,19 @@ "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", "intervalFactor": 1, "legendFormat": ".99-{{operation}}", "refId": "A" }, { - "expr": "histogram_quantile(.9, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.9, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", "hide": false, "legendFormat": ".9-{{operation}}", "refId": "B" }, { - "expr": "histogram_quantile(.5, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", + "expr": "histogram_quantile(.5, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=~\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", "hide": false, "legendFormat": ".5-{{operation}}", "refId": "C" @@ -6004,7 +6004,7 @@ "steppedLine": false, "targets": [ { - "expr": "sum(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", + "expr": "sum(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=~\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", "intervalFactor": 1, "legendFormat": "{{status_code}}-{{operation}}", "refId": "A" @@ -6074,7 +6074,7 @@ "value": "default" }, "hide": 0, - "label": "Data Source", + "label": "Data source", "name": "datasource", "options": [], "query": "prometheus", @@ -6128,7 +6128,7 @@ "multi": false, "name": "namespace", "options": [], - "query": "label_values(loki_build_info{cluster_id=~\"$cluster_id\"}, namespace)", + "query": "label_values(loki_build_info{cluster_id=~\"$cluster\"}, namespace)", "refresh": 1, "regex": "", "sort": 2, diff --git a/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-reads-resources.json b/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-reads-resources.json index f5ad2ce8..e59a9492 100644 --- a/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-reads-resources.json +++ b/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-reads-resources.json @@ -29,514 +29,397 @@ "collapsed": false, "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFC000", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, "gridPos": {}, "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "request", - "color": "#FFC000", - "fill": 0 + "options": { + "legend": { + "showLegend": true }, - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 + "tooltip": { + "mode": "single", + "sort": "none" } - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + }, "targets": [ { - "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster_id=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null }, { - "expr": "min(kube_pod_container_resource_requests{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\", resource=\"cpu\"} > 0)", + "expr": "min(kube_pod_container_resource_requests{cluster_id=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\", resource=\"cpu\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "request", - "legendLink": null, - "step": 10 + "legendLink": null }, { - "expr": "min(container_spec_cpu_quota{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"} / container_spec_cpu_period{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"})", + "expr": "min(container_spec_cpu_quota{cluster_id=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"} / container_spec_cpu_period{cluster_id=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "limit", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "CPU", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFC000", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, "gridPos": {}, "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "request", - "color": "#FFC000", - "fill": 0 + "options": { + "legend": { + "showLegend": true }, - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 + "tooltip": { + "mode": "single", + "sort": "none" } - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + }, "targets": [ { - "expr": "max by(pod) (container_memory_working_set_bytes{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"})", + "expr": "max by(pod) (container_memory_working_set_bytes{cluster_id=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null }, { - "expr": "min(kube_pod_container_resource_requests{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\", resource=\"memory\"} > 0)", + "expr": "min(kube_pod_container_resource_requests{cluster_id=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\", resource=\"memory\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "request", - "legendLink": null, - "step": 10 + "legendLink": null }, { - "expr": "min(container_spec_memory_limit_bytes{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"} > 0)", + "expr": "min(container_spec_memory_limit_bytes{cluster_id=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "limit", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Memory (workingset)", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "bytes" + }, + "overrides": [] + }, "gridPos": {}, "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "targets": [ { - "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster_id=~\"$cluster_id\", job=~\"($namespace)/(loki|enterprise-logs)-read\"})", + "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster_id=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-read\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Memory (go heap inuse)", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 10, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "Bps" + }, + "overrides": [] + }, "gridPos": {}, "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "targets": [ { - "expr": "sum by(instance, pod, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", + "expr": "sum by(instance, pod, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster_id=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}} - {{device}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Disk Writes", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 10, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "Bps" + }, + "overrides": [] + }, "gridPos": {}, "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "targets": [ { - "expr": "sum by(instance, pod, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", + "expr": "sum by(instance, pod, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster_id=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}} - {{device}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Disk Reads", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "percentunit" + }, + "overrides": [] + }, "gridPos": {}, "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "targets": [ { - "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\"}) and count by(persistentvolumeclaim) (kube_persistentvolumeclaim_labels{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\",label_name=~\"(loki|enterprise-logs)-read.*\"})", + "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster_id=~\"$cluster\", namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{cluster_id=~\"$cluster\", namespace=~\"$namespace\"}) and count by(persistentvolumeclaim) (kube_persistentvolumeclaim_labels{cluster_id=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"(loki|enterprise-logs)-read.*\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{persistentvolumeclaim}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Disk Space Utilization", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -552,280 +435,256 @@ "height": "250px", "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFC000", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] }, - "lines": true, - "linewidth": 1, + "id": 7, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "request", - "color": "#FFC000", - "fill": 0 + "options": { + "legend": { + "showLegend": true }, - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 + "tooltip": { + "mode": "single", + "sort": "none" } - ], - "spaceLength": 10, + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster_id=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null }, { - "expr": "min(kube_pod_container_resource_requests{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", resource=\"cpu\"} > 0)", + "expr": "min(kube_pod_container_resource_requests{cluster_id=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", resource=\"cpu\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "request", - "legendLink": null, - "step": 10 + "legendLink": null }, { - "expr": "min(container_spec_cpu_quota{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"} / container_spec_cpu_period{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"})", + "expr": "min(container_spec_cpu_quota{cluster_id=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"} / container_spec_cpu_period{cluster_id=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "limit", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "CPU", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 8, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFC000", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] }, - "lines": true, - "linewidth": 1, + "id": 8, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "request", - "color": "#FFC000", - "fill": 0 + "options": { + "legend": { + "showLegend": true }, - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 + "tooltip": { + "mode": "single", + "sort": "none" } - ], - "spaceLength": 10, + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "max by(pod) (container_memory_working_set_bytes{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"})", + "expr": "max by(pod) (container_memory_working_set_bytes{cluster_id=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null }, { - "expr": "min(kube_pod_container_resource_requests{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", resource=\"memory\"} > 0)", + "expr": "min(kube_pod_container_resource_requests{cluster_id=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", resource=\"memory\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "request", - "legendLink": null, - "step": 10 + "legendLink": null }, { - "expr": "min(container_spec_memory_limit_bytes{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"} > 0)", + "expr": "min(container_spec_memory_limit_bytes{cluster_id=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "limit", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Memory (workingset)", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "bytes" + }, + "overrides": [] }, - "lines": true, - "linewidth": 1, + "id": 9, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster_id=~\"$cluster_id\", job=~\"($namespace)/(loki|enterprise-logs)-write\"})", + "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster_id=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Memory (go heap inuse)", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -851,7 +710,7 @@ "value": "default" }, "hide": 0, - "label": "Data Source", + "label": "Data source", "name": "datasource", "options": [], "query": "prometheus", @@ -895,7 +754,7 @@ "multi": false, "name": "namespace", "options": [], - "query": "label_values(loki_build_info{cluster_id=~\"$cluster_id\"}, namespace)", + "query": "label_values(loki_build_info{cluster_id=~\"$cluster\"}, namespace)", "refresh": 1, "regex": "", "sort": 2, diff --git a/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-reads.json b/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-reads.json index c5e3693d..f0796ef7 100644 --- a/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-reads.json +++ b/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-reads.json @@ -35,156 +35,256 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{cluster_id=~\"$cluster_id\",job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{cluster_id=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "QPS", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "ms" + }, + "overrides": [] }, - "lines": true, - "linewidth": 1, + "id": 2, "links": [], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster_id=~\"$cluster_id\", job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\"})) * 1e3", + "expr": "histogram_quantile(0.99, sum by (le,route) (cluster_id_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster_id=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ route }} 99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "{{ route }} 99th percentile", + "refId": "A" }, { - "expr": "histogram_quantile(0.50, sum by (le,route) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster_id=~\"$cluster_id\", job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\"})) * 1e3", + "expr": "histogram_quantile(0.50, sum by (le,route) (cluster_id_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster_id=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "{{ route }} 50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "{{ route }} 50th percentile", + "refId": "B" }, { - "expr": "1e3 * sum(cluster_job_route:loki_request_duration_seconds_sum:sum_rate{cluster_id=~\"$cluster_id\", job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\"}) by (route) / sum(cluster_job_route:loki_request_duration_seconds_count:sum_rate{cluster_id=~\"$cluster_id\", job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\"}) by (route) ", + "expr": "1e3 * sum(cluster_id_job_route:loki_request_duration_seconds_sum:sum_rate{cluster_id=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\"}) by (route) / sum(cluster_id_job_route:loki_request_duration_seconds_count:sum_rate{cluster_id=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\"}) by (route) ", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{ route }} Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -205,38 +305,45 @@ ] }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "ms" + }, + "overrides": [] }, - "lines": true, - "linewidth": 1, + "id": 3, "links": [], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket{cluster_id=~\"$cluster_id\", job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\"}[$__rate_interval])) by (le,pod)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(loki_request_duration_seconds_bucket{cluster_id=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\"}[$__rate_interval])) by (le,pod)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -245,41 +352,8 @@ "step": 10 } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Per Pod Latency (p99)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -300,156 +374,256 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster_id=~\"$cluster_id\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster_id=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "QPS", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "ms" + }, + "overrides": [] }, - "lines": true, - "linewidth": 1, + "id": 5, "links": [], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster_id=~\"$cluster_id\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster_id=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { - "expr": "histogram_quantile(0.50, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster_id=~\"$cluster_id\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster_id=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { - "expr": "sum(rate(loki_boltdb_shipper_request_duration_seconds_sum{cluster_id=~\"$cluster_id\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval])) * 1e3 / sum(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster_id=~\"$cluster_id\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval]))", + "expr": "sum(rate(loki_boltdb_shipper_request_duration_seconds_sum{cluster_id=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval])) * 1e3 / sum(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster_id=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -470,38 +644,45 @@ ] }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "ms" + }, + "overrides": [] }, - "lines": true, - "linewidth": 1, + "id": 6, "links": [], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster_id=~\"$cluster_id\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval])) by (le,pod)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster_id=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval])) by (le,pod)) * 1e3", "format": "time_series", "interval": "1m", "intervalFactor": 2, @@ -510,41 +691,8 @@ "step": 10 } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Per Pod Latency (p99)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "ms", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -570,7 +718,7 @@ "value": "default" }, "hide": 0, - "label": "Data Source", + "label": "Data source", "name": "datasource", "options": [], "query": "prometheus", @@ -614,7 +762,7 @@ "multi": false, "name": "namespace", "options": [], - "query": "label_values(loki_build_info{cluster_id=~\"$cluster_id\"}, namespace)", + "query": "label_values(loki_build_info{cluster_id=~\"$cluster\"}, namespace)", "refresh": 1, "regex": "", "sort": 2, diff --git a/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-retention.json b/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-retention.json index 1fe50933..a9f0fbc0 100644 --- a/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-retention.json +++ b/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-retention.json @@ -29,280 +29,256 @@ "height": "250px", "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFC000", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] }, - "lines": true, - "linewidth": 1, + "id": 1, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "request", - "color": "#FFC000", - "fill": 0 + "options": { + "legend": { + "showLegend": true }, - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 + "tooltip": { + "mode": "single", + "sort": "none" } - ], - "spaceLength": 10, + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster_id=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null }, { - "expr": "min(kube_pod_container_resource_requests{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\", resource=\"cpu\"} > 0)", + "expr": "min(kube_pod_container_resource_requests{cluster_id=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\", resource=\"cpu\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "request", - "legendLink": null, - "step": 10 + "legendLink": null }, { - "expr": "min(container_spec_cpu_quota{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"} / container_spec_cpu_period{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"})", + "expr": "min(container_spec_cpu_quota{cluster_id=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"} / container_spec_cpu_period{cluster_id=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "limit", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "CPU", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFC000", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] }, - "lines": true, - "linewidth": 1, + "id": 2, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "request", - "color": "#FFC000", - "fill": 0 + "options": { + "legend": { + "showLegend": true }, - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 + "tooltip": { + "mode": "single", + "sort": "none" } - ], - "spaceLength": 10, + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "max by(pod) (container_memory_working_set_bytes{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"})", + "expr": "max by(pod) (container_memory_working_set_bytes{cluster_id=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null }, { - "expr": "min(kube_pod_container_resource_requests{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\", resource=\"memory\"} > 0)", + "expr": "min(kube_pod_container_resource_requests{cluster_id=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\", resource=\"memory\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "request", - "legendLink": null, - "step": 10 + "legendLink": null }, { - "expr": "min(container_spec_memory_limit_bytes{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"} > 0)", + "expr": "min(container_spec_memory_limit_bytes{cluster_id=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "limit", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Memory (workingset)", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "bytes" + }, + "overrides": [] }, - "lines": true, - "linewidth": 1, + "id": 3, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster_id=~\"$cluster_id\", job=~\"($namespace)/(loki|enterprise-logs)-read\"})", + "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster_id=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-read\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Memory (go heap inuse)", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -382,7 +358,7 @@ "steppedLine": false, "targets": [ { - "expr": "loki_boltdb_shipper_compact_tables_operation_last_successful_run_timestamp_seconds{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\"} * 1e3", + "expr": "loki_boltdb_shipper_compact_tables_operation_last_successful_run_timestamp_seconds{cluster_id=~\"$cluster\", namespace=~\"$namespace\"} * 1e3", "format": "time_series", "instant": true, "refId": "A" @@ -425,80 +401,51 @@ ] }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "s" + }, + "overrides": [] }, - "lines": true, - "linewidth": 1, + "id": 5, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "loki_boltdb_shipper_compact_tables_operation_duration_seconds{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\"}", + "expr": "loki_boltdb_shipper_compact_tables_operation_duration_seconds{cluster_id=~\"$cluster\", namespace=~\"$namespace\"}", "format": "time_series", - "intervalFactor": 2, "legendFormat": "duration", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Compact Tables Operations Duration", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -513,156 +460,98 @@ "height": "250px", "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [] }, - "lines": true, - "linewidth": 1, + "id": 6, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "sum(increase(loki_compactor_skipped_compacting_locked_table_total{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\"}[$__range]))", + "expr": "sum(increase(loki_compactor_skipped_compacting_locked_table_total{cluster_id=~\"$cluster\", namespace=~\"$namespace\"}[$__range]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{table_name}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Number of times Tables were skipped during Compaction", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [] }, - "lines": true, - "linewidth": 1, + "id": 7, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "sum by (status)(rate(loki_boltdb_shipper_compact_tables_operation_total{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\"}[$__rate_interval]))", + "expr": "sum by (status)(rate(loki_boltdb_shipper_compact_tables_operation_total{cluster_id=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{success}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Compact Tables Operations Per Status", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -742,7 +631,7 @@ "steppedLine": false, "targets": [ { - "expr": "loki_compactor_apply_retention_last_successful_run_timestamp_seconds{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\"} * 1e3", + "expr": "loki_compactor_apply_retention_last_successful_run_timestamp_seconds{cluster_id=~\"$cluster\", namespace=~\"$namespace\"} * 1e3", "format": "time_series", "instant": true, "refId": "A" @@ -785,156 +674,98 @@ ] }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 9, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "s" + }, + "overrides": [] }, - "lines": true, - "linewidth": 1, + "id": 9, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "loki_compactor_apply_retention_operation_duration_seconds{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\"}", + "expr": "loki_compactor_apply_retention_operation_duration_seconds{cluster_id=~\"$cluster\", namespace=~\"$namespace\"}", "format": "time_series", - "intervalFactor": 2, "legendFormat": "duration", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Mark Operations Duration", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 10, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [] }, - "lines": true, - "linewidth": 1, + "id": 10, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "sum by (status)(rate(loki_compactor_apply_retention_operation_total{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\"}[$__rate_interval]))", + "expr": "sum by (status)(rate(loki_compactor_apply_retention_operation_total{cluster_id=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{success}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Mark Operations Per Status", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -949,232 +780,145 @@ "height": "250px", "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 10, - "id": 11, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [] }, - "lines": true, - "linewidth": 0, + "id": 11, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": true, - "steppedLine": false, "targets": [ { - "expr": "count by(action)(loki_boltdb_shipper_retention_marker_table_processed_total{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\"})", + "expr": "count by(action)(loki_boltdb_shipper_retention_marker_table_processed_total{cluster_id=~\"$cluster\", namespace=~\"$namespace\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{action}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Processed Tables Per Action", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 10, - "id": 12, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [] }, - "lines": true, - "linewidth": 0, + "id": 12, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": true, - "steppedLine": false, "targets": [ { - "expr": "count by(table,action)(loki_boltdb_shipper_retention_marker_table_processed_total{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\" , action=~\"modified|deleted\"})", + "expr": "count by(table,action)(loki_boltdb_shipper_retention_marker_table_processed_total{cluster_id=~\"$cluster\", namespace=~\"$namespace\" , action=~\"modified|deleted\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{table}}-{{action}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Modified Tables", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 10, - "id": 13, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [] }, - "lines": true, - "linewidth": 0, + "id": 13, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (table)(rate(loki_boltdb_shipper_retention_marker_count_total{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\"}[$__rate_interval])) >0", + "expr": "sum by (table)(rate(loki_boltdb_shipper_retention_marker_count_total{cluster_id=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) >0", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{table}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Marks Creation Rate Per Table", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -1189,154 +933,113 @@ "height": "250px", "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [] + }, "format": "short", "id": 14, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "sum (increase(loki_boltdb_shipper_retention_marker_count_total{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\"}[24h]))", + "expr": "sum (increase(loki_boltdb_shipper_retention_marker_count_total{cluster_id=~\"$cluster\", namespace=~\"$namespace\"}[24h]))", "format": "time_series", "instant": true, - "intervalFactor": 2, "refId": "A" } ], "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, "title": "Marked Chunks (24h)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "singlestat" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 15, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [] }, - "lines": true, - "linewidth": 1, + "id": 15, "links": [], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_retention_marker_table_processed_duration_seconds_bucket{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_retention_marker_table_processed_duration_seconds_bucket{cluster_id=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { - "expr": "histogram_quantile(0.50, sum(rate(loki_boltdb_shipper_retention_marker_table_processed_duration_seconds_bucket{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(loki_boltdb_shipper_retention_marker_table_processed_duration_seconds_bucket{cluster_id=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { - "expr": "sum(rate(loki_boltdb_shipper_retention_marker_table_processed_duration_seconds_sum{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\"}[$__rate_interval])) * 1e3 / sum(rate(loki_boltdb_shipper_retention_marker_table_processed_duration_seconds_count{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\"}[$__rate_interval]))", + "expr": "sum(rate(loki_boltdb_shipper_retention_marker_table_processed_duration_seconds_sum{cluster_id=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) * 1e3 / sum(rate(loki_boltdb_shipper_retention_marker_table_processed_duration_seconds_count{cluster_id=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Mark Table Latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1369,154 +1072,113 @@ "height": "250px", "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [] + }, "format": "short", "id": 16, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "sum (increase(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_count{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\"}[24h]))", + "expr": "sum (increase(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_count{cluster_id=~\"$cluster\", namespace=~\"$namespace\"}[24h]))", "format": "time_series", - "instant": true, - "intervalFactor": 2, - "refId": "A" - } - ], - "thresholds": "70,80", - "timeFrom": null, - "timeShift": null, - "title": "Delete Chunks (24h)", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "singlestat", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] - }, - { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "$datasource", - "fill": 1, - "id": 17, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "instant": true, + "refId": "A" + } + ], + "thresholds": "70,80", + "title": "Delete Chunks (24h)", + "type": "singlestat" + }, + { + "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [] }, - "lines": true, - "linewidth": 1, + "id": 17, "links": [], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_bucket{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_bucket{cluster_id=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { - "expr": "histogram_quantile(0.50, sum(rate(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_bucket{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_bucket{cluster_id=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { - "expr": "sum(rate(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_sum{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\"}[$__rate_interval])) * 1e3 / sum(rate(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_count{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\"}[$__rate_interval]))", + "expr": "sum(rate(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_sum{cluster_id=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval])) * 1e3 / sum(rate(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_count{cluster_id=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Delete Latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -1549,232 +1211,145 @@ "height": "250px", "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 18, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "s" + }, + "overrides": [] }, - "lines": true, - "linewidth": 1, + "id": 18, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "time() - (loki_boltdb_shipper_retention_sweeper_marker_file_processing_current_time{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\"} > 0)", + "expr": "time() - (loki_boltdb_shipper_retention_sweeper_marker_file_processing_current_time{cluster_id=~\"$cluster\", namespace=~\"$namespace\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "lag", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Sweeper Lag", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "s", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 19, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [] }, - "lines": true, - "linewidth": 1, + "id": 19, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "sum(loki_boltdb_shipper_retention_sweeper_marker_files_current{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\"})", + "expr": "sum(loki_boltdb_shipper_retention_sweeper_marker_files_current{cluster_id=~\"$cluster\", namespace=~\"$namespace\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "count", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Marks Files to Process", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 20, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [] }, - "lines": true, - "linewidth": 1, + "id": 20, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 4, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "sum by (status)(rate(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_count{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\"}[$__rate_interval]))", + "expr": "sum by (status)(rate(loki_boltdb_shipper_retention_sweeper_chunk_deleted_duration_seconds_count{cluster_id=~\"$cluster\", namespace=~\"$namespace\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{status}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Delete Rate Per Status", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -1794,7 +1369,7 @@ "span": 12, "targets": [ { - "expr": "{cluster_id=~\"$cluster_id\", job=~\"($namespace)/(loki|enterprise-logs)-read\"}", + "expr": "{cluster_id=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-read\"}", "refId": "A" } ], @@ -1825,7 +1400,7 @@ "value": "default" }, "hide": 0, - "label": "Data Source", + "label": "Data source", "name": "datasource", "options": [], "query": "prometheus", @@ -1869,7 +1444,7 @@ "multi": false, "name": "namespace", "options": [], - "query": "label_values(loki_build_info{cluster_id=~\"$cluster_id\"}, namespace)", + "query": "label_values(loki_build_info{cluster_id=~\"$cluster\"}, namespace)", "refresh": 1, "regex": "", "sort": 2, diff --git a/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-writes-resources.json b/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-writes-resources.json index 6bda3d2e..c3f25f4e 100644 --- a/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-writes-resources.json +++ b/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-writes-resources.json @@ -29,589 +29,447 @@ "collapsed": false, "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [] + }, "gridPos": {}, "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "targets": [ { - "expr": "sum by(pod) (loki_ingester_memory_streams{cluster_id=~\"$cluster_id\", job=~\"($namespace)/(loki|enterprise-logs)-write\"})", + "expr": "sum by(pod) (loki_ingester_memory_streams{cluster_id=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "In-memory streams", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFC000", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, "gridPos": {}, "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "request", - "color": "#FFC000", - "fill": 0 + "options": { + "legend": { + "showLegend": true }, - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 + "tooltip": { + "mode": "single", + "sort": "none" } - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + }, "targets": [ { - "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"}[$__rate_interval]))", + "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster_id=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null }, { - "expr": "min(kube_pod_container_resource_requests{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", resource=\"cpu\"} > 0)", + "expr": "min(kube_pod_container_resource_requests{cluster_id=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", resource=\"cpu\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "request", - "legendLink": null, - "step": 10 + "legendLink": null }, { - "expr": "min(container_spec_cpu_quota{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"} / container_spec_cpu_period{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"})", + "expr": "min(container_spec_cpu_quota{cluster_id=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"} / container_spec_cpu_period{cluster_id=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "limit", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "CPU", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "bytes" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "request" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#FFC000", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "limit" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E02F44", + "mode": "fixed" + } + }, + { + "id": "custom.fillOpacity", + "value": 0 + } + ] + } + ] + }, "gridPos": {}, "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [ - { - "alias": "request", - "color": "#FFC000", - "fill": 0 + "options": { + "legend": { + "showLegend": true }, - { - "alias": "limit", - "color": "#E02F44", - "fill": 0 + "tooltip": { + "mode": "single", + "sort": "none" } - ], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + }, "targets": [ { - "expr": "max by(pod) (container_memory_working_set_bytes{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"})", + "expr": "max by(pod) (container_memory_working_set_bytes{cluster_id=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null }, { - "expr": "min(kube_pod_container_resource_requests{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", resource=\"memory\"} > 0)", + "expr": "min(kube_pod_container_resource_requests{cluster_id=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", resource=\"memory\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "request", - "legendLink": null, - "step": 10 + "legendLink": null }, { - "expr": "min(container_spec_memory_limit_bytes{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"} > 0)", + "expr": "min(container_spec_memory_limit_bytes{cluster_id=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"} > 0)", "format": "time_series", - "intervalFactor": 2, "legendFormat": "limit", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Memory (workingset)", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "bytes" + }, + "overrides": [] + }, "gridPos": {}, "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "targets": [ { - "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster_id=~\"$cluster_id\", job=~\"($namespace)/(loki|enterprise-logs)-write\"})", + "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster_id=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Memory (go heap inuse)", "tooltip": { "sort": 2 }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "bytes", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 10, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "Bps" + }, + "overrides": [] + }, "gridPos": {}, "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "targets": [ { - "expr": "sum by(instance, pod, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", + "expr": "sum by(instance, pod, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster_id=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}} - {{device}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Disk Writes", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 10, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "Bps" + }, + "overrides": [] + }, "gridPos": {}, "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 0, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 6, - "stack": true, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "targets": [ { - "expr": "sum by(instance, pod, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", + "expr": "sum by(instance, pod, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster_id=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{pod}} - {{device}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Disk Reads", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "Bps", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "percentunit" + }, + "overrides": [] + }, "gridPos": {}, "id": 7, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, - "span": 6, - "stack": false, - "steppedLine": false, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "targets": [ { - "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\"}) and count by(persistentvolumeclaim) (kube_persistentvolumeclaim_labels{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\",label_name=~\"(loki|enterprise-logs)-write.*\"})", + "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster_id=~\"$cluster\", namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{cluster_id=~\"$cluster\", namespace=~\"$namespace\"}) and count by(persistentvolumeclaim) (kube_persistentvolumeclaim_labels{cluster_id=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"(loki|enterprise-logs)-write.*\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{persistentvolumeclaim}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Disk Space Utilization", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "percentunit", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" } ], "repeat": null, @@ -638,7 +496,7 @@ "value": "default" }, "hide": 0, - "label": "Data Source", + "label": "Data source", "name": "datasource", "options": [], "query": "prometheus", @@ -682,7 +540,7 @@ "multi": false, "name": "namespace", "options": [], - "query": "label_values(loki_build_info{cluster_id=~\"$cluster_id\"}, namespace)", + "query": "label_values(loki_build_info{cluster_id=~\"$cluster\"}, namespace)", "refresh": 1, "regex": "", "sort": 2, diff --git a/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-writes.json b/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-writes.json index 00e98564..0e48dc59 100644 --- a/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-writes.json +++ b/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-writes.json @@ -35,156 +35,256 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 1, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{cluster_id=~\"$cluster_id\",job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"api_prom_push|loki_api_v1_push|/httpgrpc.HTTP/Handle\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{cluster_id=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"api_prom_push|loki_api_v1_push|/httpgrpc.HTTP/Handle\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "QPS", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 2, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "ms" + }, + "overrides": [] }, - "lines": true, - "linewidth": 1, + "id": 2, "links": [], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum by (le) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster_id=~\"$cluster_id\", job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"api_prom_push|loki_api_v1_push|/httpgrpc.HTTP/Handle\"})) * 1e3", + "expr": "histogram_quantile(0.99, sum by (le) (cluster_id_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster_id=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"api_prom_push|loki_api_v1_push|/httpgrpc.HTTP/Handle\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "legendFormat": "99th percentile", + "refId": "A" }, { - "expr": "histogram_quantile(0.50, sum by (le) (cluster_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster_id=~\"$cluster_id\", job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"api_prom_push|loki_api_v1_push|/httpgrpc.HTTP/Handle\"})) * 1e3", + "expr": "histogram_quantile(0.50, sum by (le) (cluster_id_job_route:loki_request_duration_seconds_bucket:sum_rate{cluster_id=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"api_prom_push|loki_api_v1_push|/httpgrpc.HTTP/Handle\"})) * 1e3", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "legendFormat": "50th percentile", + "refId": "B" }, { - "expr": "1e3 * sum(cluster_job_route:loki_request_duration_seconds_sum:sum_rate{cluster_id=~\"$cluster_id\", job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"api_prom_push|loki_api_v1_push|/httpgrpc.HTTP/Handle\"}) / sum(cluster_job_route:loki_request_duration_seconds_count:sum_rate{cluster_id=~\"$cluster_id\", job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"api_prom_push|loki_api_v1_push|/httpgrpc.HTTP/Handle\"})", + "expr": "1e3 * sum(cluster_id_job_route:loki_request_duration_seconds_sum:sum_rate{cluster_id=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"api_prom_push|loki_api_v1_push|/httpgrpc.HTTP/Handle\"}) / sum(cluster_id_job_route:loki_request_duration_seconds_count:sum_rate{cluster_id=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"api_prom_push|loki_api_v1_push|/httpgrpc.HTTP/Handle\"})", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -217,138 +317,99 @@ "height": "250px", "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 3, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [] }, - "lines": true, - "linewidth": 1, + "id": 3, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "sum (rate(loki_distributor_structured_metadata_bytes_received_total{cluster_id=~\"$cluster_id\",job=~\"($namespace)/(loki|enterprise-logs)-write\",}[$__rate_interval])) / sum(rate(loki_distributor_bytes_received_total{cluster_id=~\"$cluster_id\",job=~\"($namespace)/(loki|enterprise-logs)-write\",}[$__rate_interval]))", + "expr": "sum (rate(loki_distributor_structured_metadata_bytes_received_total{cluster_id=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\",}[$__rate_interval])) / sum(rate(loki_distributor_bytes_received_total{cluster_id=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\",}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "bytes", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Per Total Received Bytes", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 4, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [] }, - "lines": true, - "linewidth": 1, + "id": 4, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (tenant) (rate(loki_distributor_structured_metadata_bytes_received_total{cluster_id=~\"$cluster_id\",job=~\"($namespace)/(loki|enterprise-logs)-write\",}[$__rate_interval])) / ignoring(tenant) group_left sum(rate(loki_distributor_structured_metadata_bytes_received_total{cluster_id=~\"$cluster_id\",job=~\"($namespace)/(loki|enterprise-logs)-write\",}[$__rate_interval]))", + "expr": "sum by (tenant) (rate(loki_distributor_structured_metadata_bytes_received_total{cluster_id=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\",}[$__rate_interval])) / ignoring(tenant) group_left sum(rate(loki_distributor_structured_metadata_bytes_received_total{cluster_id=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\",}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{tenant}}", - "legendLink": null, - "step": 10 + "legendLink": null } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Per Tenant", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, + "type": "timeseries", "yaxes": [ { "format": "short", @@ -387,156 +448,256 @@ "3xx": "#6ED0E0", "4xx": "#EF843C", "5xx": "#E24D42", + "OK": "#7EB26D", + "cancel": "#A9A9A9", "error": "#E24D42", "success": "#7EB26D" }, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 100, + "lineWidth": 0, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "normal" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "short" + }, + "overrides": [ + { + "matcher": { + "id": "byName", + "options": "1xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EAB839", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "2xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "3xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#6ED0E0", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "4xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#EF843C", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "5xx" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "OK" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "cancel" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#A9A9A9", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "error" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#E24D42", + "mode": "fixed" + } + } + ] + }, + { + "matcher": { + "id": "byName", + "options": "success" + }, + "properties": [ + { + "id": "color", + "value": { + "fixedColor": "#7EB26D", + "mode": "fixed" + } + } + ] + } + ] + }, "fill": 10, "id": 5, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, "linewidth": 0, "links": [], - "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, "stack": true, - "steppedLine": false, "targets": [ { - "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster_id=~\"$cluster_id\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", + "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster_id=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-zA-Z]+)\"))\n", "format": "time_series", - "intervalFactor": 2, "legendFormat": "{{status}}", - "refId": "A", - "step": 10 + "refId": "A" } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "QPS", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, - "yaxes": [ - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": 0, - "show": true - }, - { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": false - } - ] + "type": "timeseries" }, { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, "datasource": "$datasource", - "fill": 1, - "id": 6, - "legend": { - "avg": false, - "current": false, - "max": false, - "min": false, - "show": true, - "total": false, - "values": false + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 10, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "thresholds": { + "mode": "absolute", + "steps": [] + }, + "unit": "ms" + }, + "overrides": [] }, - "lines": true, - "linewidth": 1, + "id": 6, "links": [], "nullPointMode": "null as zero", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "seriesOverrides": [], - "spaceLength": 10, + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, "span": 6, - "stack": false, - "steppedLine": false, "targets": [ { - "expr": "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster_id=~\"$cluster_id\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster_id=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", - "intervalFactor": 2, "legendFormat": "99th Percentile", - "refId": "A", - "step": 10 + "refId": "A" }, { - "expr": "histogram_quantile(0.50, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster_id=~\"$cluster_id\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval])) by (le)) * 1e3", + "expr": "histogram_quantile(0.50, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster_id=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", - "intervalFactor": 2, "legendFormat": "50th Percentile", - "refId": "B", - "step": 10 + "refId": "B" }, { - "expr": "sum(rate(loki_boltdb_shipper_request_duration_seconds_sum{cluster_id=~\"$cluster_id\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval])) * 1e3 / sum(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster_id=~\"$cluster_id\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval]))", + "expr": "sum(rate(loki_boltdb_shipper_request_duration_seconds_sum{cluster_id=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval])) * 1e3 / sum(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster_id=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", - "refId": "C", - "step": 10 + "refId": "C" } ], - "thresholds": [], - "timeFrom": null, - "timeShift": null, "title": "Latency", - "tooltip": { - "shared": true, - "sort": 2, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] - }, + "type": "timeseries", "yaxes": [ { "format": "ms", @@ -580,7 +741,7 @@ "value": "default" }, "hide": 0, - "label": "Data Source", + "label": "Data source", "name": "datasource", "options": [], "query": "prometheus", @@ -624,7 +785,7 @@ "multi": false, "name": "namespace", "options": [], - "query": "label_values(loki_build_info{cluster_id=~\"$cluster_id\"}, namespace)", + "query": "label_values(loki_build_info{cluster_id=~\"$cluster\"}, namespace)", "refresh": 1, "regex": "", "sort": 2, diff --git a/loki/.gitignore b/loki/.gitignore new file mode 100644 index 00000000..c5e15797 --- /dev/null +++ b/loki/.gitignore @@ -0,0 +1,5 @@ +vendor/ +dashboards_out/ +alerts.yaml +rules.yaml +jsonnetfile.* diff --git a/loki/mixin.libsonnet b/loki/mixin.libsonnet new file mode 100644 index 00000000..0b38cb6e --- /dev/null +++ b/loki/mixin.libsonnet @@ -0,0 +1,17 @@ +local loki = import 'loki-mixin/mixin-ssd.libsonnet'; + +loki{ + _config+:: { + tags: [ + "owner:team-atlas", + "topic:observability", + "component:loki" + ], + + per_cluster_label: 'cluster_id', + + canary+: { + enabled: true, + }, + }, +} diff --git a/loki/update.sh b/loki/update.sh index 94489ce9..8639858d 100755 --- a/loki/update.sh +++ b/loki/update.sh @@ -2,29 +2,40 @@ # Update Loki mixins from upstream # +# This script is used to update the Loki mixins from the upstream repository. +# +# Usage: +# ./loki/update.sh from the root of the repository set -e -TAG="helm-loki-5.47.2" - -tmpDir="$(mktemp -d)" - -git clone --depth 1 --single-branch -b "$TAG" --no-tags -q git@github.com:grafana/loki.git "$tmpDir" +BRANCH="main" +BRANCH="fix-cluster-id-label-in-mixins" +MIXIN_URL=https://github.com/grafana/loki/production/loki-mixin@$BRANCH +MIXIN_URL=https://github.com/QuentinBisson/loki-upstream/production/loki-mixin@$BRANCH +helmDir="$(pwd)/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private" -ls -l "$tmpDir"/production/loki-mixin-compiled-ssd/dashboards +cd loki +rm -rf vendor jsonnetfile.* -rm -rf ./output -mkdir -p ./output +jb init +jb install $MIXIN_URL +mixtool generate all mixin.libsonnet -for file in "$tmpDir"/production/loki-mixin-compiled-ssd/dashboards/*; do +for file in dashboards_out/*; do # Process each file here echo "$file" - # replaces cluster with cluster_id in queries and expr, then adds loki- prefix to uid - jq '.uid = "loki-" + .uid | .templating.list[].query |= gsub(", cluster\\)"; ", cluster_id)") | .templating.list[].query |= gsub("cluster=~\\\"\\$cluster\\\""; "cluster_id=~\"$cluster_id\"") | .templating.list[].query |= gsub("cluster=\\\"\\$cluster\\\""; "cluster_id=\"$cluster_id\"") | .rows[].panels[].targets[].expr |= gsub("cluster=~\\\"\\$cluster\\\""; "cluster_id=~\"$cluster_id\"") | .rows[].panels[].targets[].expr |= gsub("cluster=\\\"\\$cluster\\\""; "cluster_id=\"$cluster_id\"") | .tags |= ["owner:team-atlas", "topic:observability", "component:loki"] | .links[].tags |= ["owner:team-atlas", "topic:observability", "component:loki"]' "$file" > "./output/$(basename "$file")" -done - -cp ./output/* helm/dashboards/charts/private_dashboards_al/dashboards/shared/private + if [[ $(basename "$file") == "loki-canary.json" ]]; then + # adds missing uid + jq '.uid = "loki-canary"' "$file" > "$file.out" && mv "$file.out" "$file" + else + # adds loki- prefix to uid + jq '.uid = "loki-" + .uid' "$file" > "$file.out" && mv "$file.out" "$file" + fi + echo "Copying dashboard to $helmDir" + cp "$file" "$helmDir" +done