diff --git a/CHANGELOG.md b/CHANGELOG.md index cdadae1a..d0dcc1b3 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0 ## [Unreleased] +### Changed + +- Resync alloy, loki and mimir mixins from upstream. + ## [3.25.0] - 2024-10-14 ### Added diff --git a/Makefile.custom.mk b/Makefile.custom.mk index 1c3c8f6a..84173c34 100644 --- a/Makefile.custom.mk +++ b/Makefile.custom.mk @@ -13,7 +13,7 @@ update-alertmanager-mixin: ## Update Alertmanager mixin dashboards ./scripts/update-monitoring-mixin-dashboards.sh update-alloy-mixin: install-tools ## Update Alloy mixin dashboards - ./alloy/update-alloy-mixin.sh + ./alloy/update.sh update-kubernetes-mixin: ## Update Kubernetes mixin dashboards ./scripts/sync-kube-mixin.sh diff --git a/alloy/update-alloy-mixin.sh b/alloy/update.sh similarity index 96% rename from alloy/update-alloy-mixin.sh rename to alloy/update.sh index 4e3bb645..34f4c6f9 100755 --- a/alloy/update-alloy-mixin.sh +++ b/alloy/update.sh @@ -5,7 +5,7 @@ # This script updates the Alloy mixin dashboards from the upstream repository. # # Usage: -# ./update-alloy-mixin.sh +# ./update.sh set -eu diff --git a/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/alloy-opentelemetry.json b/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/alloy-opentelemetry.json index 47de4c40..d5262ed6 100644 --- a/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/alloy-opentelemetry.json +++ b/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/alloy-opentelemetry.json @@ -49,7 +49,7 @@ "targets": [ { "datasource": "${datasource}", - "expr": "rate(receiver_accepted_spans_ratio_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n", + "expr": "rate(otelcol_receiver_accepted_spans_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n", "instant": false, "legendFormat": "{{ pod }} / {{ transport }}", "range": true @@ -81,7 +81,7 @@ "targets": [ { "datasource": "${datasource}", - "expr": "rate(receiver_refused_spans_ratio_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n", + "expr": "rate(otelcol_receiver_refused_spans_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n", "instant": false, "legendFormat": "{{ pod }} / {{ transport }}", "range": true @@ -192,7 +192,7 @@ "targets": [ { "datasource": "${datasource}", - "expr": "sum by (le) (increase(processor_batch_batch_send_size_ratio_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))\n", + "expr": "sum by (le) (increase(otelcol_processor_batch_batch_send_size_bucket{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval]))\n", "format": "heatmap", "instant": false, "legendFormat": "{{le}}", @@ -214,7 +214,7 @@ "targets": [ { "datasource": "${datasource}", - "expr": "processor_batch_metadata_cardinality_ratio{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\", instance=~\"$instance\"}\n", + "expr": "otelcol_processor_batch_metadata_cardinality{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\", instance=~\"$instance\"}\n", "instant": false, "legendFormat": "{{ pod }}", "range": true @@ -235,7 +235,7 @@ "targets": [ { "datasource": "${datasource}", - "expr": "rate(processor_batch_timeout_trigger_send_ratio_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n", + "expr": "rate(otelcol_processor_batch_timeout_trigger_send_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n", "instant": false, "legendFormat": "{{ pod }}", "range": true @@ -278,7 +278,7 @@ "targets": [ { "datasource": "${datasource}", - "expr": "rate(exporter_sent_spans_ratio_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n", + "expr": "rate(otelcol_exporter_sent_spans_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n", "instant": false, "legendFormat": "{{ pod }}", "range": true @@ -310,7 +310,7 @@ "targets": [ { "datasource": "${datasource}", - "expr": "rate(exporter_send_failed_spans_ratio_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n", + "expr": "rate(otelcol_exporter_send_failed_spans_total{cluster=~\"$cluster\", namespace=~\"$namespace\", job=~\"$job\", instance=~\"$instance\"}[$__rate_interval])\n", "instant": false, "legendFormat": "{{ pod }}", "range": true diff --git a/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-canary.json b/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-canary.json index 50f4adad..1595d57d 100644 --- a/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-canary.json +++ b/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-canary.json @@ -1,23 +1,19 @@ { + "__requires": [ + { + "id": "grafana", + "name": "Grafana", + "type": "grafana", + "version": "8.0.0" + } + ], "annotations": { - "list": [ - { - "builtIn": 1, - "datasource": { - "type": "grafana", - "uid": "-- Grafana --" - }, - "enable": true, - "hide": true, - "iconColor": "rgba(0, 211, 255, 1)", - "name": "Annotations & Alerts", - "type": "dashboard" - } - ] + "list": [] }, "editable": true, - "fiscalYearStartMonth": 0, - "graphTooltip": 0, + "gnetId": null, + "graphTooltip": 1, + "hideControls": false, "links": [ { "asDropdown": true, @@ -40,16 +36,13 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "uid": "$datasource" - }, + "datasource": "$datasource", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 1, - "mappings": [], "noValue": "", "thresholds": { "mode": "absolute", @@ -66,7 +59,6 @@ "x": 0, "y": 0 }, - "id": 1, "legend": { "avg": false, "current": false, @@ -78,25 +70,9 @@ }, "lines": true, "linewidth": 1, + "links": [], "nullPointMode": "null as zero", - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, "percentage": false, - "pluginVersion": "10.4.0", "pointradius": 5, "points": false, "renderer": "flot", @@ -107,17 +83,19 @@ "steppedLine": false, "targets": [ { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(count(loki_canary_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}))", + "expr": "sum(count(loki_canary_entries_total{cluster_id=~\"$cluster\", namespace=~\"$namespace\"}))", + "format": null, "instant": false, "interval": "", + "intervalFactor": null, "legendFormat": "", - "refId": "A" + "legendLink": null, + "step": null } ], "thresholds": [], + "timeFrom": null, + "timeShift": null, "title": "Canary Entries Total", "tooltip": { "shared": false, @@ -126,20 +104,27 @@ }, "type": "stat", "xaxis": { + "buckets": null, "mode": "time", + "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", + "label": null, "logBase": 1, + "max": null, "min": 0, "show": true }, { "format": "short", + "label": null, "logBase": 1, + "max": null, + "min": null, "show": false } ] @@ -149,16 +134,13 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "uid": "$datasource" - }, + "datasource": "$datasource", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 1, - "mappings": [], "noValue": "", "thresholds": { "mode": "absolute", @@ -175,7 +157,6 @@ "x": 3, "y": 0 }, - "id": 2, "legend": { "avg": false, "current": false, @@ -187,25 +168,9 @@ }, "lines": true, "linewidth": 1, + "links": [], "nullPointMode": "null as zero", - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, "percentage": false, - "pluginVersion": "10.4.0", "pointradius": 5, "points": false, "renderer": "flot", @@ -216,17 +181,19 @@ "steppedLine": false, "targets": [ { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(increase(loki_canary_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))", + "expr": "sum(increase(loki_canary_entries_total{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))", + "format": null, "instant": false, "interval": "", + "intervalFactor": null, "legendFormat": "", - "refId": "A" + "legendLink": null, + "step": null } ], "thresholds": [], + "timeFrom": null, + "timeShift": null, "title": "Canary Logs Total", "tooltip": { "shared": false, @@ -235,20 +202,27 @@ }, "type": "stat", "xaxis": { + "buckets": null, "mode": "time", + "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", + "label": null, "logBase": 1, + "max": null, "min": 0, "show": true }, { "format": "short", + "label": null, "logBase": 1, + "max": null, + "min": null, "show": false } ] @@ -258,16 +232,13 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "uid": "$datasource" - }, + "datasource": "$datasource", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 1, - "mappings": [], "noValue": "", "thresholds": { "mode": "absolute", @@ -284,7 +255,6 @@ "x": 6, "y": 0 }, - "id": 3, "legend": { "avg": false, "current": false, @@ -296,25 +266,9 @@ }, "lines": true, "linewidth": 1, + "links": [], "nullPointMode": "null as zero", - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, "percentage": false, - "pluginVersion": "10.4.0", "pointradius": 5, "points": false, "renderer": "flot", @@ -325,17 +279,19 @@ "steppedLine": false, "targets": [ { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(increase(loki_canary_missing_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))", + "expr": "sum(increase(loki_canary_missing_entries_total{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))", + "format": null, "instant": false, "interval": "", + "intervalFactor": null, "legendFormat": "", - "refId": "A" + "legendLink": null, + "step": null } ], "thresholds": [], + "timeFrom": null, + "timeShift": null, "title": "Missing", "tooltip": { "shared": false, @@ -344,20 +300,27 @@ }, "type": "stat", "xaxis": { + "buckets": null, "mode": "time", + "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", + "label": null, "logBase": 1, + "max": null, "min": 0, "show": true }, { "format": "short", + "label": null, "logBase": 1, + "max": null, + "min": null, "show": false } ] @@ -367,16 +330,13 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "uid": "$datasource" - }, + "datasource": "$datasource", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 1, - "mappings": [], "noValue": "", "thresholds": { "mode": "absolute", @@ -393,7 +353,6 @@ "x": 9, "y": 0 }, - "id": 4, "legend": { "avg": false, "current": false, @@ -405,25 +364,9 @@ }, "lines": true, "linewidth": 1, + "links": [], "nullPointMode": "null as zero", - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, "percentage": false, - "pluginVersion": "10.4.0", "pointradius": 5, "points": false, "renderer": "flot", @@ -434,17 +377,19 @@ "steppedLine": false, "targets": [ { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(increase(loki_canary_spot_check_missing_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))", + "expr": "sum(increase(loki_canary_spot_check_missing_entries_total{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))", + "format": null, "instant": false, "interval": "", + "intervalFactor": null, "legendFormat": "", - "refId": "A" + "legendLink": null, + "step": null } ], "thresholds": [], + "timeFrom": null, + "timeShift": null, "title": "Spotcheck Missing", "tooltip": { "shared": false, @@ -453,146 +398,43 @@ }, "type": "stat", "xaxis": { + "buckets": null, "mode": "time", + "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", + "label": null, "logBase": 1, + "max": null, "min": 0, "show": true }, { "format": "short", + "label": null, "logBase": 1, + "max": null, + "min": null, "show": false } ] }, - { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 0 - }, - "id": 13, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "histogram_quantile(0.95, sum(rate(loki_canary_response_latency_seconds_bucket{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "p95", - "refId": "A", - "step": 10 - }, - { - "datasource": { - "uid": "$datasource" - }, - "expr": "histogram_quantile(0.50, sum(rate(loki_canary_response_latency_seconds_bucket{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) by (le))", - "format": "time_series", - "intervalFactor": 2, - "legendFormat": "p50", - "refId": "B", - "step": 10 - } - ], - "title": "Log Write to read Latency Percentiles", - "type": "timeseries" - }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "uid": "$datasource" - }, + "datasource": "$datasource", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 1, - "mappings": [], "noValue": "", "thresholds": { "mode": "absolute", @@ -609,7 +451,6 @@ "x": 0, "y": 4 }, - "id": 5, "legend": { "avg": false, "current": false, @@ -621,25 +462,9 @@ }, "lines": true, "linewidth": 1, + "links": [], "nullPointMode": "null as zero", - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, "percentage": false, - "pluginVersion": "10.4.0", "pointradius": 5, "points": false, "renderer": "flot", @@ -650,17 +475,19 @@ "steppedLine": false, "targets": [ { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(increase(loki_canary_spot_check_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))", + "expr": "sum(increase(loki_canary_spot_check_entries_total{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))", + "format": null, "instant": false, "interval": "", + "intervalFactor": null, "legendFormat": "", - "refId": "A" + "legendLink": null, + "step": null } ], "thresholds": [], + "timeFrom": null, + "timeShift": null, "title": "Spotcheck Total", "tooltip": { "shared": false, @@ -669,20 +496,27 @@ }, "type": "stat", "xaxis": { + "buckets": null, "mode": "time", + "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", + "label": null, "logBase": 1, + "max": null, "min": 0, "show": true }, { "format": "short", + "label": null, "logBase": 1, + "max": null, + "min": null, "show": false } ] @@ -692,18 +526,13 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "uid": "$datasource" - }, + "datasource": "$datasource", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 1, - "mappings": [], - "max": 1, - "min": 0, "noValue": "", "thresholds": { "mode": "absolute", @@ -720,7 +549,6 @@ "x": 3, "y": 4 }, - "id": 6, "legend": { "avg": false, "current": false, @@ -732,25 +560,9 @@ }, "lines": true, "linewidth": 1, + "links": [], "nullPointMode": "null as zero", - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, "percentage": false, - "pluginVersion": "10.4.0", "pointradius": 5, "points": false, "renderer": "flot", @@ -761,17 +573,19 @@ "steppedLine": false, "targets": [ { - "datasource": { - "uid": "$datasource" - }, - "expr": "((sum(loki_canary_metric_test_expected{cluster=~\"$cluster\",namespace=~\"$namespace\"}) - sum(loki_canary_metric_test_actual{cluster=~\"$cluster\",namespace=~\"$namespace\"}))/(sum(loki_canary_metric_test_actual{cluster=~\"$cluster\",namespace=~\"$namespace\"}))) * 100", + "expr": "((sum(loki_canary_metric_test_expected{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}) - sum(loki_canary_metric_test_actual{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}))/(sum(loki_canary_metric_test_actual{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}))) * 100", + "format": null, "instant": false, "interval": "", + "intervalFactor": null, "legendFormat": "", - "refId": "A" + "legendLink": null, + "step": null } ], "thresholds": [], + "timeFrom": null, + "timeShift": null, "title": "Metric Test Error %", "tooltip": { "shared": false, @@ -780,20 +594,27 @@ }, "type": "stat", "xaxis": { + "buckets": null, "mode": "time", + "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", + "label": null, "logBase": 1, + "max": null, "min": 0, "show": true }, { "format": "short", + "label": null, "logBase": 1, + "max": null, + "min": null, "show": false } ] @@ -803,18 +624,13 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "uid": "$datasource" - }, + "datasource": "$datasource", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 1, - "mappings": [], - "max": 1, - "min": 0, "noValue": "", "thresholds": { "mode": "absolute", @@ -831,7 +647,6 @@ "x": 6, "y": 4 }, - "id": 7, "legend": { "avg": false, "current": false, @@ -843,25 +658,9 @@ }, "lines": true, "linewidth": 1, + "links": [], "nullPointMode": "null as zero", - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, "percentage": false, - "pluginVersion": "10.4.0", "pointradius": 5, "points": false, "renderer": "flot", @@ -872,17 +671,19 @@ "steppedLine": false, "targets": [ { - "datasource": { - "uid": "$datasource" - }, - "expr": "(sum(increase(loki_canary_missing_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))/sum(increase(loki_canary_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__range])))*100", + "expr": "(sum(increase(loki_canary_missing_entries_total{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))/sum(increase(loki_canary_entries_total{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[$__range])))*100", + "format": null, "instant": false, "interval": "", + "intervalFactor": null, "legendFormat": "", - "refId": "A" + "legendLink": null, + "step": null } ], "thresholds": [], + "timeFrom": null, + "timeShift": null, "title": "Missing %", "tooltip": { "shared": false, @@ -891,20 +692,27 @@ }, "type": "stat", "xaxis": { + "buckets": null, "mode": "time", + "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", + "label": null, "logBase": 1, + "max": null, "min": 0, "show": true }, { "format": "short", + "label": null, "logBase": 1, + "max": null, + "min": null, "show": false } ] @@ -914,18 +722,13 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "uid": "$datasource" - }, + "datasource": "$datasource", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 1, - "mappings": [], - "max": 1, - "min": 0, "noValue": "", "thresholds": { "mode": "absolute", @@ -942,7 +745,6 @@ "x": 9, "y": 4 }, - "id": 8, "legend": { "avg": false, "current": false, @@ -954,25 +756,9 @@ }, "lines": true, "linewidth": 1, + "links": [], "nullPointMode": "null as zero", - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, "percentage": false, - "pluginVersion": "10.4.0", "pointradius": 5, "points": false, "renderer": "flot", @@ -983,17 +769,19 @@ "steppedLine": false, "targets": [ { - "datasource": { - "uid": "$datasource" - }, - "expr": "(sum(increase(loki_canary_spot_check_missing_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))/sum(increase(loki_canary_spot_check_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))) * 100", + "expr": "(sum(increase(loki_canary_spot_check_missing_entries_total{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))/sum(increase(loki_canary_spot_check_entries_total{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))) * 100", + "format": null, "instant": false, "interval": "", + "intervalFactor": null, "legendFormat": "", - "refId": "A" + "legendLink": null, + "step": null } ], "thresholds": [], + "timeFrom": null, + "timeShift": null, "title": "Spotcheck Missing %", "tooltip": { "shared": false, @@ -1002,147 +790,43 @@ }, "type": "stat", "xaxis": { + "buckets": null, "mode": "time", + "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", + "label": null, "logBase": 1, + "max": null, "min": 0, "show": true }, { "format": "short", + "label": null, "logBase": 1, + "max": null, + "min": null, "show": false } ] }, - { - "cards": {}, - "color": { - "cardColor": "#b4ff00", - "colorScale": "sqrt", - "colorScheme": "interpolateReds", - "exponent": 0.5, - "mode": "spectrum" - }, - "dataFormat": "timeseries", - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "custom": { - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "scaleDistribution": { - "type": "linear" - } - } - }, - "overrides": [] - }, - "gridPos": { - "h": 6, - "w": 12, - "x": 12, - "y": 6 - }, - "heatmap": {}, - "hideZeroBuckets": false, - "highlightCards": true, - "id": 14, - "legend": { - "show": false - }, - "options": { - "calculate": true, - "calculation": {}, - "cellGap": 2, - "cellValues": {}, - "color": { - "exponent": 0.5, - "fill": "#b4ff00", - "mode": "scheme", - "reverse": false, - "scale": "exponential", - "scheme": "Reds", - "steps": 128 - }, - "exemplars": { - "color": "rgba(255,0,255,0.7)" - }, - "filterValues": { - "le": 1e-9 - }, - "legend": { - "show": false - }, - "rowsFrame": { - "layout": "auto" - }, - "showValue": "never", - "tooltip": { - "mode": "single", - "showColorScale": false, - "yHistogram": true - }, - "yAxis": { - "axisPlacement": "left", - "reverse": false, - "unit": "short" - } - }, - "pluginVersion": "10.4.0", - "targets": [ - { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(rate(loki_canary_response_latency_seconds_bucket{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) by (le)", - "format": "heatmap", - "intervalFactor": 2, - "legendFormat": "{{le}}", - "refId": "A" - } - ], - "title": "Log Write to Read Latency", - "tooltip": { - "show": true, - "showHistogram": true - }, - "type": "heatmap", - "xAxis": { - "show": true - }, - "yAxis": { - "format": "short", - "logBase": 1, - "show": true - }, - "yBucketBound": "auto" - }, { "aliasColors": {}, "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "uid": "$datasource" - }, + "datasource": "$datasource", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 1, - "mappings": [], "noValue": "", "thresholds": { "mode": "absolute", @@ -1159,7 +843,6 @@ "x": 0, "y": 8 }, - "id": 9, "legend": { "avg": false, "current": false, @@ -1171,25 +854,9 @@ }, "lines": true, "linewidth": 1, + "links": [], "nullPointMode": "null as zero", - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, "percentage": false, - "pluginVersion": "10.4.0", "pointradius": 5, "points": false, "renderer": "flot", @@ -1200,17 +867,19 @@ "steppedLine": false, "targets": [ { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(loki_canary_metric_test_expected{cluster=~\"$cluster\",namespace=~\"$namespace\"})", + "expr": "sum(loki_canary_metric_test_expected{cluster_id=~\"$cluster\",namespace=~\"$namespace\"})", + "format": null, "instant": false, "interval": "", + "intervalFactor": null, "legendFormat": "", - "refId": "A" + "legendLink": null, + "step": null } ], "thresholds": [], + "timeFrom": null, + "timeShift": null, "title": "Metric Test Expected", "tooltip": { "shared": false, @@ -1219,20 +888,27 @@ }, "type": "stat", "xaxis": { + "buckets": null, "mode": "time", + "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", + "label": null, "logBase": 1, + "max": null, "min": 0, "show": true }, { "format": "short", + "label": null, "logBase": 1, + "max": null, + "min": null, "show": false } ] @@ -1242,16 +918,13 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "uid": "$datasource" - }, + "datasource": "$datasource", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 1, - "mappings": [], "noValue": "", "thresholds": { "mode": "absolute", @@ -1268,7 +941,6 @@ "x": 3, "y": 8 }, - "id": 10, "legend": { "avg": false, "current": false, @@ -1280,25 +952,9 @@ }, "lines": true, "linewidth": 1, + "links": [], "nullPointMode": "null as zero", - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, "percentage": false, - "pluginVersion": "10.4.0", "pointradius": 5, "points": false, "renderer": "flot", @@ -1309,17 +965,19 @@ "steppedLine": false, "targets": [ { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(loki_canary_metric_test_actual{cluster=~\"$cluster\",namespace=~\"$namespace\"})", + "expr": "sum(loki_canary_metric_test_actual{cluster_id=~\"$cluster\",namespace=~\"$namespace\"})", + "format": null, "instant": false, "interval": "", + "intervalFactor": null, "legendFormat": "", - "refId": "A" + "legendLink": null, + "step": null } ], "thresholds": [], + "timeFrom": null, + "timeShift": null, "title": "Metric Test Actual", "tooltip": { "shared": false, @@ -1328,20 +986,27 @@ }, "type": "stat", "xaxis": { + "buckets": null, "mode": "time", + "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", + "label": null, "logBase": 1, + "max": null, "min": 0, "show": true }, { "format": "short", + "label": null, "logBase": 1, + "max": null, + "min": null, "show": false } ] @@ -1351,16 +1016,13 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "uid": "$datasource" - }, + "datasource": "$datasource", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 1, - "mappings": [], "noValue": "", "thresholds": { "mode": "absolute", @@ -1377,7 +1039,6 @@ "x": 6, "y": 8 }, - "id": 11, "legend": { "avg": false, "current": false, @@ -1389,25 +1050,9 @@ }, "lines": true, "linewidth": 1, + "links": [], "nullPointMode": "null as zero", - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, "percentage": false, - "pluginVersion": "10.4.0", "pointradius": 5, "points": false, "renderer": "flot", @@ -1418,17 +1063,19 @@ "steppedLine": false, "targets": [ { - "datasource": { - "uid": "$datasource" - }, - "expr": "sum(increase(loki_canary_websocket_missing_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))", + "expr": "sum(increase(loki_canary_websocket_missing_entries_total{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))", + "format": null, "instant": false, "interval": "", + "intervalFactor": null, "legendFormat": "", - "refId": "A" + "legendLink": null, + "step": null } ], "thresholds": [], + "timeFrom": null, + "timeShift": null, "title": "Websocket Missing", "tooltip": { "shared": false, @@ -1437,20 +1084,27 @@ }, "type": "stat", "xaxis": { + "buckets": null, "mode": "time", + "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", + "label": null, "logBase": 1, + "max": null, "min": 0, "show": true }, { "format": "short", + "label": null, "logBase": 1, + "max": null, + "min": null, "show": false } ] @@ -1460,18 +1114,13 @@ "bars": false, "dashLength": 10, "dashes": false, - "datasource": { - "uid": "$datasource" - }, + "datasource": "$datasource", "fieldConfig": { "defaults": { "color": { "mode": "thresholds" }, "decimals": 1, - "mappings": [], - "max": 1, - "min": 0, "noValue": "", "thresholds": { "mode": "absolute", @@ -1488,7 +1137,6 @@ "x": 9, "y": 8 }, - "id": 12, "legend": { "avg": false, "current": false, @@ -1500,25 +1148,9 @@ }, "lines": true, "linewidth": 1, + "links": [], "nullPointMode": "null as zero", - "options": { - "colorMode": "value", - "graphMode": "area", - "justifyMode": "auto", - "orientation": "auto", - "reduceOptions": { - "calcs": [ - "lastNotNull" - ], - "fields": "", - "values": false - }, - "showPercentChange": false, - "textMode": "auto", - "wideLayout": true - }, "percentage": false, - "pluginVersion": "10.4.0", "pointradius": 5, "points": false, "renderer": "flot", @@ -1529,17 +1161,19 @@ "steppedLine": false, "targets": [ { - "datasource": { - "uid": "$datasource" - }, - "expr": "(sum(increase(loki_canary_websocket_missing_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))/sum(increase(loki_canary_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__range])))*100", + "expr": "(sum(increase(loki_canary_websocket_missing_entries_total{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))/sum(increase(loki_canary_entries_total{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[$__range])))*100", + "format": null, "instant": false, "interval": "", + "intervalFactor": null, "legendFormat": "", - "refId": "A" + "legendLink": null, + "step": null } ], "thresholds": [], + "timeFrom": null, + "timeShift": null, "title": "Websocket Missing %", "tooltip": { "shared": false, @@ -1548,433 +1182,508 @@ }, "type": "stat", "xaxis": { + "buckets": null, "mode": "time", + "name": null, "show": true, "values": [] }, "yaxes": [ { "format": "short", + "label": null, "logBase": 1, + "max": null, "min": 0, "show": true }, { "format": "short", + "label": null, "logBase": 1, + "max": null, + "min": null, "show": false } ] }, { - "datasource": { - "uid": "$datasource" - }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" - }, - "overrides": [] - }, + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, "gridPos": { "h": 6, "w": 12, - "x": 0, - "y": 12 + "x": 12, + "y": 0 }, - "id": 15, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false }, - "pluginVersion": "10.4.0", + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, "targets": [ { - "datasource": { - "uid": "$datasource" - }, - "expr": "histogram_quantile(0.99, sum(rate(loki_canary_spot_check_request_duration_seconds_bucket{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.95, sum(rate(loki_canary_response_latency_seconds_bucket{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) by (le))", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "p99", - "refId": "A", - "step": 10 + "legendFormat": "p95", + "legendLink": null }, { - "datasource": { - "uid": "$datasource" - }, - "expr": "histogram_quantile(0.50, sum(rate(loki_canary_spot_check_request_duration_seconds_bucket{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) by (le))", + "expr": "histogram_quantile(0.50, sum(rate(loki_canary_response_latency_seconds_bucket{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) by (le))", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "p95", - "refId": "B", - "step": 10 + "legendFormat": "p50", + "legendLink": null } ], - "title": "Spot Check Query", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Log Write to read Latency Percentiles", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" + "show": true }, - "overrides": [] - }, - "gridPos": { + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "cards": { + "cardPadding": null, + "cardRound": null + }, + "color": { + "cardColor": "#b4ff00", + "colorScale": "sqrt", + "colorScheme": "interpolateReds", + "exponent": 0.5, + "mode": "spectrum" + }, + "dataFormat": "timeseries", + "datasource": "$datasource", + "gridPos": { "h": 6, "w": 12, "x": 12, "y": 12 }, - "id": 16, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" + "heatmap": {}, + "hideZeroBuckets": false, + "highlightCards": true, + "legend": { + "show": false + }, + "targets": [ + { + "expr": "sum(rate(loki_canary_response_latency_seconds_bucket{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) by (le)", + "format": "heatmap", + "intervalFactor": 2, + "legendFormat": "{{le}}", + "refId": "A" } + ], + "title": "Log Write to Read Latency", + "tooltip": { + "show": true, + "showHistogram": true }, - "pluginVersion": "10.4.0", + "type": "heatmap", + "xAxis": { + "show": true + }, + "xBucketNumber": null, + "xBucketSize": null, + "yAxis": { + "decimals": null, + "format": "short", + "logBase": 1, + "max": null, + "min": null, + "show": true, + "splitFactor": null + }, + "yBucketBound": "auto", + "yBucketNumber": null, + "yBucketSize": null + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 0, + "y": 14 + }, + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, "targets": [ { - "datasource": { - "uid": "$datasource" - }, - "expr": "histogram_quantile(0.99, sum(rate(loki_canary_metric_test_request_duration_seconds_bucket{cluster=~\"$cluster\",namespace=~\"$namespace\"}[15m])) by (le))", + "expr": "histogram_quantile(0.99, sum(rate(loki_canary_spot_check_request_duration_seconds_bucket{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) by (le))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "p99", - "refId": "A", - "step": 10 + "legendLink": null }, { - "datasource": { - "uid": "$datasource" - }, - "expr": "histogram_quantile(0.50, sum(rate(loki_canary_metric_test_request_duration_seconds_bucket{cluster=~\"$cluster\",namespace=~\"$namespace\"}[15m])) by (le))", + "expr": "histogram_quantile(0.50, sum(rate(loki_canary_spot_check_request_duration_seconds_bucket{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) by (le))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "p95", - "refId": "B", - "step": 10 + "legendLink": null } ], - "title": "Metric Test Query", - "type": "timeseries" + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Spot Check Query", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] }, { - "datasource": { - "uid": "$datasource" + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, + "gridPos": { + "h": 6, + "w": 12, + "x": 12, + "y": 14 }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], - "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(loki_canary_metric_test_request_duration_seconds_bucket{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[15m])) by (le))", + "format": "time_series", + "legendFormat": "p99", + "legendLink": null }, - "overrides": [] + { + "expr": "histogram_quantile(0.50, sum(rate(loki_canary_metric_test_request_duration_seconds_bucket{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[15m])) by (le))", + "format": "time_series", + "legendFormat": "p95", + "legendLink": null + } + ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "Metric Test Query", + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, "gridPos": { "h": 6, "w": 12, "x": 0, - "y": 18 - }, - "id": 17, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "single", - "sort": "none" - } + "y": 20 }, - "pluginVersion": "10.4.0", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, "targets": [ { - "datasource": { - "uid": "$datasource" - }, - "expr": "topk(20, (sum by (cluster, pod) (increase(loki_canary_spot_check_missing_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval]))/sum by (cluster, pod) (increase(loki_canary_spot_check_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) * 100)) > 0", + "expr": "topk(20, (sum by (cluster_id, pod) (increase(loki_canary_spot_check_missing_entries_total{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval]))/sum by (cluster_id, pod) (increase(loki_canary_spot_check_entries_total{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) * 100)) > 0", "format": "time_series", - "intervalFactor": 2, "legendFormat": "", - "refId": "A", - "step": 10 + "legendLink": null } ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, "title": "Spot Check Missing %", - "type": "timeseries" - }, - { - "datasource": { - "uid": "$datasource" + "tooltip": { + "shared": false, + "sort": 0, + "value_type": "individual" }, - "fieldConfig": { - "defaults": { - "color": { - "mode": "palette-classic" - }, - "custom": { - "axisBorderShow": false, - "axisCenteredZero": false, - "axisColorMode": "text", - "axisLabel": "", - "axisPlacement": "auto", - "barAlignment": 0, - "drawStyle": "line", - "fillOpacity": 10, - "gradientMode": "none", - "hideFrom": { - "legend": false, - "tooltip": false, - "viz": false - }, - "insertNulls": false, - "lineInterpolation": "linear", - "lineWidth": 1, - "pointSize": 5, - "scaleDistribution": { - "type": "linear" - }, - "showPoints": "never", - "spanNulls": false, - "stacking": { - "group": "A", - "mode": "none" - }, - "thresholdsStyle": { - "mode": "off" - } - }, - "mappings": [], + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, "min": 0, - "thresholds": { - "mode": "absolute", - "steps": [ - { - "color": "green", - "value": null - }, - { - "color": "red", - "value": 80 - } - ] - }, - "unit": "short" + "show": true }, - "overrides": [] - }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "$datasource", + "fill": 1, "gridPos": { "h": 6, "w": 12, "x": 12, - "y": 18 - }, - "id": 18, - "options": { - "legend": { - "calcs": [], - "displayMode": "list", - "placement": "bottom", - "showLegend": true - }, - "tooltip": { - "mode": "multi", - "sort": "desc" - } + "y": 20 }, - "pluginVersion": "10.4.0", + "legend": { + "avg": false, + "current": false, + "max": false, + "min": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null as zero", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "seriesOverrides": [], + "spaceLength": 10, + "span": 6, + "stack": false, + "steppedLine": false, "targets": [ { - "datasource": { - "uid": "$datasource" - }, - "expr": "topk(20,(sum by (cluster, pod)(increase(loki_canary_missing_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval]))/sum by (cluster, pod)(increase(loki_canary_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])))*100) > 0", + "expr": "topk(20,(sum by (cluster_id, pod)(increase(loki_canary_missing_entries_total{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval]))/sum by (cluster_id, pod)(increase(loki_canary_entries_total{cluster_id=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])))*100) > 0", "format": "time_series", - "intervalFactor": 2, - "legendFormat": "Missing {{ cluster }} {{ pod }}", - "refId": "A", - "step": 10 + "legendFormat": "Missing {{ cluster_id }} {{ pod }}", + "legendLink": null } ], + "thresholds": [], + "timeFrom": null, + "timeShift": null, "title": "Missing logs", - "type": "timeseries" + "tooltip": { + "shared": true, + "sort": 2, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] } ], - "refresh": "10s", - "schemaVersion": 39, + "refresh": "5m", + "rows": null, + "schemaVersion": 27, + "style": "dark", "tags": [ "owner:team-atlas", "topic:observability", @@ -1984,72 +1693,60 @@ "list": [ { "current": { - "selected": false, "text": "default", "value": "default" }, "hide": 0, - "includeAll": false, "label": "Data source", - "multi": false, "name": "datasource", "options": [], "query": "prometheus", "refresh": 1, "regex": "", - "skipUrlSync": false, "type": "datasource" }, { + "allValue": null, "current": { - "selected": false, - "text": "loki", - "value": "loki" + "text": "prod", + "value": "prod" }, - "datasource": { - "type": "prometheus", - "uid": "$datasource" - }, - "definition": "", + "datasource": "$datasource", "hide": 0, "includeAll": false, "label": "cluster", "multi": false, "name": "cluster", "options": [], - "query": "label_values(loki_build_info, cluster)", + "query": "label_values(loki_build_info, cluster_id)", "refresh": 1, "regex": "", - "skipUrlSync": false, "sort": 2, "tagValuesQuery": "", + "tags": [], "tagsQuery": "", "type": "query", "useTags": false }, { + "allValue": null, "current": { - "selected": false, - "text": "loki", - "value": "loki" - }, - "datasource": { - "type": "prometheus", - "uid": "$datasource" + "text": "prod", + "value": "prod" }, - "definition": "", + "datasource": "$datasource", "hide": 0, "includeAll": false, "label": "namespace", "multi": false, "name": "namespace", "options": [], - "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)", + "query": "label_values(loki_build_info{cluster_id=~\"$cluster\"}, namespace)", "refresh": 1, "regex": "", - "skipUrlSync": false, "sort": 2, "tagValuesQuery": "", + "tags": [], "tagsQuery": "", "type": "query", "useTags": false @@ -2088,6 +1785,5 @@ "timezone": "utc", "title": "Loki / Canary", "uid": "loki-canary", - "version": 1, - "weekStart": "" + "version": 0 } diff --git a/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-retention.json b/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-retention.json index 72727984..41df4bfa 100644 --- a/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-retention.json +++ b/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-retention.json @@ -497,7 +497,7 @@ "span": 6, "targets": [ { - "expr": "sum(increase(loki_compactor_skipped_compacting_locked_table_total{cluster_id=~\"$cluster\", namespace=~\"$namespace\"}[$__range]))", + "expr": "sum(loki_compactor_locked_table_successive_compaction_skips{cluster_id=~\"$cluster\", namespace=~\"$namespace\"})", "format": "time_series", "legendFormat": "{{table_name}}", "legendLink": null diff --git a/helm/dashboards/charts/private_dashboards_mz/dashboards/shared/private/mimir-alertmanager.json b/helm/dashboards/charts/private_dashboards_mz/dashboards/shared/private/mimir-alertmanager.json index 0ab9dfa6..3194e613 100644 --- a/helm/dashboards/charts/private_dashboards_mz/dashboards/shared/private/mimir-alertmanager.json +++ b/helm/dashboards/charts/private_dashboards_mz/dashboards/shared/private/mimir-alertmanager.json @@ -624,7 +624,7 @@ "span": 12, "targets": [ { - "expr": "sum(cluster_id_job:cortex_alertmanager_alerts_received_total:rate5m{cluster_id=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n-\nsum(cluster_id_job:cortex_alertmanager_alerts_invalid_total:rate5m{cluster_id=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n", + "expr": "sum(cluster_id_job:cortex_alertmanager_alerts_received_total:rate5m{cluster_id=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n-\n(sum(cluster_id_job:cortex_alertmanager_alerts_invalid_total:rate5m{cluster_id=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}) or vector(0))\n", "format": "time_series", "legendFormat": "successful", "legendLink": null @@ -2169,7 +2169,7 @@ "span": 4, "targets": [ { - "expr": "sum(cluster_id_job:cortex_alertmanager_state_replication_total:rate5m{cluster_id=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n-\nsum(cluster_id_job:cortex_alertmanager_state_replication_failed_total:rate5m{cluster_id=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n", + "expr": "sum(cluster_id_job:cortex_alertmanager_state_replication_total:rate5m{cluster_id=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n-\n(sum(cluster_id_job:cortex_alertmanager_state_replication_failed_total:rate5m{cluster_id=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}) or vector(0))\n", "format": "time_series", "legendFormat": "successful", "legendLink": null @@ -2254,7 +2254,7 @@ "span": 4, "targets": [ { - "expr": "sum(cluster_id_job:cortex_alertmanager_partial_state_merges_total:rate5m{cluster_id=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n-\nsum(cluster_id_job:cortex_alertmanager_partial_state_merges_failed_total:rate5m{cluster_id=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n", + "expr": "sum(cluster_id_job:cortex_alertmanager_partial_state_merges_total:rate5m{cluster_id=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"})\n-\n(sum(cluster_id_job:cortex_alertmanager_partial_state_merges_failed_total:rate5m{cluster_id=~\"$cluster\", job=~\"($namespace)/((alertmanager|cortex|mimir|mimir-backend.*))\"}) or vector(0))\n", "format": "time_series", "legendFormat": "successful", "legendLink": null diff --git a/helm/dashboards/charts/private_dashboards_mz/dashboards/shared/private/mimir-compactor.json b/helm/dashboards/charts/private_dashboards_mz/dashboards/shared/private/mimir-compactor.json index 127cc5df..45b5b67f 100644 --- a/helm/dashboards/charts/private_dashboards_mz/dashboards/shared/private/mimir-compactor.json +++ b/helm/dashboards/charts/private_dashboards_mz/dashboards/shared/private/mimir-compactor.json @@ -622,7 +622,7 @@ "sort": "none" } }, - "span": 4, + "span": 3, "targets": [ { "expr": "sum(cortex_bucket_index_estimated_compaction_jobs{cluster_id=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}) and (sum(rate(cortex_bucket_index_estimated_compaction_jobs_errors_total{cluster_id=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) == 0)", @@ -634,6 +634,86 @@ "title": "Estimated Compaction Jobs", "type": "timeseries" }, + { + "datasource": "$datasource", + "description": "### Source blocks age\nThe difference between the maximum timestamp of the block being compacted and the current time.\nA steadily increasing value indicates that the compactor cannot keep up with the produced blocks by the ingesters.\nIncrease the number of compactors when this value is consistently increasing.\n\n", + "fieldConfig": { + "defaults": { + "custom": { + "drawStyle": "line", + "fillOpacity": 1, + "lineWidth": 1, + "pointSize": 5, + "showPoints": "never", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "min": 0, + "thresholds": { + "mode": "absolute", + "steps": [ ] + }, + "unit": "ms" + }, + "overrides": [ ] + }, + "id": 6, + "links": [ ], + "nullPointMode": "null as zero", + "options": { + "legend": { + "showLegend": true + }, + "tooltip": { + "mode": "multi", + "sort": "none" + } + }, + "span": 3, + "targets": [ + { + "expr": "histogram_quantile(0.99, sum(rate(cortex_compactor_block_max_time_delta_seconds_bucket{cluster_id=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3", + "format": "time_series", + "legendFormat": "99th Percentile", + "refId": "A" + }, + { + "expr": "histogram_quantile(0.50, sum(rate(cortex_compactor_block_max_time_delta_seconds_bucket{cluster_id=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3", + "format": "time_series", + "legendFormat": "50th Percentile", + "refId": "B" + }, + { + "expr": "sum(rate(cortex_compactor_block_max_time_delta_seconds_sum{cluster_id=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) * 1e3 / sum(rate(cortex_compactor_block_max_time_delta_seconds_count{cluster_id=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))", + "format": "time_series", + "legendFormat": "Average", + "refId": "C" + } + ], + "title": "Source blocks age", + "type": "timeseries", + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] + }, { "datasource": "$datasource", "description": "### TSDB compactions / sec\nRate of TSDB compactions. Single TSDB compaction takes one or more input blocks and produces one or more (during \"split\" phase) output blocks.\n\n", @@ -660,7 +740,7 @@ }, "overrides": [ ] }, - "id": 6, + "id": 7, "links": [ ], "options": { "legend": { @@ -671,7 +751,7 @@ "sort": "none" } }, - "span": 4, + "span": 3, "targets": [ { "expr": "sum(rate(prometheus_tsdb_compactions_total{cluster_id=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))", @@ -709,7 +789,7 @@ }, "overrides": [ ] }, - "id": 7, + "id": 8, "links": [ ], "nullPointMode": "null as zero", "options": { @@ -721,7 +801,7 @@ "sort": "none" } }, - "span": 4, + "span": 3, "targets": [ { "expr": "histogram_quantile(0.99, sum(rate(prometheus_tsdb_compaction_duration_seconds_bucket{cluster_id=~\"$cluster\", job=~\"($namespace)/((compactor.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval])) by (le)) * 1e3", @@ -800,7 +880,7 @@ }, "overrides": [ ] }, - "id": 8, + "id": 9, "links": [ ], "options": { "legend": { @@ -849,7 +929,7 @@ }, "overrides": [ ] }, - "id": 9, + "id": 10, "links": [ ], "options": { "legend": { @@ -909,7 +989,7 @@ }, "overrides": [ ] }, - "id": 10, + "id": 11, "links": [ ], "options": { "legend": { @@ -988,7 +1068,7 @@ } ] }, - "id": 11, + "id": 12, "links": [ ], "options": { "legend": { @@ -1085,7 +1165,7 @@ } ] }, - "id": 12, + "id": 13, "links": [ ], "options": { "legend": { @@ -1139,7 +1219,7 @@ }, "overrides": [ ] }, - "id": 13, + "id": 14, "links": [ ], "nullPointMode": "null as zero", "options": { @@ -1230,7 +1310,7 @@ }, "overrides": [ ] }, - "id": 14, + "id": 15, "links": [ ], "options": { "legend": { @@ -1263,7 +1343,7 @@ "unit": "percentunit" } }, - "id": 15, + "id": 16, "links": [ ], "options": { "legend": { @@ -1311,7 +1391,7 @@ }, "overrides": [ ] }, - "id": 16, + "id": 17, "links": [ ], "nullPointMode": "null as zero", "options": { @@ -1390,7 +1470,7 @@ }, "overrides": [ ] }, - "id": 17, + "id": 18, "links": [ ], "nullPointMode": "null as zero", "options": { @@ -1481,7 +1561,7 @@ }, "overrides": [ ] }, - "id": 18, + "id": 19, "links": [ ], "nullPointMode": "null as zero", "options": { @@ -1560,7 +1640,7 @@ }, "overrides": [ ] }, - "id": 19, + "id": 20, "links": [ ], "nullPointMode": "null as zero", "options": { @@ -1639,7 +1719,7 @@ }, "overrides": [ ] }, - "id": 20, + "id": 21, "links": [ ], "nullPointMode": "null as zero", "options": { @@ -1718,7 +1798,7 @@ }, "overrides": [ ] }, - "id": 21, + "id": 22, "links": [ ], "nullPointMode": "null as zero", "options": { @@ -1945,7 +2025,7 @@ } ] }, - "id": 22, + "id": 23, "links": [ ], "options": { "legend": { @@ -1993,7 +2073,7 @@ }, "overrides": [ ] }, - "id": 23, + "id": 24, "links": [ ], "nullPointMode": "null as zero", "options": { diff --git a/helm/dashboards/charts/private_dashboards_mz/dashboards/shared/private/mimir-queries.json b/helm/dashboards/charts/private_dashboards_mz/dashboards/shared/private/mimir-queries.json index 04a2a428..295987f2 100644 --- a/helm/dashboards/charts/private_dashboards_mz/dashboards/shared/private/mimir-queries.json +++ b/helm/dashboards/charts/private_dashboards_mz/dashboards/shared/private/mimir-queries.json @@ -580,7 +580,7 @@ "span": 4, "targets": [ { - "expr": "# Query the new metric introduced in Mimir 2.10.\n(\n sum by(request_type) (rate(cortex_frontend_query_result_cache_hits_total{$read_path_matcher}[$__rate_interval]))\n /\n sum by(request_type) (rate(cortex_frontend_query_result_cache_requests_total{$read_path_matcher}[$__rate_interval]))\n)\n# Otherwise fallback to the previous general-purpose metrics.\nor\n(\n label_replace(\n # Query metrics before and after dskit cache refactor.\n sum (\n rate(thanos_cache_memcached_hits_total{name=\"frontend-cache\", $read_path_matcher}[$__rate_interval])\n or ignoring(backend)\n rate(thanos_cache_hits_total{name=\"frontend-cache\", $read_path_matcher}[$__rate_interval])\n )\n /\n sum (\n rate(thanos_cache_memcached_requests_total{name=~\"frontend-cache\", $read_path_matcher}[$__rate_interval])\n or ignoring(backend)\n rate(thanos_cache_requests_total{name=~\"frontend-cache\", $read_path_matcher}[$__rate_interval])\n ),\n \"request_type\", \"query_range\", \"\", \"\")\n)\n", + "expr": "# Query the new metric introduced in Mimir 2.10.\n(\n sum by(request_type) (rate(cortex_frontend_query_result_cache_hits_total{$read_path_matcher}[$__rate_interval]))\n /\n sum by(request_type) (rate(cortex_frontend_query_result_cache_requests_total{$read_path_matcher}[$__rate_interval]))\n)\n# Otherwise fallback to the previous general-purpose metrics.\nor\n(\n label_replace(\n sum (\n rate(thanos_cache_hits_total{name=\"frontend-cache\", $read_path_matcher}[$__rate_interval])\n )\n /\n sum (\n rate(thanos_cache_requests_total{name=~\"frontend-cache\", $read_path_matcher}[$__rate_interval])\n ),\n \"request_type\", \"query_range\", \"\", \"\")\n)\n", "format": "time_series", "legendFormat": "{{request_type}}", "legendLink": null @@ -1512,18 +1512,19 @@ "panels": [ { "datasource": "$datasource", + "description": "### Blocks queried / sec by compaction level\nIncreased volume of lower levels (for example levels 1 and 2) can indicate that the compactor is not keeping up.\nIn that case the store-gateway will start serving more blocks which aren't that well compacted.\n\n", "fieldConfig": { "defaults": { "custom": { "drawStyle": "line", - "fillOpacity": 1, - "lineWidth": 1, + "fillOpacity": 100, + "lineWidth": 0, "pointSize": 5, "showPoints": "never", "spanNulls": false, "stacking": { "group": "A", - "mode": "none" + "mode": "normal" } }, "min": 0, @@ -1549,13 +1550,19 @@ "span": 4, "targets": [ { - "expr": "sum(rate(cortex_bucket_store_series_blocks_queried_sum{component=\"store-gateway\",cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))", + "expr": "sum by (level) (rate(cortex_bucket_store_series_blocks_queried_sum{component=\"store-gateway\",level=~\"[0-4]\",cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))", + "format": "time_series", + "legendFormat": "{{level}}", + "legendLink": null + }, + { + "expr": "sum(rate(cortex_bucket_store_series_blocks_queried_sum{component=\"store-gateway\",level!~\"[0-4]\",cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"}[$__rate_interval]))", "format": "time_series", - "legendFormat": "blocks", + "legendFormat": "5+", "legendLink": null } ], - "title": "Blocks queried / sec", + "title": "Blocks queried / sec by compaction level", "type": "timeseries" }, { diff --git a/helm/dashboards/charts/private_dashboards_mz/dashboards/shared/private/mimir-reads.json b/helm/dashboards/charts/private_dashboards_mz/dashboards/shared/private/mimir-reads.json index 794cf435..29433a69 100644 --- a/helm/dashboards/charts/private_dashboards_mz/dashboards/shared/private/mimir-reads.json +++ b/helm/dashboards/charts/private_dashboards_mz/dashboards/shared/private/mimir-reads.json @@ -1348,7 +1348,7 @@ "span": 6, "targets": [ { - "expr": "sum (\n rate(thanos_memcached_operations_total{name=\"frontend-cache\", cluster_id=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])\n or ignoring(backend)\n rate(thanos_cache_operations_total{name=\"frontend-cache\", cluster_id=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])\n)\n", + "expr": "sum (\n rate(thanos_cache_operations_total{name=\"frontend-cache\", cluster_id=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\"}[$__rate_interval])\n)\n", "format": "time_series", "legendFormat": "Requests/s", "legendLink": null @@ -1374,7 +1374,6 @@ } }, "min": 0, - "noValue": 0, "thresholds": { "mode": "absolute", "steps": [ ] @@ -1385,6 +1384,7 @@ }, "id": 17, "links": [ ], + "nullPointMode": "null as zero", "options": { "legend": { "showLegend": true @@ -1397,29 +1397,44 @@ "span": 6, "targets": [ { - "expr": "histogram_quantile(0.99, sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_bucket{cluster_id=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_bucket{cluster_id=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])\n) by (le)) * 1e3\n", + "expr": "histogram_quantile(0.99, sum(rate(thanos_cache_operation_duration_seconds_bucket{cluster_id=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", - "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A" }, { - "expr": "histogram_quantile(0.50, sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_bucket{cluster_id=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_bucket{cluster_id=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])\n) by (le)) * 1e3\n", + "expr": "histogram_quantile(0.50, sum(rate(thanos_cache_operation_duration_seconds_bucket{cluster_id=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])) by (le)) * 1e3", "format": "time_series", - "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B" }, { - "expr": "sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_sum{cluster_id=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_sum{cluster_id=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])\n) * 1e3\n/\nsum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_count{cluster_id=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_count{cluster_id=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])\n)\n", + "expr": "sum(rate(thanos_cache_operation_duration_seconds_sum{cluster_id=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval])) * 1e3 / sum(rate(thanos_cache_operation_duration_seconds_count{cluster_id=~\"$cluster\", job=~\"($namespace)/((query-frontend.*|cortex|mimir|mimir-read.*))\", name=\"frontend-cache\"}[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", "refId": "C" } ], "title": "Latency", - "type": "timeseries" + "type": "timeseries", + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] } ], "repeat": null, @@ -2505,19 +2520,19 @@ "span": 4, "targets": [ { - "expr": "max by (scaletargetref_name) (\n kube_horizontalpodautoscaler_spec_max_replicas{cluster_id=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"mimir-querier\"}\n # Add the scaletargetref_name label which is more readable than \"kube-hpa-...\"\n + on (cluster_id, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n 0*kube_horizontalpodautoscaler_info{cluster_id=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"mimir-querier\"}\n)\n", + "expr": "max by (scaletargetref_name) (\n kube_horizontalpodautoscaler_spec_max_replicas{cluster_id=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"mimir-querier\"}\n # Add the scaletargetref_name label which is more readable than \"kube-hpa-...\"\n * on (cluster_id, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n group by (cluster_id, namespace, horizontalpodautoscaler, scaletargetref_name) (kube_horizontalpodautoscaler_info{cluster_id=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"mimir-querier\"})\n)\n", "format": "time_series", "legendFormat": "Max {{ scaletargetref_name }}", "legendLink": null }, { - "expr": "max by (scaletargetref_name) (\n kube_horizontalpodautoscaler_status_current_replicas{cluster_id=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"mimir-querier\"}\n # Add the scaletargetref_name label which is more readable than \"kube-hpa-...\"\n + on (cluster_id, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n 0*kube_horizontalpodautoscaler_info{cluster_id=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"mimir-querier\"}\n)\n", + "expr": "max by (scaletargetref_name) (\n kube_horizontalpodautoscaler_status_current_replicas{cluster_id=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"mimir-querier\"}\n # Add the scaletargetref_name label which is more readable than \"kube-hpa-...\"\n * on (cluster_id, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n group by (cluster_id, namespace, horizontalpodautoscaler, scaletargetref_name) (kube_horizontalpodautoscaler_info{cluster_id=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"mimir-querier\"})\n)\n", "format": "time_series", "legendFormat": "Current {{ scaletargetref_name }}", "legendLink": null }, { - "expr": "max by (scaletargetref_name) (\n kube_horizontalpodautoscaler_spec_min_replicas{cluster_id=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"mimir-querier\"}\n # Add the scaletargetref_name label which is more readable than \"kube-hpa-...\"\n + on (cluster_id, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n 0*kube_horizontalpodautoscaler_info{cluster_id=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"mimir-querier\"}\n)\n", + "expr": "max by (scaletargetref_name) (\n kube_horizontalpodautoscaler_spec_min_replicas{cluster_id=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"mimir-querier\"}\n # Add the scaletargetref_name label which is more readable than \"kube-hpa-...\"\n * on (cluster_id, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n group by (cluster_id, namespace, horizontalpodautoscaler, scaletargetref_name) (kube_horizontalpodautoscaler_info{cluster_id=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"mimir-querier\"})\n)\n", "format": "time_series", "legendFormat": "Min {{ scaletargetref_name }}", "legendLink": null @@ -2566,7 +2581,7 @@ "span": 4, "targets": [ { - "expr": "sum by (scaler) (\n label_replace(\n keda_scaler_metrics_value{cluster_id=~\"$cluster\", exported_namespace=~\"$namespace\"},\n \"namespace\", \"$1\", \"exported_namespace\", \"(.*)\"\n )\n /\n on(cluster_id, namespace, scaledObject, metric) group_left\n label_replace(label_replace(\n kube_horizontalpodautoscaler_spec_target_metric{cluster_id=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"mimir-querier\"},\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n ), \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"mimir-(.*)\")\n)\n", + "expr": "sum by (scaler) (\n label_replace(\n keda_scaler_metrics_value{cluster_id=~\"$cluster\", exported_namespace=~\"$namespace\"},\n \"namespace\", \"$1\", \"exported_namespace\", \"(.*)\"\n )\n /\n on(cluster_id, namespace, scaledObject, metric) group_left\n max by (cluster_id, namespace, scaledObject, metric) (\n label_replace(label_replace(\n kube_horizontalpodautoscaler_spec_target_metric{cluster_id=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"mimir-querier\"},\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n ), \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"mimir-(.*)\")\n )\n)\n", "format": "time_series", "legendFormat": "{{ scaler }}", "legendLink": null @@ -2615,7 +2630,7 @@ "span": 4, "targets": [ { - "expr": "sum by(cluster_id, namespace, scaler, metric, scaledObject) (\n label_replace(\n rate(keda_scaler_errors[$__rate_interval]),\n \"namespace\", \"$1\", \"exported_namespace\", \"(.+)\"\n )\n) +\non(cluster_id, namespace, metric, scaledObject) group_left\nlabel_replace(\n label_replace(\n kube_horizontalpodautoscaler_spec_target_metric{cluster_id=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"mimir-querier\"} * 0,\n \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"mimir-(.*)\"\n ),\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n)\n", + "expr": "sum by(cluster_id, namespace, metric, scaledObject, scaler) (\n label_replace(\n rate(keda_scaler_errors[$__rate_interval]),\n \"namespace\", \"$1\", \"exported_namespace\", \"(.+)\"\n )\n) +\non(cluster_id, namespace, metric, scaledObject) group_left\n # Using `max by ()` so that series churn doesn't break the promQL join\n max by (cluster_id, namespace, metric, scaledObject) (\n label_replace(\n label_replace(\n kube_horizontalpodautoscaler_spec_target_metric{cluster_id=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"mimir-querier\"} * 0,\n \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"mimir-(.*)\"\n ),\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n )\n )\n", "format": "time_series", "legendFormat": "{{scaler}} failures", "legendLink": null @@ -2950,7 +2965,7 @@ "span": 4, "targets": [ { - "expr": "sum by(operation) (\n # Backwards compatibility\n rate(\n thanos_memcached_operations_total{\n component=\"store-gateway\",\n name=\"index-cache\",\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"\n }[$__rate_interval]\n )\n or ignoring(backend)\n rate(\n thanos_cache_operations_total{\n component=\"store-gateway\",\n name=\"index-cache\",\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"\n }[$__rate_interval]\n )\n)\n", + "expr": "sum by(operation) (\n rate(\n thanos_cache_operations_total{\n component=\"store-gateway\",\n name=\"index-cache\",\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\"\n }[$__rate_interval]\n )\n)\n", "format": "time_series", "legendFormat": "{{operation}}", "legendLink": null @@ -2976,7 +2991,6 @@ } }, "min": 0, - "noValue": 0, "thresholds": { "mode": "absolute", "steps": [ ] @@ -2987,6 +3001,7 @@ }, "id": 33, "links": [ ], + "nullPointMode": "null as zero", "options": { "legend": { "showLegend": true @@ -2999,29 +3014,44 @@ "span": 4, "targets": [ { - "expr": "histogram_quantile(0.99, sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_bucket{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])\n) by (le)) * 1e3\n", + "expr": "histogram_quantile(0.99, sum(rate(thanos_cache_operation_duration_seconds_bucket{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])) by (le)) * 1e3", "format": "time_series", - "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A" }, { - "expr": "histogram_quantile(0.50, sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_bucket{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])\n) by (le)) * 1e3\n", + "expr": "histogram_quantile(0.50, sum(rate(thanos_cache_operation_duration_seconds_bucket{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])) by (le)) * 1e3", "format": "time_series", - "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B" }, { - "expr": "sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_sum{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_sum{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])\n) * 1e3\n/\nsum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_count{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_count{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])\n)\n", + "expr": "sum(rate(thanos_cache_operation_duration_seconds_sum{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval])) * 1e3 / sum(rate(thanos_cache_operation_duration_seconds_count{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"index-cache\"\n}\n[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", "refId": "C" } ], "title": "Latency (getmulti)", - "type": "timeseries" + "type": "timeseries", + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] }, { "datasource": "$datasource", @@ -3123,7 +3153,7 @@ "span": 4, "targets": [ { - "expr": "sum by(operation) (\n # Backwards compatibility\n rate(thanos_memcached_operations_total{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval])\n or ignoring(backend)\n rate(thanos_cache_operations_total{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval])\n)\n", + "expr": "sum by(operation) (\n rate(thanos_cache_operations_total{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval])\n)\n", "format": "time_series", "legendFormat": "{{operation}}", "legendLink": null @@ -3149,7 +3179,6 @@ } }, "min": 0, - "noValue": 0, "thresholds": { "mode": "absolute", "steps": [ ] @@ -3160,6 +3189,7 @@ }, "id": 36, "links": [ ], + "nullPointMode": "null as zero", "options": { "legend": { "showLegend": true @@ -3172,29 +3202,44 @@ "span": 4, "targets": [ { - "expr": "histogram_quantile(0.99, sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_bucket{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])\n) by (le)) * 1e3\n", + "expr": "histogram_quantile(0.99, sum(rate(thanos_cache_operation_duration_seconds_bucket{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])) by (le)) * 1e3", "format": "time_series", - "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A" }, { - "expr": "histogram_quantile(0.50, sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_bucket{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])\n) by (le)) * 1e3\n", + "expr": "histogram_quantile(0.50, sum(rate(thanos_cache_operation_duration_seconds_bucket{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])) by (le)) * 1e3", "format": "time_series", - "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B" }, { - "expr": "sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_sum{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_sum{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])\n) * 1e3\n/\nsum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_count{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_count{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])\n)\n", + "expr": "sum(rate(thanos_cache_operation_duration_seconds_sum{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval])) * 1e3 / sum(rate(thanos_cache_operation_duration_seconds_count{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n}\n[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", "refId": "C" } ], "title": "Latency (getmulti)", - "type": "timeseries" + "type": "timeseries", + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] }, { "datasource": "$datasource", @@ -3235,7 +3280,7 @@ "span": 4, "targets": [ { - "expr": "sum(\n # Backwards compatibility\n rate(thanos_cache_memcached_hits_total{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval])\n or\n rate(thanos_cache_hits_total{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval])\n)\n/\nsum(\n # Backwards compatibility\n rate(thanos_cache_memcached_requests_total{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval])\n or\n rate(thanos_cache_requests_total{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval])\n)\n", + "expr": "sum(\n rate(thanos_cache_hits_total{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval])\n)\n/\nsum(\n rate(thanos_cache_requests_total{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"chunks-cache\"\n }[$__rate_interval])\n)\n", "format": "time_series", "legendFormat": "items", "legendLink": null @@ -3295,7 +3340,7 @@ "span": 4, "targets": [ { - "expr": "sum by(operation) (\n # Backwards compatibility\n rate(thanos_memcached_operations_total{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n or ignoring(backend)\n rate(thanos_cache_operations_total{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n)\n", + "expr": "sum by(operation) (\n rate(thanos_cache_operations_total{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n)\n", "format": "time_series", "legendFormat": "{{operation}}", "legendLink": null @@ -3321,7 +3366,6 @@ } }, "min": 0, - "noValue": 0, "thresholds": { "mode": "absolute", "steps": [ ] @@ -3332,6 +3376,7 @@ }, "id": 39, "links": [ ], + "nullPointMode": "null as zero", "options": { "legend": { "showLegend": true @@ -3344,29 +3389,44 @@ "span": 4, "targets": [ { - "expr": "histogram_quantile(0.99, sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_bucket{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n) by (le)) * 1e3\n", + "expr": "histogram_quantile(0.99, sum(rate(thanos_cache_operation_duration_seconds_bucket{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])) by (le)) * 1e3", "format": "time_series", - "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A" }, { - "expr": "histogram_quantile(0.50, sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_bucket{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n) by (le)) * 1e3\n", + "expr": "histogram_quantile(0.50, sum(rate(thanos_cache_operation_duration_seconds_bucket{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])) by (le)) * 1e3", "format": "time_series", - "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B" }, { - "expr": "sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_sum{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_sum{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n) * 1e3\n/\nsum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_count{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_count{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n)\n", + "expr": "sum(rate(thanos_cache_operation_duration_seconds_sum{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])) * 1e3 / sum(rate(thanos_cache_operation_duration_seconds_count{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n operation=\"getmulti\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n}\n[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", "refId": "C" } ], "title": "Latency (getmulti)", - "type": "timeseries" + "type": "timeseries", + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] }, { "datasource": "$datasource", @@ -3407,7 +3467,7 @@ "span": 4, "targets": [ { - "expr": "sum(\n # Backwards compatibility\n rate(thanos_cache_memcached_hits_total{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n or\n rate(thanos_cache_hits_total{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n)\n/\nsum(\n # Backwards compatibility\n rate(thanos_cache_memcached_requests_total{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n or\n rate(thanos_cache_requests_total{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n)\n", + "expr": "sum(\n rate(thanos_cache_hits_total{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n)\n/\nsum(\n rate(thanos_cache_requests_total{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((store-gateway.*|cortex|mimir|mimir-backend.*))\",\n component=\"store-gateway\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n)\n", "format": "time_series", "legendFormat": "items", "legendLink": null @@ -3467,7 +3527,7 @@ "span": 4, "targets": [ { - "expr": "sum by(operation) (\n # Backwards compatibility\n rate(thanos_memcached_operations_total{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n or ignoring(backend)\n rate(thanos_cache_operations_total{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n)\n", + "expr": "sum by(operation) (\n rate(thanos_cache_operations_total{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n)\n", "format": "time_series", "legendFormat": "{{operation}}", "legendLink": null @@ -3493,7 +3553,6 @@ } }, "min": 0, - "noValue": 0, "thresholds": { "mode": "absolute", "steps": [ ] @@ -3504,6 +3563,7 @@ }, "id": 42, "links": [ ], + "nullPointMode": "null as zero", "options": { "legend": { "showLegend": true @@ -3516,29 +3576,44 @@ "span": 4, "targets": [ { - "expr": "histogram_quantile(0.99, sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_bucket{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n) by (le)) * 1e3\n", + "expr": "histogram_quantile(0.99, sum(rate(thanos_cache_operation_duration_seconds_bucket{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])) by (le)) * 1e3", "format": "time_series", - "intervalFactor": 2, "legendFormat": "99th Percentile", "refId": "A" }, { - "expr": "histogram_quantile(0.50, sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_bucket{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_bucket{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n) by (le)) * 1e3\n", + "expr": "histogram_quantile(0.50, sum(rate(thanos_cache_operation_duration_seconds_bucket{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])) by (le)) * 1e3", "format": "time_series", - "intervalFactor": 2, "legendFormat": "50th Percentile", "refId": "B" }, { - "expr": "sum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_sum{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_sum{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n) * 1e3\n/\nsum(\n # Backwards compatibility\n rate(thanos_memcached_operation_duration_seconds_count{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n or\n rate(thanos_cache_operation_duration_seconds_count{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])\n)\n", + "expr": "sum(rate(thanos_cache_operation_duration_seconds_sum{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval])) * 1e3 / sum(rate(thanos_cache_operation_duration_seconds_count{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n operation=\"getmulti\",\n component=\"querier\",\n name=\"metadata-cache\"\n}\n[$__rate_interval]))", "format": "time_series", - "intervalFactor": 2, "legendFormat": "Average", "refId": "C" } ], "title": "Latency (getmulti)", - "type": "timeseries" + "type": "timeseries", + "yaxes": [ + { + "format": "ms", + "label": null, + "logBase": 1, + "max": null, + "min": 0, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": false + } + ] }, { "datasource": "$datasource", @@ -3579,7 +3654,7 @@ "span": 4, "targets": [ { - "expr": "sum(\n # Backwards compatibility\n rate(thanos_cache_memcached_hits_total{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n or\n rate(thanos_cache_hits_total{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n)\n/\nsum(\n # Backwards compatibility\n rate(thanos_cache_memcached_requests_total{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n or\n rate(thanos_cache_requests_total{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n)\n", + "expr": "sum(\n rate(thanos_cache_hits_total{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n)\n/\nsum(\n rate(thanos_cache_requests_total{\n cluster_id=~\"$cluster\", job=~\"($namespace)/((querier.*|cortex|mimir|mimir-read.*))\",\n component=\"querier\",\n name=\"metadata-cache\"\n }[$__rate_interval])\n)\n", "format": "time_series", "legendFormat": "items", "legendLink": null diff --git a/helm/dashboards/charts/private_dashboards_mz/dashboards/shared/private/mimir-tenants.json b/helm/dashboards/charts/private_dashboards_mz/dashboards/shared/private/mimir-tenants.json index 54e5eb1c..75287df7 100644 --- a/helm/dashboards/charts/private_dashboards_mz/dashboards/shared/private/mimir-tenants.json +++ b/helm/dashboards/charts/private_dashboards_mz/dashboards/shared/private/mimir-tenants.json @@ -2660,7 +2660,7 @@ "query": "label_values(cortex_ingester_active_series{cluster_id=~\"$cluster\", namespace=~\"$namespace\"}, user)", "refresh": 1, "regex": "", - "sort": 1, + "sort": 7, "tagValuesQuery": "", "tags": [ ], "tagsQuery": "", diff --git a/helm/dashboards/charts/private_dashboards_mz/dashboards/shared/private/mimir-writes.json b/helm/dashboards/charts/private_dashboards_mz/dashboards/shared/private/mimir-writes.json index 24854196..c87d783d 100644 --- a/helm/dashboards/charts/private_dashboards_mz/dashboards/shared/private/mimir-writes.json +++ b/helm/dashboards/charts/private_dashboards_mz/dashboards/shared/private/mimir-writes.json @@ -1216,19 +1216,19 @@ "span": 3, "targets": [ { - "expr": "max by (scaletargetref_name) (\n kube_horizontalpodautoscaler_spec_max_replicas{cluster_id=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"mimir-distributor\"}\n # Add the scaletargetref_name label for readability\n + on (cluster_id, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n 0*kube_horizontalpodautoscaler_info{cluster_id=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"mimir-distributor\"}\n)\n", + "expr": "max by (scaletargetref_name) (\n kube_horizontalpodautoscaler_spec_max_replicas{cluster_id=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"mimir-distributor\"}\n # Add the scaletargetref_name label for readability\n * on (cluster_id, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n group by (cluster_id, namespace, horizontalpodautoscaler, scaletargetref_name) (kube_horizontalpodautoscaler_info{cluster_id=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"mimir-distributor\"})\n)\n", "format": "time_series", "legendFormat": "Max {{ scaletargetref_name }}", "legendLink": null }, { - "expr": "max by (scaletargetref_name) (\n kube_horizontalpodautoscaler_status_current_replicas{cluster_id=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"mimir-distributor\"}\n # Add the scaletargetref_name label for readability\n + on (cluster_id, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n 0*kube_horizontalpodautoscaler_info{cluster_id=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"mimir-distributor\"}\n)\n", + "expr": "max by (scaletargetref_name) (\n kube_horizontalpodautoscaler_status_current_replicas{cluster_id=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"mimir-distributor\"}\n # Add the scaletargetref_name label for readability\n * on (cluster_id, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n group by (cluster_id, namespace, horizontalpodautoscaler, scaletargetref_name) (kube_horizontalpodautoscaler_info{cluster_id=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"mimir-distributor\"})\n)\n", "format": "time_series", "legendFormat": "Current {{ scaletargetref_name }}", "legendLink": null }, { - "expr": "max by (scaletargetref_name) (\n kube_horizontalpodautoscaler_spec_min_replicas{cluster_id=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"mimir-distributor\"}\n # Add the scaletargetref_name label for readability\n + on (cluster_id, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n 0*kube_horizontalpodautoscaler_info{cluster_id=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"mimir-distributor\"}\n)\n", + "expr": "max by (scaletargetref_name) (\n kube_horizontalpodautoscaler_spec_min_replicas{cluster_id=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"mimir-distributor\"}\n # Add the scaletargetref_name label for readability\n * on (cluster_id, namespace, horizontalpodautoscaler) group_left (scaletargetref_name)\n group by (cluster_id, namespace, horizontalpodautoscaler, scaletargetref_name) (kube_horizontalpodautoscaler_info{cluster_id=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"mimir-distributor\"})\n)\n", "format": "time_series", "legendFormat": "Min {{ scaletargetref_name }}", "legendLink": null @@ -1277,7 +1277,7 @@ "span": 3, "targets": [ { - "expr": "sum by (scaler) (\n label_replace(\n keda_scaler_metrics_value{cluster_id=~\"$cluster\", exported_namespace=~\"$namespace\", scaler=~\".*cpu.*\"},\n \"namespace\", \"$1\", \"exported_namespace\", \"(.*)\"\n )\n /\n on(cluster_id, namespace, scaledObject, metric) group_left label_replace(\n label_replace(\n kube_horizontalpodautoscaler_spec_target_metric{cluster_id=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"mimir-distributor\"},\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n ),\n \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"mimir-(.*)\"\n )\n)\n", + "expr": "sum by (scaler) (\n # Using `max by ()` so that series churn doesn't break the promQL join\n max by (cluster_id, namespace, scaledObject, metric, scaler) (\n label_replace(\n keda_scaler_metrics_value{cluster_id=~\"$cluster\", exported_namespace=~\"$namespace\", scaler=~\".*cpu.*\"},\n \"namespace\", \"$1\", \"exported_namespace\", \"(.*)\"\n )\n )\n /\n on(cluster_id, namespace, scaledObject, metric) group_left\n # Using `max by ()` so that series churn doesn't break the promQL join\n max by (cluster_id, namespace, scaledObject, metric) (\n label_replace(\n label_replace(\n kube_horizontalpodautoscaler_spec_target_metric{cluster_id=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"mimir-distributor\"},\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n ),\n \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"mimir-(.*)\"\n )\n )\n)\n", "format": "time_series", "legendFormat": "{{ scaler }}", "legendLink": null @@ -1326,7 +1326,7 @@ "span": 3, "targets": [ { - "expr": "sum by (scaler) (\n label_replace(\n keda_scaler_metrics_value{cluster_id=~\"$cluster\", exported_namespace=~\"$namespace\", scaler=~\".*memory.*\"},\n \"namespace\", \"$1\", \"exported_namespace\", \"(.*)\"\n )\n /\n on(cluster_id, namespace, scaledObject, metric) group_left label_replace(\n label_replace(\n kube_horizontalpodautoscaler_spec_target_metric{cluster_id=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"mimir-distributor\"},\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n ),\n \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"mimir-(.*)\"\n )\n)\n", + "expr": "sum by (scaler) (\n # Using `max by ()` so that series churn doesn't break the promQL join\n max by (cluster_id, namespace, scaledObject, metric, scaler) (\n label_replace(\n keda_scaler_metrics_value{cluster_id=~\"$cluster\", exported_namespace=~\"$namespace\", scaler=~\".*memory.*\"},\n \"namespace\", \"$1\", \"exported_namespace\", \"(.*)\"\n )\n )\n /\n on(cluster_id, namespace, scaledObject, metric) group_left\n # Using `max by ()` so that series churn doesn't break the promQL join\n max by (cluster_id, namespace, scaledObject, metric) (\n label_replace(\n label_replace(\n kube_horizontalpodautoscaler_spec_target_metric{cluster_id=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"mimir-distributor\"},\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n ),\n \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"mimir-(.*)\"\n )\n )\n)\n", "format": "time_series", "legendFormat": "{{ scaler }}", "legendLink": null @@ -1375,7 +1375,7 @@ "span": 3, "targets": [ { - "expr": "sum by(cluster_id, namespace, scaler, metric, scaledObject) (\n label_replace(\n rate(keda_scaler_errors[$__rate_interval]),\n \"namespace\", \"$1\", \"exported_namespace\", \"(.+)\"\n )\n) +\non(cluster_id, namespace, metric, scaledObject) group_left\nlabel_replace(\n label_replace(\n kube_horizontalpodautoscaler_spec_target_metric{cluster_id=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"mimir-distributor\"} * 0,\n \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"mimir-(.*)\"\n ),\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n)\n", + "expr": "sum by(cluster_id, namespace, metric, scaledObject, scaler) (\n label_replace(\n rate(keda_scaler_errors[$__rate_interval]),\n \"namespace\", \"$1\", \"exported_namespace\", \"(.+)\"\n )\n) +\non(cluster_id, namespace, metric, scaledObject) group_left\n # Using `max by ()` so that series churn doesn't break the promQL join\n max by (cluster_id, namespace, metric, scaledObject) (\n label_replace(\n label_replace(\n kube_horizontalpodautoscaler_spec_target_metric{cluster_id=~\"$cluster\", namespace=~\"$namespace\", horizontalpodautoscaler=~\"mimir-distributor\"} * 0,\n \"scaledObject\", \"$1\", \"horizontalpodautoscaler\", \"mimir-(.*)\"\n ),\n \"metric\", \"$1\", \"metric_name\", \"(.+)\"\n )\n )\n", "format": "time_series", "legendFormat": "{{scaler}} failures", "legendLink": null diff --git a/loki/README.md b/loki/README.md deleted file mode 100644 index 55e09b25..00000000 --- a/loki/README.md +++ /dev/null @@ -1,97 +0,0 @@ -# Intro - -Dashboards were grabbed from https://github.com/grafana/loki/tree/main/production/loki-mixin-compiled-ssd/dashboards / tag: `helm-loki-5.26.0` -then tuned to fit our metrics. - -## Changes - -Most of the changes were these: - -* Add cluster_id variable for filtering - add: - ``` - { - "datasource": "$datasource", - "hide": 0, - "includeAll": false, - "label": "Kube cluster", - "multi": false, - "name": "cluster_id", - "options": [ ], - "query": "label_values(loki_build_info, cluster_id)", - "refresh": 1, - "regex": "", - "sort": 2, - "tagValuesQuery": "", - "tags": [ ], - "tagsQuery": "", - "type": "query", - "useTags": false - }, - ``` -* update all `"expr":` statements with `cluster_id=\"$cluster_id\", ` (except metrics from recording rules for now) -* change UID -* non-loki metrics: remove `cluster` filtering -* Logs: update job name, and add `component` filtering -* loki-deletion logs: add `loki_datasource` datasource: - ``` - { - "hide": 0, - "label": null, - "name": "loki_datasource", - "options": [ ], - "query": "loki", - "refresh": 1, - "regex": "", - "type": "datasource" - }, - ``` - -## Notes on specific files: -* loki-chunks.json - ok -* loki-deletion.json - ok -* loki-logs - ok -* loki-mixin-recording-rules - requires `loki_ruler_wal_.*` metrics, which we don't have. -* loki-operational - ok -* loki-read - ok - no boltdb-shipper data, because it's in `write` -* loki-read-resources - ok, but no disk data (because not sts?) -* loki-writes - ok -* loki-writes-resources - ok, but no disk data - -## Diffs - -Generating diffs, for future reference -``` -LOKI_GIT="/home/herve/github/loki" -DASHBOARDS="/home/herve/github/giantswarm/dashboards/helm/dashboards/dashboards/shared/private" -mkdir -p diffs -for dashboard in "$LOKI_GIT"/production/loki-mixin-compiled-ssd/dashboards/*.json; do diff "$dashboard" "$DASHBOARDS"/"$(basename "$dashboard")" > diffs/"$(basename "$dashboard")".diff; done -``` - -# Extra changes - -There's been some extra changes done interactively with Grafana UI. - -## Loki Overview - -* Added a `Backend Path` panel -* Added `disk usage` to Write and Backend path panel -* Added `total pods` to Write, Read and Backend panels - -# Loki canary - -This one was generated from the [mixins](https://github.com/grafana/loki/tree/main/production/loki-mixin) following these steps: - -* Update config (mixin-ssd.libsonnet) with: - ``` - { - _config+:: { - canary+: { - enabled: true, - }, - }, - } - ``` -* Generate with `mixtool generate all mixin-ssd.libsonnet` -* Dashboard is `loki-canary.json` -* Then we just changed the `uid` of the dashboard to `loki-canary` diff --git a/loki/diffs/loki-chunks.json.diff b/loki/diffs/loki-chunks.json.diff deleted file mode 100644 index dd2088e7..00000000 --- a/loki/diffs/loki-chunks.json.diff +++ /dev/null @@ -1,128 +0,0 @@ -61c61 -< "expr": "sum(loki_ingester_memory_chunks{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"})", ---- -> "expr": "sum(loki_ingester_memory_chunks{cluster_id=\"$cluster_id\", cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"})", -137c137 -< "expr": "sum(loki_ingester_memory_chunks{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}) / sum(loki_ingester_memory_streams{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"})", ---- -> "expr": "sum(loki_ingester_memory_chunks{cluster_id=\"$cluster_id\", cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}) / sum(loki_ingester_memory_streams{cluster_id=\"$cluster_id\", cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"})", -225c225 -< "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_utilization_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1", ---- -> "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_utilization_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1", -233c233 -< "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_utilization_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1", ---- -> "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_utilization_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1", -241c241 -< "expr": "sum(rate(loki_ingester_chunk_utilization_sum{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) * 1 / sum(rate(loki_ingester_chunk_utilization_count{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]))", ---- -> "expr": "sum(rate(loki_ingester_chunk_utilization_sum{cluster_id=\"$cluster_id\", cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) * 1 / sum(rate(loki_ingester_chunk_utilization_count{cluster_id=\"$cluster_id\", cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]))", -317c317 -< "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_age_seconds_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1e3", ---- -> "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_age_seconds_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1e3", -325c325 -< "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_age_seconds_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1e3", ---- -> "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_age_seconds_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1e3", -333c333 -< "expr": "sum(rate(loki_ingester_chunk_age_seconds_sum{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) * 1e3 / sum(rate(loki_ingester_chunk_age_seconds_count{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]))", ---- -> "expr": "sum(rate(loki_ingester_chunk_age_seconds_sum{cluster_id=\"$cluster_id\", cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) * 1e3 / sum(rate(loki_ingester_chunk_age_seconds_count{cluster_id=\"$cluster_id\", cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]))", -421c421 -< "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_entries_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1", ---- -> "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_entries_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1", -429c429 -< "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_entries_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1", ---- -> "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_entries_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)) * 1", -437c437 -< "expr": "sum(rate(loki_ingester_chunk_entries_sum{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) * 1 / sum(rate(loki_ingester_chunk_entries_count{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]))", ---- -> "expr": "sum(rate(loki_ingester_chunk_entries_sum{cluster_id=\"$cluster_id\", cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) * 1 / sum(rate(loki_ingester_chunk_entries_count{cluster_id=\"$cluster_id\", cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]))", -513c513 -< "expr": "sum(rate(loki_chunk_store_index_entries_per_chunk_sum{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[5m])) / sum(rate(loki_chunk_store_index_entries_per_chunk_count{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[5m]))", ---- -> "expr": "sum(rate(loki_chunk_store_index_entries_per_chunk_sum{cluster_id=\"$cluster_id\", cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[5m])) / sum(rate(loki_chunk_store_index_entries_per_chunk_count{cluster_id=\"$cluster_id\", cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[5m]))", -601c601 -< "expr": "cortex_ingester_flush_queue_length{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}", ---- -> "expr": "cortex_ingester_flush_queue_length{cluster_id=\"$cluster_id\", cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}", -685c685 -< "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_ingester_chunk_age_seconds_count{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", ---- -> "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_ingester_chunk_age_seconds_count{cluster_id=\"$cluster_id\", cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", -773c773 -< "expr": "sum(rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]))", ---- -> "expr": "sum(rate(loki_ingester_chunks_flushed_total{cluster_id=\"$cluster_id\", cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]))", -849c849 -< "expr": "sum by (reason) (rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) / ignoring(reason) group_left sum(rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]))", ---- -> "expr": "sum by (reason) (rate(loki_ingester_chunks_flushed_total{cluster_id=\"$cluster_id\", cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) / ignoring(reason) group_left sum(rate(loki_ingester_chunks_flushed_total{cluster_id=\"$cluster_id\", cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]))", -929c929 -< "expr": "sum by (le) (rate(loki_ingester_chunk_utilization_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]))", ---- -> "expr": "sum by (le) (rate(loki_ingester_chunk_utilization_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval]))", -991c991 -< "expr": "sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)", ---- -> "expr": "sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (le)", -1061c1061 -< "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[1m])) by (le))", ---- -> "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[1m])) by (le))", -1069c1069 -< "expr": "histogram_quantile(0.90, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[1m])) by (le))", ---- -> "expr": "histogram_quantile(0.90, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[1m])) by (le))", -1077c1077 -< "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[1m])) by (le))", ---- -> "expr": "histogram_quantile(0.50, sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[1m])) by (le))", -1165c1165 -< "expr": "histogram_quantile(0.5, sum(rate(loki_ingester_chunk_bounds_hours_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[5m])) by (le))", ---- -> "expr": "histogram_quantile(0.5, sum(rate(loki_ingester_chunk_bounds_hours_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[5m])) by (le))", -1173c1173 -< "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_bounds_hours_bucket{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[5m])) by (le))", ---- -> "expr": "histogram_quantile(0.99, sum(rate(loki_ingester_chunk_bounds_hours_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[5m])) by (le))", -1181c1181 -< "expr": "sum(rate(loki_ingester_chunk_bounds_hours_sum{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[5m])) / sum(rate(loki_ingester_chunk_bounds_hours_count{cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[5m]))", ---- -> "expr": "sum(rate(loki_ingester_chunk_bounds_hours_sum{cluster_id=\"$cluster_id\", cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[5m])) / sum(rate(loki_ingester_chunk_bounds_hours_count{cluster_id=\"$cluster_id\", cluster=\"$cluster\", job=~\"$namespace/(loki|enterprise-logs)-write\"}[5m]))", -1255a1256,1273 -> "datasource": "$datasource", -> "hide": 0, -> "includeAll": false, -> "label": "Kube cluster", -> "multi": false, -> "name": "cluster_id", -> "options": [ ], -> "query": "label_values(loki_build_info, cluster_id)", -> "refresh": 1, -> "regex": "", -> "sort": 2, -> "tagValuesQuery": "", -> "tags": [ ], -> "tagsQuery": "", -> "type": "query", -> "useTags": false -> }, -> { -1291c1309 -< "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)", ---- -> "query": "label_values(loki_build_info{cluster_id=\"$cluster_id\", cluster=~\"$cluster\"}, namespace)", -1334c1352 -< "uid": "chunks", ---- -> "uid": "loki-chunks", -1336c1354 -< } -\ No newline at end of file ---- -> } diff --git a/loki/diffs/loki-deletion.json.diff b/loki/diffs/loki-deletion.json.diff deleted file mode 100644 index c733e358..00000000 --- a/loki/diffs/loki-deletion.json.diff +++ /dev/null @@ -1,87 +0,0 @@ -62c62 -< "expr": "sum(loki_compactor_pending_delete_requests_count{cluster=~\"$cluster\", namespace=~\"$namespace\"})", ---- -> "expr": "sum(loki_compactor_pending_delete_requests_count{cluster_id=\"$cluster_id\", cluster=~\"$cluster\", namespace=~\"$namespace\"})", -138c138 -< "expr": "max(loki_compactor_oldest_pending_delete_request_age_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"})", ---- -> "expr": "max(loki_compactor_oldest_pending_delete_request_age_seconds{cluster_id=\"$cluster_id\", cluster=~\"$cluster\", namespace=~\"$namespace\"})", -225c225 -< "expr": "(loki_compactor_delete_requests_received_total{cluster=~\"$cluster\", namespace=~\"$namespace\"} or on() vector(0)) - on () (loki_compactor_delete_requests_processed_total{cluster=~\"$cluster\", namespace=~\"$namespace\"} or on () vector(0))", ---- -> "expr": "(loki_compactor_delete_requests_received_total{cluster_id=\"$cluster_id\", cluster=~\"$cluster\", namespace=~\"$namespace\"} or on() vector(0)) - on () (loki_compactor_delete_requests_processed_total{cluster_id=\"$cluster_id\", cluster=~\"$cluster\", namespace=~\"$namespace\"} or on () vector(0))", -301c301 -< "expr": "sum(increase(loki_compactor_delete_requests_received_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[1d]))", ---- -> "expr": "sum(increase(loki_compactor_delete_requests_received_total{cluster_id=\"$cluster_id\", cluster=~\"$cluster\", namespace=~\"$namespace\"}[1d]))", -377c377 -< "expr": "sum(increase(loki_compactor_delete_requests_processed_total{cluster=~\"$cluster\", namespace=~\"$namespace\"}[1d]))", ---- -> "expr": "sum(increase(loki_compactor_delete_requests_processed_total{cluster_id=\"$cluster_id\", cluster=~\"$cluster\", namespace=~\"$namespace\"}[1d]))", -541c541 -< "expr": "go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"compactor\"} / 1024 / 1024 ", ---- -> "expr": "go_memstats_heap_inuse_bytes{cluster_id=\"$cluster_id\", cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", job=\"loki/loki-backend\"} / 1024 / 1024 ", -617c617 -< "expr": "loki_boltdb_shipper_compact_tables_operation_duration_seconds{cluster=~\"$cluster\", namespace=~\"$namespace\"}", ---- -> "expr": "loki_boltdb_shipper_compact_tables_operation_duration_seconds{cluster_id=\"$cluster_id\", cluster=~\"$cluster\", namespace=~\"$namespace\"}", -705c705 -< "expr": "sum(increase(loki_compactor_load_pending_requests_attempts_total{status=\"fail\", cluster=~\"$cluster\", namespace=~\"$namespace\"}[1h]))", ---- -> "expr": "sum(increase(loki_compactor_load_pending_requests_attempts_total{status=\"fail\", cluster_id=\"$cluster_id\", cluster=~\"$cluster\", namespace=~\"$namespace\"}[1h]))", -781c781 -< "expr": "sum(rate(loki_compactor_deleted_lines{cluster=~\"$cluster\",job=~\"$namespace/(loki|enterprise-logs)-read\"}[$__rate_interval])) by (user)", ---- -> "expr": "sum(rate(loki_compactor_deleted_lines{cluster_id=\"$cluster_id\", cluster=~\"$cluster\",job=~\"$namespace/(loki|enterprise-logs)-read\"}[$__rate_interval])) by (user)", -843c843 -< "expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"compactor\"} |~ \"Started processing delete request|delete request for user marked as processed\" | logfmt | line_format \"{{.ts}} user={{.user}} delete_request_id={{.delete_request_id}} msg={{.msg}}\" ", ---- -> "expr": "{cluster_id=\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", component=\"backend\"} |~ \"Started processing delete request|delete request for user marked as processed\" | logfmt | line_format \"{{.ts}} user={{.user}} delete_request_id={{.delete_request_id}} msg={{.msg}}\" ", -856c856 -< "expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"compactor\"} |~ \"delete request for user added\" | logfmt | line_format \"{{.ts}} user={{.user}} query='{{.query}}'\"", ---- -> "expr": "{cluster_id=\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", component=\"backend\"} |~ \"delete request for user added\" | logfmt | line_format \"{{.ts}} user={{.user}} query='{{.query}}'\"", -893a894,903 -> "hide": 0, -> "label": null, -> "name": "loki_datasource", -> "options": [ ], -> "query": "loki", -> "refresh": 1, -> "regex": "", -> "type": "datasource" -> }, -> { -916a927,944 -> "datasource": "$datasource", -> "hide": 0, -> "includeAll": false, -> "label": "Kube cluster", -> "multi": false, -> "name": "cluster_id", -> "options": [ ], -> "query": "label_values(loki_build_info, cluster_id)", -> "refresh": 1, -> "regex": "", -> "sort": 2, -> "tagValuesQuery": "", -> "tags": [ ], -> "tagsQuery": "", -> "type": "query", -> "useTags": false -> }, -> { -929c957 -< "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)", ---- -> "query": "label_values(loki_build_info{cluster_id=\"$cluster_id\", cluster=~\"$cluster\"}, namespace)", -972c1000 -< "uid": "deletion", ---- -> "uid": "loki-deletion", -974c1002 -< } -\ No newline at end of file ---- -> } diff --git a/loki/diffs/loki-logs.json.diff b/loki/diffs/loki-logs.json.diff deleted file mode 100644 index 681d90f3..00000000 --- a/loki/diffs/loki-logs.json.diff +++ /dev/null @@ -1,84 +0,0 @@ -66c66 -< "expr": "sum(go_goroutines{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"})", ---- -> "expr": "sum(go_goroutines{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"})", -152c152 -< "expr": "sum(go_gc_duration_seconds{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"}) by (quantile)", ---- -> "expr": "sum(go_gc_duration_seconds{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"}) by (quantile)", -239c239 -< "expr": "sum(rate(container_cpu_usage_seconds_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"}[5m]))", ---- -> "expr": "sum(rate(container_cpu_usage_seconds_total{cluster_id=\"$cluster_id\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"}[5m]))", -325c325 -< "expr": "sum(container_memory_working_set_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"})", ---- -> "expr": "sum(container_memory_working_set_bytes{cluster_id=\"$cluster_id\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"})", -411c411 -< "expr": "sum(rate(container_network_transmit_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"}[5m]))", ---- -> "expr": "sum(rate(container_network_transmit_bytes_total{cluster_id=\"$cluster_id\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"}[5m]))", -497c497 -< "expr": "sum(rate(container_network_receive_bytes_total{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"}[5m]))", ---- -> "expr": "sum(rate(container_network_receive_bytes_total{cluster_id=\"$cluster_id\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\"}[5m]))", -583c583 -< "expr": "increase(kube_pod_container_status_last_terminated_reason{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"}[30m]) > 0", ---- -> "expr": "increase(kube_pod_container_status_last_terminated_reason{cluster_id=\"$cluster_id\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"}[30m]) > 0", -670c670 -< "expr": "sum(rate(promtail_custom_bad_words_total{cluster=\"$cluster\", exported_namespace=\"$namespace\", exported_pod=~\"$deployment.*\", exported_pod=~\"$pod\", container=~\"$container\"}[5m])) by (level)", ---- -> "expr": "sum(rate(promtail_custom_bad_words_total{cluster_id=\"$cluster_id\", cluster=\"$cluster\", exported_namespace=\"$namespace\", exported_pod=~\"$deployment.*\", exported_pod=~\"$pod\", container=~\"$container\"}[5m])) by (level)", -774c774 -< "expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\" } |logfmt| level=\"$level\" |= \"$filter\" [5m])) by (level)", ---- -> "expr": "sum(rate({cluster_id=\"$cluster_id\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\" } |logfmt| level=~\"$level\" |= \"$filter\" [5m])) by (level)", -839c839 -< "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"} | logfmt | level=\"$level\" |= \"$filter\"", ---- -> "expr": "{cluster_id=\"$cluster_id\", namespace=\"$namespace\", pod=~\"$deployment.*\", pod=~\"$pod\", container=~\"$container\"} | logfmt | level=~\"$level\" |= \"$filter\"", -895a896,913 -> "datasource": "$datasource", -> "hide": 0, -> "includeAll": false, -> "label": "Kube cluster", -> "multi": false, -> "name": "cluster_id", -> "options": [ ], -> "query": "label_values(loki_build_info, cluster_id)", -> "refresh": 1, -> "regex": "", -> "sort": 2, -> "tagValuesQuery": "", -> "tags": [ ], -> "tagsQuery": "", -> "type": "query", -> "useTags": false -> }, -> { -908c926 -< "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)", ---- -> "query": "label_values(loki_build_info{cluster_id=\"$cluster_id\", cluster=~\"$cluster\"}, namespace)", -938c956 -< "query": "label_values(kube_deployment_created{cluster=\"$cluster\", namespace=\"$namespace\"}, deployment)", ---- -> "query": "label_values(kube_deployment_created{cluster_id=\"$cluster_id\", namespace=\"$namespace\"}, deployment)", -958c976 -< "query": "label_values(kube_pod_container_info{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$deployment.*\"}, pod)", ---- -> "query": "label_values(kube_pod_container_info{cluster_id=\"$cluster_id\", namespace=\"$namespace\", pod=~\"$deployment.*\"}, pod)", -978c996 -< "query": "label_values(kube_pod_container_info{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"$pod\", pod=~\"$deployment.*\"}, container)", ---- -> "query": "label_values(kube_pod_container_info{cluster_id=\"$cluster_id\", namespace=\"$namespace\", pod=~\"$pod\", pod=~\"$deployment.*\"}, container)", -1070c1088 -< "uid": "logs", ---- -> "uid": "loki-logs", -1072c1090 -< } -\ No newline at end of file ---- -> } diff --git a/loki/diffs/loki-mixin-recording-rules.json.diff b/loki/diffs/loki-mixin-recording-rules.json.diff deleted file mode 100644 index a98cc002..00000000 --- a/loki/diffs/loki-mixin-recording-rules.json.diff +++ /dev/null @@ -1,64 +0,0 @@ -78c78 -< "expr": "sum(loki_ruler_wal_appender_ready) by (pod, tenant) == 0", ---- -> "expr": "sum(loki_ruler_wal_appender_ready{cluster_id=\"$cluster_id\"}) by (pod, tenant) == 0", -162c162 -< "expr": "sum(rate(loki_ruler_wal_samples_appended_total{tenant=~\"${tenant}\"}[$__rate_interval])) by (tenant) > 0", ---- -> "expr": "sum(rate(loki_ruler_wal_samples_appended_total{cluster_id=\"$cluster_id\", tenant=~\"${tenant}\"}[$__rate_interval])) by (tenant) > 0", -245c245 -< "expr": "sum(rate(loki_ruler_wal_storage_created_series_total{tenant=~\"${tenant}\"}[$__rate_interval])) by (tenant) > 0", ---- -> "expr": "sum(rate(loki_ruler_wal_storage_created_series_total{cluster_id=\"$cluster_id\", tenant=~\"${tenant}\"}[$__rate_interval])) by (tenant) > 0", -328c328 -< "expr": "loki_ruler_wal_prometheus_remote_storage_highest_timestamp_in_seconds{tenant=~\"${tenant}\"}\n- on (tenant)\n (\n loki_ruler_wal_prometheus_remote_storage_queue_highest_sent_timestamp_seconds{tenant=~\"${tenant}\"}\n or vector(0)\n )", ---- -> "expr": "loki_ruler_wal_prometheus_remote_storage_highest_timestamp_in_seconds{cluster_id=\"$cluster_id\", tenant=~\"${tenant}\"}\n- on (tenant)\n (\n loki_ruler_wal_prometheus_remote_storage_queue_highest_sent_timestamp_seconds{cluster_id=\"$cluster_id\", tenant=~\"${tenant}\"}\n or vector(0)\n )", -411c411 -< "expr": "sum(rate(loki_ruler_wal_prometheus_remote_storage_samples_total{tenant=~\"${tenant}\"}[$__rate_interval])) by (tenant) > 0", ---- -> "expr": "sum(rate(loki_ruler_wal_prometheus_remote_storage_samples_total{cluster_id=\"$cluster_id\", tenant=~\"${tenant}\"}[$__rate_interval])) by (tenant) > 0", -495c495 -< "expr": "sum by (tenant) (loki_ruler_wal_disk_size{tenant=~\"${tenant}\"})", ---- -> "expr": "sum by (tenant) (loki_ruler_wal_disk_size{cluster_id=\"$cluster_id\", tenant=~\"${tenant}\"})", -578c578 -< "expr": "max(loki_ruler_wal_prometheus_remote_storage_samples_pending{tenant=~\"${tenant}\"}) by (tenant,pod) > 0", ---- -> "expr": "max(loki_ruler_wal_prometheus_remote_storage_samples_pending{cluster_id=\"$cluster_id\", tenant=~\"${tenant}\"}) by (tenant,pod) > 0", -634a635,652 -> "datasource": "$datasource", -> "hide": 0, -> "includeAll": false, -> "label": "Kube cluster", -> "multi": false, -> "name": "cluster_id", -> "options": [ ], -> "query": "label_values(loki_build_info, cluster_id)", -> "refresh": 1, -> "regex": "", -> "sort": 2, -> "tagValuesQuery": "", -> "tags": [ ], -> "tagsQuery": "", -> "type": "query", -> "useTags": false -> }, -> { -647c665 -< "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)", ---- -> "query": "label_values(loki_build_info{cluster_id=\"$cluster_id\", cluster=~\"$cluster\"}, namespace)", -677c695 -< "query": "query_result(sum by (id) (grafanacloud_logs_instance_info) and sum(label_replace(loki_tenant:active_streams{cluster=\"$cluster\",namespace=\"$namespace\"},\"id\",\"$1\",\"tenant\",\"(.*)\")) by(id))", ---- -> "query": "query_result(sum by (id) (grafanacloud_logs_instance_info) and sum(label_replace(loki_tenant:active_streams{cluster_id=\"$cluster_id\", cluster=\"$cluster\",namespace=\"$namespace\"},\"id\",\"$1\",\"tenant\",\"(.*)\")) by(id))", -720c738 -< "uid": "recording-rules", ---- -> "uid": "loki-recording-rules", -723c741 -< } -\ No newline at end of file ---- -> } diff --git a/loki/diffs/loki-operational.json.diff b/loki/diffs/loki-operational.json.diff deleted file mode 100644 index 143f677d..00000000 --- a/loki/diffs/loki-operational.json.diff +++ /dev/null @@ -1,360 +0,0 @@ -90c90 -< "expr": "sum by (status) (\nlabel_replace(\n label_replace(\n rate(loki_request_duration_seconds_count{cluster=\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\"}[5m]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\")\n)", ---- -> "expr": "sum by (status) (\nlabel_replace(\n label_replace(\n rate(loki_request_duration_seconds_count{cluster_id=\"$cluster_id\", cluster=\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_label|loki_api_v1_label_name_values\"}[5m]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\")\n)", -186c186 -< "expr": "sum by (status) (\nlabel_replace(\n label_replace(\n rate(loki_request_duration_seconds_count{cluster=\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"api_prom_push|loki_api_v1_push\"}[5m]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))", ---- -> "expr": "sum by (status) (\nlabel_replace(\n label_replace(\n rate(loki_request_duration_seconds_count{cluster_id=\"$cluster_id\", cluster=\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"api_prom_push|loki_api_v1_push\"}[5m]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n\"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))", -281c281 -< "expr": "topk(10, sum(rate(loki_distributor_lines_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (tenant))", ---- -> "expr": "topk(10, sum(rate(loki_distributor_lines_received_total{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (tenant))", -377c377 -< "expr": "topk(10, sum(rate(loki_distributor_bytes_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (tenant)) / 1024 / 1024", ---- -> "expr": "topk(10, sum(rate(loki_distributor_bytes_received_total{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (tenant)) / 1024 / 1024", -471c471 -< "expr": "increase(kube_pod_container_status_restarts_total{cluster=\"$cluster\", namespace=\"$namespace\"}[10m]) > 0", ---- -> "expr": "increase(kube_pod_container_status_restarts_total{cluster_id=\"$cluster_id\", namespace=\"$namespace\"}[10m]) > 0", -781c781 -< "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (route) > 0", ---- -> "expr": "sum(rate(loki_request_duration_seconds_count{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (route) > 0", -987c987 -< "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", status_code!~\"5[0-9]{2}\", route=\"/logproto.Pusher/Push\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", route=\"/logproto.Pusher/Push\"}[$__rate_interval])) by (route) > 0", ---- -> "expr": "sum(rate(loki_request_duration_seconds_count{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", status_code!~\"5[0-9]{2}\", route=\"/logproto.Pusher/Push\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", route=\"/logproto.Pusher/Push\"}[$__rate_interval])) by (route) > 0", -1299c1299 -< "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-read\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-read\"}[$__rate_interval])) by (route) > 0", ---- -> "expr": "sum(rate(loki_request_duration_seconds_count{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-read\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-read\"}[$__rate_interval])) by (route) > 0", -1504c1504 -< "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", status_code!~\"5[0-9]{2}\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"}[$__rate_interval])) by (route) > 0", ---- -> "expr": "sum(rate(loki_request_duration_seconds_count{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", status_code!~\"5[0-9]{2}\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"/logproto.Querier/Query|/logproto.Querier/Label|/logproto.Querier/Series|/logproto.Querier/QuerySample|/logproto.Querier/GetChunkIDs\"}[$__rate_interval])) by (route) > 0", -1605c1605 -< "expr": "topk(10,sum by (tenant, reason) (rate(loki_discarded_samples_total{cluster=\"$cluster\",namespace=\"$namespace\"}[1m])))", ---- -> "expr": "topk(10,sum by (tenant, reason) (rate(loki_discarded_samples_total{cluster_id=\"$cluster_id\", cluster=\"$cluster\",namespace=\"$namespace\"}[1m])))", -1730c1730 -< "expr": "topk(10, sum by (tenant, reason) (sum_over_time(increase(loki_discarded_samples_total{cluster=\"$cluster\",namespace=\"$namespace\"}[1m])[$__range:1m])))", ---- -> "expr": "topk(10, sum by (tenant, reason) (sum_over_time(increase(loki_discarded_samples_total{cluster_id=\"$cluster_id\", cluster=\"$cluster\",namespace=\"$namespace\"}[1m])[$__range:1m])))", -1802c1802 -< "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(loki|enterprise-logs)-write.*\"}[$__rate_interval]))", ---- -> "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster_id=\"$cluster_id\", namespace=~\"$namespace\", pod=~\"(loki|enterprise-logs)-write.*\"}[$__rate_interval]))", -1893c1893 -< "expr": "go_memstats_heap_inuse_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"(loki|enterprise-logs)-write.*\"}", ---- -> "expr": "go_memstats_heap_inuse_bytes{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"(loki|enterprise-logs)-write.*\"}", -1988c1988 -< "expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\"} | logfmt | level=\"error\"[1m]))", ---- -> "expr": "sum(rate({cluster_id=\"$cluster_id\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)\", component=\"write\"} | logfmt | level=\"error\"[1m]))", -2051c2051 -< "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\"} |= \"level=error\"", ---- -> "expr": "{cluster_id=\"$cluster_id\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)\", component=\"write\"} |= \"level=error\"", -2102c2102 -< "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (route) > 0", ---- -> "expr": "sum(rate(loki_request_duration_seconds_count{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-write\"}[$__rate_interval])) by (route) > 0", -2192c2192 -< "expr": "sum(rate(loki_distributor_ingester_append_failures_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (pod)", ---- -> "expr": "sum(rate(loki_distributor_ingester_append_failures_total{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (pod)", -2281c2281 -< "expr": "sum(rate(loki_distributor_bytes_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (pod)", ---- -> "expr": "sum(rate(loki_distributor_bytes_received_total{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (pod)", -2370c2370 -< "expr": "sum(rate(loki_distributor_lines_received_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (pod)", ---- -> "expr": "sum(rate(loki_distributor_lines_received_total{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (pod)", -2477c2477 -< "expr": "topk(10,sum by (tenant) (loki_ingester_memory_streams{cluster=\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\"}))", ---- -> "expr": "topk(10,sum by (tenant) (loki_ingester_memory_streams{cluster_id=\"$cluster_id\", cluster=\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\"}))", -2568c2568 -< "expr": "topk(10, sum by (tenant) (rate(loki_ingester_streams_created_total{cluster=\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m]) > 0))", ---- -> "expr": "topk(10, sum by (tenant) (rate(loki_ingester_streams_created_total{cluster_id=\"$cluster_id\", cluster=\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m]) > 0))", -2678c2678 -< "expr": "sum(rate(loki_ingester_chunks_flushed_total{cluster=\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m]))", ---- -> "expr": "sum(rate(loki_ingester_chunks_flushed_total{cluster_id=\"$cluster_id\", cluster=\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m]))", -2684c2684 -< "expr": "sum(increase(loki_chunk_store_deduped_chunks_total{cluster=\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m]))/sum(increase(loki_ingester_chunks_flushed_total{cluster=\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m])) < 1", ---- -> "expr": "sum(increase(loki_chunk_store_deduped_chunks_total{cluster_id=\"$cluster_id\", cluster=\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m]))/sum(increase(loki_ingester_chunks_flushed_total{cluster_id=\"$cluster_id\", cluster=\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m])) < 1", -2762c2762 -< "expr": "sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster=\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m])) by (le)", ---- -> "expr": "sum(rate(loki_ingester_chunk_size_bytes_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m])) by (le)", -2838c2838 -< "expr": "sum by(reason) (rate(loki_ingester_chunks_flushed_total{cluster=~\"$cluster\",job=~\"$namespace/ingester\", namespace=~\"$namespace\"}[$__rate_interval])) / ignoring(reason) group_left sum(rate(loki_ingester_chunks_flushed_total{cluster=~\"$cluster\",job=~\"$namespace/ingester\", namespace=~\"$namespace\"}[$__rate_interval]))", ---- -> "expr": "sum by(reason) (rate(loki_ingester_chunks_flushed_total{cluster_id=\"$cluster_id\", cluster=~\"$cluster\",job=~\"$namespace/ingester\", namespace=~\"$namespace\"}[$__rate_interval])) / ignoring(reason) group_left sum(rate(loki_ingester_chunks_flushed_total{cluster_id=\"$cluster_id\", cluster=~\"$cluster\",job=~\"$namespace/ingester\", namespace=~\"$namespace\"}[$__rate_interval]))", -2917c2917 -< "expr": "sum by (le) (rate(loki_ingester_chunk_utilization_bucket{cluster=\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m]))", ---- -> "expr": "sum by (le) (rate(loki_ingester_chunk_utilization_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"}[1m]))", -3009c3009 -< "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(loki|enterprise-logs)-read.*\"}[$__rate_interval]))", ---- -> "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster_id=\"$cluster_id\", namespace=~\"$namespace\", pod=~\"(loki|enterprise-logs)-read.*\"}[$__rate_interval]))", -3100c3100 -< "expr": "go_memstats_heap_inuse_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"(loki|enterprise-logs)-read.*\"}", ---- -> "expr": "go_memstats_heap_inuse_bytes{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"(loki|enterprise-logs)-read.*\"}", -3195c3195 -< "expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-read\"} | logfmt | level=\"error\"[1m]))", ---- -> "expr": "sum(rate({cluster_id=\"$cluster_id\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)\", component=\"read\"} | logfmt | level=\"error\"[1m]))", -3258c3258 -< "expr": "{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-read\"} |= \"level=error\"", ---- -> "expr": "{cluster_id=\"$cluster_id\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)\", component=\"read\"} |= \"level=error\"", -3309c3309 -< "expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-read\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-read\"}[$__rate_interval])) by (route) > 0", ---- -> "expr": "sum(rate(loki_request_duration_seconds_count{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-read\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki|enterprise-logs)-read\"}[$__rate_interval])) by (route) > 0", -3418c3418 -< "expr": "histogram_quantile(.99, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (method, name, le, container))", ---- -> "expr": "histogram_quantile(.99, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (method, name, le, container))", -3424c3424 -< "expr": "histogram_quantile(.9, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (method, name, le, container))", ---- -> "expr": "histogram_quantile(.9, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (method, name, le, container))", -3430c3430 -< "expr": "histogram_quantile(.5, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (method, name, le, container))", ---- -> "expr": "histogram_quantile(.5, sum(rate(loki_memcache_request_duration_seconds_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (method, name, le, container))", -3522c3522 -< "expr": "sum(rate(loki_memcache_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, method, name, container)", ---- -> "expr": "sum(rate(loki_memcache_request_duration_seconds_count{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, method, name, container)", -3630c3630 -< "expr": "histogram_quantile(.99, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", ---- -> "expr": "histogram_quantile(.99, sum(rate(loki_consul_request_duration_seconds_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", -3636c3636 -< "expr": "histogram_quantile(.9, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", ---- -> "expr": "histogram_quantile(.9, sum(rate(loki_consul_request_duration_seconds_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", -3642c3642 -< "expr": "histogram_quantile(.5, sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", ---- -> "expr": "histogram_quantile(.5, sum(rate(loki_consul_request_duration_seconds_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", -3734c3734 -< "expr": "sum(rate(loki_consul_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, status_code, method)", ---- -> "expr": "sum(rate(loki_consul_request_duration_seconds_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, status_code, method)", -3840c3840 -< "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (operation, le))", ---- -> "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (operation, le))", -3846c3846 -< "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (operation, le))", ---- -> "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (operation, le))", -3850c3850 -< "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (operation, le))", ---- -> "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (operation, le))", -3938c3938 -< "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (operation, le))", ---- -> "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (operation, le))", -3945c3945 -< "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (operation, le))", ---- -> "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (operation, le))", -3951c3951 -< "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (operation, le))", ---- -> "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (operation, le))", -4041c4041 -< "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (operation, le))", ---- -> "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (operation, le))", -4048c4048 -< "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (operation, le))", ---- -> "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (operation, le))", -4054c4054 -< "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (operation, le))", ---- -> "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (operation, le))", -4144c4144 -< "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (operation, le))", ---- -> "expr": "histogram_quantile(.99, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (operation, le))", -4150c4150 -< "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (operation, le))", ---- -> "expr": "histogram_quantile(.9, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (operation, le))", -4154c4154 -< "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (operation, le))", ---- -> "expr": "histogram_quantile(.5, sum(rate(loki_bigtable_request_duration_seconds_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (operation, le))", -4242c4242 -< "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (status_code)", ---- -> "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/MutateRows\"}[5m])) by (status_code)", -4332c4332 -< "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (status_code)", ---- -> "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.v2.Bigtable/ReadRows\"}[5m])) by (status_code)", -4422c4422 -< "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (status_code)", ---- -> "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/GetTable\"}[5m])) by (status_code)", -4512c4512 -< "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (status_code)", ---- -> "expr": "sum(rate(loki_bigtable_request_duration_seconds_count{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\", operation=\"/google.bigtable.admin.v2.BigtableTableAdmin/ListTables\"}[5m])) by (status_code)", -4620c4620 -< "expr": "histogram_quantile(.99, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", ---- -> "expr": "histogram_quantile(.99, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", -4626c4626 -< "expr": "histogram_quantile(.9, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", ---- -> "expr": "histogram_quantile(.9, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", -4632c4632 -< "expr": "histogram_quantile(.5, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", ---- -> "expr": "histogram_quantile(.5, sum(rate(loki_gcs_request_duration_seconds_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", -4724c4724 -< "expr": "sum(rate(loki_gcs_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", ---- -> "expr": "sum(rate(loki_gcs_request_duration_seconds_count{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", -4828c4828 -< "expr": "sum(rate(cortex_dynamo_failures_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))", ---- -> "expr": "sum(rate(cortex_dynamo_failures_total{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))", -4914c4914 -< "expr": "sum(rate(cortex_dynamo_consumed_capacity_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))", ---- -> "expr": "sum(rate(cortex_dynamo_consumed_capacity_total{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))", -5000c5000 -< "expr": "sum(rate(cortex_dynamo_throttled_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))", ---- -> "expr": "sum(rate(cortex_dynamo_throttled_total{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))", -5086c5086 -< "expr": "sum(rate(cortex_dynamo_dropped_requests_total{cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))", ---- -> "expr": "sum(rate(cortex_dynamo_dropped_requests_total{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\"}[5m]))", -5172c5172 -< "expr": "histogram_quantile(.99, sum(rate(cortex_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])))", ---- -> "expr": "histogram_quantile(.99, sum(rate(cortex_dynamo_query_pages_count{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\"}[5m])))", -5177c5177 -< "expr": "histogram_quantile(.9, sum(rate(cortex_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])))", ---- -> "expr": "histogram_quantile(.9, sum(rate(cortex_dynamo_query_pages_count{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\"}[5m])))", -5182c5182 -< "expr": "histogram_quantile(.5, sum(rate(cortex_dynamo_query_pages_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])))", ---- -> "expr": "histogram_quantile(.5, sum(rate(cortex_dynamo_query_pages_count{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\"}[5m])))", -5272c5272 -< "expr": "histogram_quantile(.99, sum(rate(cortex_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", ---- -> "expr": "histogram_quantile(.99, sum(rate(cortex_dynamo_request_duration_seconds_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", -5278c5278 -< "expr": "histogram_quantile(.9, sum(rate(cortex_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", ---- -> "expr": "histogram_quantile(.9, sum(rate(cortex_dynamo_request_duration_seconds_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", -5284c5284 -< "expr": "histogram_quantile(.5, sum(rate(cortex_dynamo_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", ---- -> "expr": "histogram_quantile(.5, sum(rate(cortex_dynamo_request_duration_seconds_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", -5375c5375 -< "expr": "sum(rate(cortex_dynamo_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", ---- -> "expr": "sum(rate(cortex_dynamo_request_duration_seconds_count{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", -5482c5482 -< "expr": "histogram_quantile(.99, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", ---- -> "expr": "histogram_quantile(.99, sum(rate(loki_s3_request_duration_seconds_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", -5488c5488 -< "expr": "histogram_quantile(.9, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", ---- -> "expr": "histogram_quantile(.9, sum(rate(loki_s3_request_duration_seconds_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", -5494c5494 -< "expr": "histogram_quantile(.5, sum(rate(loki_s3_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", ---- -> "expr": "histogram_quantile(.5, sum(rate(loki_s3_request_duration_seconds_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", -5585c5585 -< "expr": "sum(rate(loki_s3_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", ---- -> "expr": "sum(rate(loki_s3_request_duration_seconds_count{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", -5692c5692 -< "expr": "histogram_quantile(.99, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", ---- -> "expr": "histogram_quantile(.99, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", -5698c5698 -< "expr": "histogram_quantile(.9, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", ---- -> "expr": "histogram_quantile(.9, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", -5704c5704 -< "expr": "histogram_quantile(.5, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", ---- -> "expr": "histogram_quantile(.5, sum(rate(loki_azure_blob_request_duration_seconds_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", -5795c5795 -< "expr": "sum(rate(loki_azure_blob_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", ---- -> "expr": "sum(rate(loki_azure_blob_request_duration_seconds_count{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", -5902c5902 -< "expr": "histogram_quantile(.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", ---- -> "expr": "histogram_quantile(.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", -5908c5908 -< "expr": "histogram_quantile(.9, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", ---- -> "expr": "histogram_quantile(.9, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", -5914c5914 -< "expr": "histogram_quantile(.5, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", ---- -> "expr": "histogram_quantile(.5, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (operation, le))", -6005c6005 -< "expr": "sum(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", ---- -> "expr": "sum(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster_id=\"$cluster_id\", cluster=\"$cluster\", namespace=\"$namespace\"}[5m])) by (status_code, operation)", -6114a6115,6132 -> "datasource": "$datasource", -> "hide": 0, -> "includeAll": false, -> "label": "Kube cluster", -> "multi": false, -> "name": "cluster_id", -> "options": [ ], -> "query": "label_values(loki_build_info, cluster_id)", -> "refresh": 1, -> "regex": "", -> "sort": 2, -> "tagValuesQuery": "", -> "tags": [ ], -> "tagsQuery": "", -> "type": "query", -> "useTags": false -> }, -> { -6127c6145 -< "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)", ---- -> "query": "label_values(loki_build_info{cluster_id=\"$cluster_id\", cluster=~\"$cluster\"}, namespace)", -6170c6188 -< "uid": "operational", ---- -> "uid": "loki-operational", -6172c6190 -< } -\ No newline at end of file ---- -> } diff --git a/loki/diffs/loki-reads-resources.json.diff b/loki/diffs/loki-reads-resources.json.diff deleted file mode 100644 index 67a8daf2..00000000 --- a/loki/diffs/loki-reads-resources.json.diff +++ /dev/null @@ -1,96 +0,0 @@ -73c73 -< "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"}[$__rate_interval]))", ---- -> "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster_id=\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"}[$__rate_interval]))", -81c81 -< "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\", resource=\"cpu\"} > 0)", ---- -> "expr": "min(kube_pod_container_resource_requests{cluster_id=\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\", resource=\"cpu\"} > 0)", -89c89 -< "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"})", ---- -> "expr": "min(container_spec_cpu_quota{cluster_id=\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"} / container_spec_cpu_period{cluster_id=\"$cluster_id\",namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"})", -175c175 -< "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"})", ---- -> "expr": "max by(pod) (container_memory_working_set_bytes{cluster_id=\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"})", -183c183 -< "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\", resource=\"memory\"} > 0)", ---- -> "expr": "min(kube_pod_container_resource_requests{cluster_id=\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\", resource=\"memory\"} > 0)", -191c191 -< "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"} > 0)", ---- -> "expr": "min(container_spec_memory_limit_bytes{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\"} > 0)", -266c266 -< "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-read\"})", ---- -> "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster_id=\"$cluster_id\", cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-read\"})", -341c341 -< "expr": "sum by(instance, pod, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", ---- -> "expr": "sum by(instance, pod, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster_id=\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", -418c418 -< "expr": "sum by(instance, pod, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", ---- -> "expr": "sum by(instance, pod, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-read.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", -495c495 -< "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}) and count by(persistentvolumeclaim) (kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"(loki|enterprise-logs)-read.*\"})", ---- -> "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\"}) and count by(persistentvolumeclaim) (kube_persistentvolumeclaim_labels{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\",label_name=~\"(loki|enterprise-logs)-read.*\"})", -595c595 -< "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"}[$__rate_interval]))", ---- -> "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"}[$__rate_interval]))", -603c603 -< "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", resource=\"cpu\"} > 0)", ---- -> "expr": "min(kube_pod_container_resource_requests{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", resource=\"cpu\"} > 0)", -611c611 -< "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"})", ---- -> "expr": "min(container_spec_cpu_quota{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"} / container_spec_cpu_period{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"})", -696c696 -< "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"})", ---- -> "expr": "max by(pod) (container_memory_working_set_bytes{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"})", -704c704 -< "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", resource=\"memory\"} > 0)", ---- -> "expr": "min(kube_pod_container_resource_requests{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", resource=\"memory\"} > 0)", -712c712 -< "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"} > 0)", ---- -> "expr": "min(container_spec_memory_limit_bytes{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"} > 0)", -786c786 -< "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"})", ---- -> "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster_id=~\"$cluster_id\", job=~\"($namespace)/(loki|enterprise-logs)-write\"})", -858a859,876 -> "datasource": "$datasource", -> "hide": 0, -> "includeAll": false, -> "label": "Kube cluster", -> "multi": false, -> "name": "cluster_id", -> "options": [ ], -> "query": "label_values(loki_build_info, cluster_id)", -> "refresh": 1, -> "regex": "", -> "sort": 2, -> "tagValuesQuery": "", -> "tags": [ ], -> "tagsQuery": "", -> "type": "query", -> "useTags": false -> }, -> { -937c955 -< "uid": "reads-resources", ---- -> "uid": "loki-reads-resources", -939c957 -< } -\ No newline at end of file ---- -> } diff --git a/loki/diffs/loki-reads.json.diff b/loki/diffs/loki-reads.json.diff deleted file mode 100644 index 31447cfa..00000000 --- a/loki/diffs/loki-reads.json.diff +++ /dev/null @@ -1,56 +0,0 @@ -69c69 -< "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", ---- -> "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{cluster_id=\"$cluster_id\", cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", -247c247 -< "expr": "histogram_quantile(0.99,\n sum(\n rate(loki_request_duration_seconds_bucket{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\"}[$__rate_interval])\n ) by (pod, le)\n )\n", ---- -> "expr": "histogram_quantile(0.99,\n sum(\n rate(loki_request_duration_seconds_bucket{cluster_id=\"$cluster_id\", cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-read\", route=~\"loki_api_v1_series|api_prom_series|api_prom_query|api_prom_label|api_prom_label_name_values|loki_api_v1_query|loki_api_v1_query_range|loki_api_v1_labels|loki_api_v1_label_name_values\"}[$__rate_interval])\n ) by (pod, le)\n )\n", -342c342 -< "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", ---- -> "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster_id=\"$cluster_id\", cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", -418c418 -< "expr": "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval])) by (le)) * 1e3", ---- -> "expr": "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster_id=\"$cluster_id\", cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval])) by (le)) * 1e3", -426c426 -< "expr": "histogram_quantile(0.50, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval])) by (le)) * 1e3", ---- -> "expr": "histogram_quantile(0.50, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster_id=\"$cluster_id\", cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval])) by (le)) * 1e3", -434c434 -< "expr": "sum(rate(loki_boltdb_shipper_request_duration_seconds_sum{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval])) * 1e3 / sum(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval]))", ---- -> "expr": "sum(rate(loki_boltdb_shipper_request_duration_seconds_sum{cluster_id=\"$cluster_id\", cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval])) * 1e3 / sum(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster_id=\"$cluster_id\", cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval]))", -520c520 -< "expr": "histogram_quantile(0.99,\n sum(\n rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval])\n ) by (pod, le)\n )\n", ---- -> "expr": "histogram_quantile(0.99,\n sum(\n rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster_id=\"$cluster_id\", cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-read\", operation=\"Shipper.Query\"}[$__rate_interval])\n ) by (pod, le)\n )\n", -616a617,634 -> "datasource": "$datasource", -> "hide": 0, -> "includeAll": false, -> "label": "Kube cluster", -> "multi": false, -> "name": "cluster_id", -> "options": [ ], -> "query": "label_values(loki_build_info, cluster_id)", -> "refresh": 1, -> "regex": "", -> "sort": 2, -> "tagValuesQuery": "", -> "tags": [ ], -> "tagsQuery": "", -> "type": "query", -> "useTags": false -> }, -> { -672c690 -< "uid": "reads", ---- -> "uid": "loki-reads", -674c692 -< } -\ No newline at end of file ---- -> } diff --git a/loki/diffs/loki-retention.json.diff b/loki/diffs/loki-retention.json.diff deleted file mode 100644 index e69de29b..00000000 diff --git a/loki/diffs/loki-writes-resources.json.diff b/loki/diffs/loki-writes-resources.json.diff deleted file mode 100644 index 8c750ce8..00000000 --- a/loki/diffs/loki-writes-resources.json.diff +++ /dev/null @@ -1,72 +0,0 @@ -62c62 -< "expr": "sum by(pod) (loki_ingester_memory_streams{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"})", ---- -> "expr": "sum by(pod) (loki_ingester_memory_streams{cluster_id=\"$cluster_id\", cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"})", -148c148 -< "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"}[$__rate_interval]))", ---- -> "expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"}[$__rate_interval]))", -156c156 -< "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", resource=\"cpu\"} > 0)", ---- -> "expr": "min(kube_pod_container_resource_requests{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", resource=\"cpu\"} > 0)", -164c164 -< "expr": "min(container_spec_cpu_quota{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"} / container_spec_cpu_period{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"})", ---- -> "expr": "min(container_spec_cpu_quota{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"} / container_spec_cpu_period{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"})", -250c250 -< "expr": "max by(pod) (container_memory_working_set_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"})", ---- -> "expr": "max by(pod) (container_memory_working_set_bytes{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"})", -258c258 -< "expr": "min(kube_pod_container_resource_requests{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", resource=\"memory\"} > 0)", ---- -> "expr": "min(kube_pod_container_resource_requests{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", resource=\"memory\"} > 0)", -266c266 -< "expr": "min(container_spec_memory_limit_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"} > 0)", ---- -> "expr": "min(container_spec_memory_limit_bytes{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\"} > 0)", -341c341 -< "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", job=~\"($namespace)/(loki|enterprise-logs)-write\"})", ---- -> "expr": "sum by(pod) (go_memstats_heap_inuse_bytes{cluster_id=~\"$cluster_id\", job=~\"($namespace)/(loki|enterprise-logs)-write\"})", -416c416 -< "expr": "sum by(instance, pod, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", ---- -> "expr": "sum by(instance, pod, device) (rate(node_disk_written_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", -493c493 -< "expr": "sum by(instance, pod, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", ---- -> "expr": "sum by(instance, pod, device) (rate(node_disk_read_bytes_total[$__rate_interval])) + ignoring(pod) group_right() (label_replace(count by(instance, pod, device) (container_fs_writes_bytes_total{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki|enterprise-logs)-write.*\", device!~\".*sda.*\"}), \"device\", \"$1\", \"device\", \"/dev/(.*)\") * 0)\n", -570c570 -< "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\"}) and count by(persistentvolumeclaim) (kube_persistentvolumeclaim_labels{cluster=~\"$cluster\", namespace=~\"$namespace\",label_name=~\"(loki|enterprise-logs)-write.*\"})", ---- -> "expr": "max by(persistentvolumeclaim) (kubelet_volume_stats_used_bytes{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\"} / kubelet_volume_stats_capacity_bytes{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\"}) and count by(persistentvolumeclaim) (kube_persistentvolumeclaim_labels{cluster_id=~\"$cluster_id\", namespace=~\"$namespace\",label_name=~\"(loki|enterprise-logs)-write.*\"})", -645a646,663 -> "datasource": "$datasource", -> "hide": 0, -> "includeAll": false, -> "label": "Kube cluster", -> "multi": false, -> "name": "cluster_id", -> "options": [ ], -> "query": "label_values(loki_build_info, cluster_id)", -> "refresh": 1, -> "regex": "", -> "sort": 2, -> "tagValuesQuery": "", -> "tags": [ ], -> "tagsQuery": "", -> "type": "query", -> "useTags": false -> }, -> { -724c742 -< "uid": "writes-resources", ---- -> "uid": "loki-writes-resources", -726c744 -< } -\ No newline at end of file ---- -> } diff --git a/loki/diffs/loki-writes.json.diff b/loki/diffs/loki-writes.json.diff deleted file mode 100644 index f1e9881d..00000000 --- a/loki/diffs/loki-writes.json.diff +++ /dev/null @@ -1,48 +0,0 @@ -69c69 -< "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"api_prom_push|loki_api_v1_push|/httpgrpc.HTTP/Handle\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", ---- -> "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_request_duration_seconds_count{cluster_id=\"$cluster_id\", cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", route=~\"api_prom_push|loki_api_v1_push|/httpgrpc.HTTP/Handle\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", -257c257 -< "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", ---- -> "expr": "sum by (status) (\n label_replace(label_replace(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster_id=\"$cluster_id\", cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval]),\n \"status\", \"${1}xx\", \"status_code\", \"([0-9])..\"),\n \"status\", \"${1}\", \"status_code\", \"([a-z]+)\"))\n", -333c333 -< "expr": "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval])) by (le)) * 1e3", ---- -> "expr": "histogram_quantile(0.99, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster_id=\"$cluster_id\", cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval])) by (le)) * 1e3", -341c341 -< "expr": "histogram_quantile(0.50, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval])) by (le)) * 1e3", ---- -> "expr": "histogram_quantile(0.50, sum(rate(loki_boltdb_shipper_request_duration_seconds_bucket{cluster_id=\"$cluster_id\", cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval])) by (le)) * 1e3", -349c349 -< "expr": "sum(rate(loki_boltdb_shipper_request_duration_seconds_sum{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval])) * 1e3 / sum(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval]))", ---- -> "expr": "sum(rate(loki_boltdb_shipper_request_duration_seconds_sum{cluster_id=\"$cluster_id\", cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval])) * 1e3 / sum(rate(loki_boltdb_shipper_request_duration_seconds_count{cluster_id=\"$cluster_id\", cluster=~\"$cluster\",job=~\"($namespace)/(loki|enterprise-logs)-write\", operation=\"WRITE\"}[$__rate_interval]))", -423a424,441 -> "datasource": "$datasource", -> "hide": 0, -> "includeAll": false, -> "label": "Kube cluster", -> "multi": false, -> "name": "cluster_id", -> "options": [ ], -> "query": "label_values(loki_build_info, cluster_id)", -> "refresh": 1, -> "regex": "", -> "sort": 2, -> "tagValuesQuery": "", -> "tags": [ ], -> "tagsQuery": "", -> "type": "query", -> "useTags": false -> }, -> { -502c520 -< "uid": "writes", ---- -> "uid": "loki-writes", -504c522 -< } -\ No newline at end of file ---- -> } diff --git a/loki/mixin.libsonnet b/loki/mixin.libsonnet index ff2f687d..6fe5de4d 100644 --- a/loki/mixin.libsonnet +++ b/loki/mixin.libsonnet @@ -1,11 +1,9 @@ -local loki = import 'loki-mixin/mixin-ssd.libsonnet'; - -loki{ +(import 'loki-mixin/mixin-ssd.libsonnet') + { _config+:: { tags: [ - "owner:team-atlas", - "topic:observability", - "component:loki" + 'owner:team-atlas', + 'topic:observability', + 'component:loki', ], per_node_label: 'node', @@ -15,6 +13,10 @@ loki{ enabled: false, }, + canary+: { + enabled: true, + }, + operational: { memcached: false, consul: false, diff --git a/loki/update.sh b/loki/update.sh index bae67b6f..74d12011 100755 --- a/loki/update.sh +++ b/loki/update.sh @@ -15,7 +15,7 @@ helmDir="$(pwd)/helm/dashboards/charts/private_dashboards_al/dashboards/shared/p set -x cd loki -rm -rf vendor jsonnetfile.* +rm -rf vendor jsonnetfile.* dashboards_out jb init jb install $MIXIN_URL diff --git a/mimir/mixin.libsonnet b/mimir/mixin.libsonnet index 52b6f8c7..3c851a77 100644 --- a/mimir/mixin.libsonnet +++ b/mimir/mixin.libsonnet @@ -1,11 +1,9 @@ -local mimir = import 'mimir-mixin/mixin.libsonnet'; - -mimir{ +(import 'mimir-mixin/mixin.libsonnet') + { _config+:: { tags: [ - "owner:team-atlas", - "topic:observability", - "component:mimir" + 'owner:team-atlas', + 'topic:observability', + 'component:mimir', ], per_cluster_label: 'cluster_id', diff --git a/tools/.gitkeep b/tools/.gitkeep deleted file mode 100644 index e69de29b..00000000