diff --git a/CHANGELOG.md b/CHANGELOG.md
index 395e2f41..599f4a67 100644
--- a/CHANGELOG.md
+++ b/CHANGELOG.md
@@ -7,6 +7,10 @@ and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0
 
 ## [Unreleased]
 
+### Added
+
+- dashboard for loki-canary (private for now)
+
 ### Changed
 
 - Migrate panels from graph to timeseries in DNS dashboard.
diff --git a/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-canary.json b/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-canary.json
new file mode 100644
index 00000000..ad998a7e
--- /dev/null
+++ b/helm/dashboards/charts/private_dashboards_al/dashboards/shared/private/loki-canary.json
@@ -0,0 +1,1801 @@
+{
+      "__requires": [
+         {
+            "id": "grafana",
+            "name": "Grafana",
+            "type": "grafana",
+            "version": "8.0.0"
+         }
+      ],
+      "annotations": {
+         "list": [ ]
+      },
+      "editable": true,
+      "gnetId": null,
+      "graphTooltip": 0,
+      "hideControls": false,
+      "links": [
+         {
+            "asDropdown": true,
+            "icon": "external link",
+            "includeVars": true,
+            "keepTime": true,
+            "tags": [
+               "loki"
+            ],
+            "targetBlank": false,
+            "title": "Loki Dashboards",
+            "type": "dashboards"
+         }
+      ],
+      "panels": [
+         {
+            "aliasColors": { },
+            "bars": false,
+            "dashLength": 10,
+            "dashes": false,
+            "datasource": "$datasource",
+            "fieldConfig": {
+               "defaults": {
+                  "color": {
+                     "mode": "thresholds"
+                  },
+                  "decimals": 1,
+                  "noValue": "",
+                  "thresholds": {
+                     "mode": "absolute",
+                     "steps": [ ]
+                  },
+                  "unit": "short"
+               },
+               "overrides": [ ]
+            },
+            "fill": 1,
+            "gridPos": {
+               "h": 4,
+               "w": 3,
+               "x": 0,
+               "y": 0
+            },
+            "legend": {
+               "avg": false,
+               "current": false,
+               "max": false,
+               "min": false,
+               "show": true,
+               "total": false,
+               "values": false
+            },
+            "lines": true,
+            "linewidth": 1,
+            "links": [ ],
+            "nullPointMode": "null as zero",
+            "percentage": false,
+            "pointradius": 5,
+            "points": false,
+            "renderer": "flot",
+            "seriesOverrides": [ ],
+            "spaceLength": 10,
+            "span": 6,
+            "stack": false,
+            "steppedLine": false,
+            "targets": [
+               {
+                  "expr": "sum(count(loki_canary_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}))",
+                  "format": null,
+                  "instant": false,
+                  "interval": "",
+                  "intervalFactor": null,
+                  "legendFormat": "",
+                  "legendLink": null,
+                  "step": null
+               }
+            ],
+            "thresholds": [ ],
+            "timeFrom": null,
+            "timeShift": null,
+            "title": "Canary Entries Total",
+            "tooltip": {
+               "shared": false,
+               "sort": 0,
+               "value_type": "individual"
+            },
+            "type": "stat",
+            "xaxis": {
+               "buckets": null,
+               "mode": "time",
+               "name": null,
+               "show": true,
+               "values": [ ]
+            },
+            "yaxes": [
+               {
+                  "format": "short",
+                  "label": null,
+                  "logBase": 1,
+                  "max": null,
+                  "min": 0,
+                  "show": true
+               },
+               {
+                  "format": "short",
+                  "label": null,
+                  "logBase": 1,
+                  "max": null,
+                  "min": null,
+                  "show": false
+               }
+            ]
+         },
+         {
+            "aliasColors": { },
+            "bars": false,
+            "dashLength": 10,
+            "dashes": false,
+            "datasource": "$datasource",
+            "fieldConfig": {
+               "defaults": {
+                  "color": {
+                     "mode": "thresholds"
+                  },
+                  "decimals": 1,
+                  "noValue": "",
+                  "thresholds": {
+                     "mode": "absolute",
+                     "steps": [ ]
+                  },
+                  "unit": "short"
+               },
+               "overrides": [ ]
+            },
+            "fill": 1,
+            "gridPos": {
+               "h": 4,
+               "w": 3,
+               "x": 3,
+               "y": 0
+            },
+            "legend": {
+               "avg": false,
+               "current": false,
+               "max": false,
+               "min": false,
+               "show": true,
+               "total": false,
+               "values": false
+            },
+            "lines": true,
+            "linewidth": 1,
+            "links": [ ],
+            "nullPointMode": "null as zero",
+            "percentage": false,
+            "pointradius": 5,
+            "points": false,
+            "renderer": "flot",
+            "seriesOverrides": [ ],
+            "spaceLength": 10,
+            "span": 6,
+            "stack": false,
+            "steppedLine": false,
+            "targets": [
+               {
+                  "expr": "sum(increase(loki_canary_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))",
+                  "format": null,
+                  "instant": false,
+                  "interval": "",
+                  "intervalFactor": null,
+                  "legendFormat": "",
+                  "legendLink": null,
+                  "step": null
+               }
+            ],
+            "thresholds": [ ],
+            "timeFrom": null,
+            "timeShift": null,
+            "title": "Canary Logs Total",
+            "tooltip": {
+               "shared": false,
+               "sort": 0,
+               "value_type": "individual"
+            },
+            "type": "stat",
+            "xaxis": {
+               "buckets": null,
+               "mode": "time",
+               "name": null,
+               "show": true,
+               "values": [ ]
+            },
+            "yaxes": [
+               {
+                  "format": "short",
+                  "label": null,
+                  "logBase": 1,
+                  "max": null,
+                  "min": 0,
+                  "show": true
+               },
+               {
+                  "format": "short",
+                  "label": null,
+                  "logBase": 1,
+                  "max": null,
+                  "min": null,
+                  "show": false
+               }
+            ]
+         },
+         {
+            "aliasColors": { },
+            "bars": false,
+            "dashLength": 10,
+            "dashes": false,
+            "datasource": "$datasource",
+            "fieldConfig": {
+               "defaults": {
+                  "color": {
+                     "mode": "thresholds"
+                  },
+                  "decimals": 1,
+                  "noValue": "",
+                  "thresholds": {
+                     "mode": "absolute",
+                     "steps": [ ]
+                  },
+                  "unit": "short"
+               },
+               "overrides": [ ]
+            },
+            "fill": 1,
+            "gridPos": {
+               "h": 4,
+               "w": 3,
+               "x": 6,
+               "y": 0
+            },
+            "legend": {
+               "avg": false,
+               "current": false,
+               "max": false,
+               "min": false,
+               "show": true,
+               "total": false,
+               "values": false
+            },
+            "lines": true,
+            "linewidth": 1,
+            "links": [ ],
+            "nullPointMode": "null as zero",
+            "percentage": false,
+            "pointradius": 5,
+            "points": false,
+            "renderer": "flot",
+            "seriesOverrides": [ ],
+            "spaceLength": 10,
+            "span": 6,
+            "stack": false,
+            "steppedLine": false,
+            "targets": [
+               {
+                  "expr": "sum(increase(loki_canary_missing_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))",
+                  "format": null,
+                  "instant": false,
+                  "interval": "",
+                  "intervalFactor": null,
+                  "legendFormat": "",
+                  "legendLink": null,
+                  "step": null
+               }
+            ],
+            "thresholds": [ ],
+            "timeFrom": null,
+            "timeShift": null,
+            "title": "Missing",
+            "tooltip": {
+               "shared": false,
+               "sort": 0,
+               "value_type": "individual"
+            },
+            "type": "stat",
+            "xaxis": {
+               "buckets": null,
+               "mode": "time",
+               "name": null,
+               "show": true,
+               "values": [ ]
+            },
+            "yaxes": [
+               {
+                  "format": "short",
+                  "label": null,
+                  "logBase": 1,
+                  "max": null,
+                  "min": 0,
+                  "show": true
+               },
+               {
+                  "format": "short",
+                  "label": null,
+                  "logBase": 1,
+                  "max": null,
+                  "min": null,
+                  "show": false
+               }
+            ]
+         },
+         {
+            "aliasColors": { },
+            "bars": false,
+            "dashLength": 10,
+            "dashes": false,
+            "datasource": "$datasource",
+            "fieldConfig": {
+               "defaults": {
+                  "color": {
+                     "mode": "thresholds"
+                  },
+                  "decimals": 1,
+                  "noValue": "",
+                  "thresholds": {
+                     "mode": "absolute",
+                     "steps": [ ]
+                  },
+                  "unit": "short"
+               },
+               "overrides": [ ]
+            },
+            "fill": 1,
+            "gridPos": {
+               "h": 4,
+               "w": 3,
+               "x": 9,
+               "y": 0
+            },
+            "legend": {
+               "avg": false,
+               "current": false,
+               "max": false,
+               "min": false,
+               "show": true,
+               "total": false,
+               "values": false
+            },
+            "lines": true,
+            "linewidth": 1,
+            "links": [ ],
+            "nullPointMode": "null as zero",
+            "percentage": false,
+            "pointradius": 5,
+            "points": false,
+            "renderer": "flot",
+            "seriesOverrides": [ ],
+            "spaceLength": 10,
+            "span": 6,
+            "stack": false,
+            "steppedLine": false,
+            "targets": [
+               {
+                  "expr": "sum(increase(loki_canary_spot_check_missing_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))",
+                  "format": null,
+                  "instant": false,
+                  "interval": "",
+                  "intervalFactor": null,
+                  "legendFormat": "",
+                  "legendLink": null,
+                  "step": null
+               }
+            ],
+            "thresholds": [ ],
+            "timeFrom": null,
+            "timeShift": null,
+            "title": "Spotcheck Missing",
+            "tooltip": {
+               "shared": false,
+               "sort": 0,
+               "value_type": "individual"
+            },
+            "type": "stat",
+            "xaxis": {
+               "buckets": null,
+               "mode": "time",
+               "name": null,
+               "show": true,
+               "values": [ ]
+            },
+            "yaxes": [
+               {
+                  "format": "short",
+                  "label": null,
+                  "logBase": 1,
+                  "max": null,
+                  "min": 0,
+                  "show": true
+               },
+               {
+                  "format": "short",
+                  "label": null,
+                  "logBase": 1,
+                  "max": null,
+                  "min": null,
+                  "show": false
+               }
+            ]
+         },
+         {
+            "aliasColors": { },
+            "bars": false,
+            "dashLength": 10,
+            "dashes": false,
+            "datasource": "$datasource",
+            "fieldConfig": {
+               "defaults": {
+                  "color": {
+                     "mode": "thresholds"
+                  },
+                  "decimals": 1,
+                  "noValue": "",
+                  "thresholds": {
+                     "mode": "absolute",
+                     "steps": [ ]
+                  },
+                  "unit": "short"
+               },
+               "overrides": [ ]
+            },
+            "fill": 1,
+            "gridPos": {
+               "h": 4,
+               "w": 3,
+               "x": 0,
+               "y": 4
+            },
+            "legend": {
+               "avg": false,
+               "current": false,
+               "max": false,
+               "min": false,
+               "show": true,
+               "total": false,
+               "values": false
+            },
+            "lines": true,
+            "linewidth": 1,
+            "links": [ ],
+            "nullPointMode": "null as zero",
+            "percentage": false,
+            "pointradius": 5,
+            "points": false,
+            "renderer": "flot",
+            "seriesOverrides": [ ],
+            "spaceLength": 10,
+            "span": 6,
+            "stack": false,
+            "steppedLine": false,
+            "targets": [
+               {
+                  "expr": "sum(increase(loki_canary_spot_check_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))",
+                  "format": null,
+                  "instant": false,
+                  "interval": "",
+                  "intervalFactor": null,
+                  "legendFormat": "",
+                  "legendLink": null,
+                  "step": null
+               }
+            ],
+            "thresholds": [ ],
+            "timeFrom": null,
+            "timeShift": null,
+            "title": "Spotcheck Total",
+            "tooltip": {
+               "shared": false,
+               "sort": 0,
+               "value_type": "individual"
+            },
+            "type": "stat",
+            "xaxis": {
+               "buckets": null,
+               "mode": "time",
+               "name": null,
+               "show": true,
+               "values": [ ]
+            },
+            "yaxes": [
+               {
+                  "format": "short",
+                  "label": null,
+                  "logBase": 1,
+                  "max": null,
+                  "min": 0,
+                  "show": true
+               },
+               {
+                  "format": "short",
+                  "label": null,
+                  "logBase": 1,
+                  "max": null,
+                  "min": null,
+                  "show": false
+               }
+            ]
+         },
+         {
+            "aliasColors": { },
+            "bars": false,
+            "dashLength": 10,
+            "dashes": false,
+            "datasource": "$datasource",
+            "fieldConfig": {
+               "defaults": {
+                  "color": {
+                     "mode": "thresholds"
+                  },
+                  "decimals": 1,
+                  "noValue": "",
+                  "thresholds": {
+                     "mode": "absolute",
+                     "steps": [ ]
+                  },
+                  "unit": "percentunit"
+               },
+               "overrides": [ ]
+            },
+            "fill": 1,
+            "gridPos": {
+               "h": 4,
+               "w": 3,
+               "x": 3,
+               "y": 4
+            },
+            "legend": {
+               "avg": false,
+               "current": false,
+               "max": false,
+               "min": false,
+               "show": true,
+               "total": false,
+               "values": false
+            },
+            "lines": true,
+            "linewidth": 1,
+            "links": [ ],
+            "nullPointMode": "null as zero",
+            "percentage": false,
+            "pointradius": 5,
+            "points": false,
+            "renderer": "flot",
+            "seriesOverrides": [ ],
+            "spaceLength": 10,
+            "span": 6,
+            "stack": false,
+            "steppedLine": false,
+            "targets": [
+               {
+                  "expr": "((sum(loki_canary_metric_test_expected{cluster=~\"$cluster\",namespace=~\"$namespace\"}) - sum(loki_canary_metric_test_actual{cluster=~\"$cluster\",namespace=~\"$namespace\"}))/(sum(loki_canary_metric_test_actual{cluster=~\"$cluster\",namespace=~\"$namespace\"}))) * 100",
+                  "format": null,
+                  "instant": false,
+                  "interval": "",
+                  "intervalFactor": null,
+                  "legendFormat": "",
+                  "legendLink": null,
+                  "step": null
+               }
+            ],
+            "thresholds": [ ],
+            "timeFrom": null,
+            "timeShift": null,
+            "title": "Metric Test Error %",
+            "tooltip": {
+               "shared": false,
+               "sort": 0,
+               "value_type": "individual"
+            },
+            "type": "stat",
+            "xaxis": {
+               "buckets": null,
+               "mode": "time",
+               "name": null,
+               "show": true,
+               "values": [ ]
+            },
+            "yaxes": [
+               {
+                  "format": "short",
+                  "label": null,
+                  "logBase": 1,
+                  "max": null,
+                  "min": 0,
+                  "show": true
+               },
+               {
+                  "format": "short",
+                  "label": null,
+                  "logBase": 1,
+                  "max": null,
+                  "min": null,
+                  "show": false
+               }
+            ]
+         },
+         {
+            "aliasColors": { },
+            "bars": false,
+            "dashLength": 10,
+            "dashes": false,
+            "datasource": "$datasource",
+            "fieldConfig": {
+               "defaults": {
+                  "color": {
+                     "mode": "thresholds"
+                  },
+                  "decimals": 1,
+                  "noValue": "",
+                  "thresholds": {
+                     "mode": "absolute",
+                     "steps": [ ]
+                  },
+                  "unit": "percentunit"
+               },
+               "overrides": [ ]
+            },
+            "fill": 1,
+            "gridPos": {
+               "h": 4,
+               "w": 3,
+               "x": 6,
+               "y": 4
+            },
+            "legend": {
+               "avg": false,
+               "current": false,
+               "max": false,
+               "min": false,
+               "show": true,
+               "total": false,
+               "values": false
+            },
+            "lines": true,
+            "linewidth": 1,
+            "links": [ ],
+            "nullPointMode": "null as zero",
+            "percentage": false,
+            "pointradius": 5,
+            "points": false,
+            "renderer": "flot",
+            "seriesOverrides": [ ],
+            "spaceLength": 10,
+            "span": 6,
+            "stack": false,
+            "steppedLine": false,
+            "targets": [
+               {
+                  "expr": "(sum(increase(loki_canary_missing_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))/sum(increase(loki_canary_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__range])))*100",
+                  "format": null,
+                  "instant": false,
+                  "interval": "",
+                  "intervalFactor": null,
+                  "legendFormat": "",
+                  "legendLink": null,
+                  "step": null
+               }
+            ],
+            "thresholds": [ ],
+            "timeFrom": null,
+            "timeShift": null,
+            "title": "Missing %",
+            "tooltip": {
+               "shared": false,
+               "sort": 0,
+               "value_type": "individual"
+            },
+            "type": "stat",
+            "xaxis": {
+               "buckets": null,
+               "mode": "time",
+               "name": null,
+               "show": true,
+               "values": [ ]
+            },
+            "yaxes": [
+               {
+                  "format": "short",
+                  "label": null,
+                  "logBase": 1,
+                  "max": null,
+                  "min": 0,
+                  "show": true
+               },
+               {
+                  "format": "short",
+                  "label": null,
+                  "logBase": 1,
+                  "max": null,
+                  "min": null,
+                  "show": false
+               }
+            ]
+         },
+         {
+            "aliasColors": { },
+            "bars": false,
+            "dashLength": 10,
+            "dashes": false,
+            "datasource": "$datasource",
+            "fieldConfig": {
+               "defaults": {
+                  "color": {
+                     "mode": "thresholds"
+                  },
+                  "decimals": 1,
+                  "noValue": "",
+                  "thresholds": {
+                     "mode": "absolute",
+                     "steps": [ ]
+                  },
+                  "unit": "percentunit"
+               },
+               "overrides": [ ]
+            },
+            "fill": 1,
+            "gridPos": {
+               "h": 4,
+               "w": 3,
+               "x": 9,
+               "y": 4
+            },
+            "legend": {
+               "avg": false,
+               "current": false,
+               "max": false,
+               "min": false,
+               "show": true,
+               "total": false,
+               "values": false
+            },
+            "lines": true,
+            "linewidth": 1,
+            "links": [ ],
+            "nullPointMode": "null as zero",
+            "percentage": false,
+            "pointradius": 5,
+            "points": false,
+            "renderer": "flot",
+            "seriesOverrides": [ ],
+            "spaceLength": 10,
+            "span": 6,
+            "stack": false,
+            "steppedLine": false,
+            "targets": [
+               {
+                  "expr": "(sum(increase(loki_canary_spot_check_missing_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))/sum(increase(loki_canary_spot_check_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))) * 100",
+                  "format": null,
+                  "instant": false,
+                  "interval": "",
+                  "intervalFactor": null,
+                  "legendFormat": "",
+                  "legendLink": null,
+                  "step": null
+               }
+            ],
+            "thresholds": [ ],
+            "timeFrom": null,
+            "timeShift": null,
+            "title": "Spotcheck Missing %",
+            "tooltip": {
+               "shared": false,
+               "sort": 0,
+               "value_type": "individual"
+            },
+            "type": "stat",
+            "xaxis": {
+               "buckets": null,
+               "mode": "time",
+               "name": null,
+               "show": true,
+               "values": [ ]
+            },
+            "yaxes": [
+               {
+                  "format": "short",
+                  "label": null,
+                  "logBase": 1,
+                  "max": null,
+                  "min": 0,
+                  "show": true
+               },
+               {
+                  "format": "short",
+                  "label": null,
+                  "logBase": 1,
+                  "max": null,
+                  "min": null,
+                  "show": false
+               }
+            ]
+         },
+         {
+            "aliasColors": { },
+            "bars": false,
+            "dashLength": 10,
+            "dashes": false,
+            "datasource": "$datasource",
+            "fieldConfig": {
+               "defaults": {
+                  "color": {
+                     "mode": "thresholds"
+                  },
+                  "decimals": 1,
+                  "noValue": "",
+                  "thresholds": {
+                     "mode": "absolute",
+                     "steps": [ ]
+                  },
+                  "unit": "short"
+               },
+               "overrides": [ ]
+            },
+            "fill": 1,
+            "gridPos": {
+               "h": 4,
+               "w": 3,
+               "x": 0,
+               "y": 8
+            },
+            "legend": {
+               "avg": false,
+               "current": false,
+               "max": false,
+               "min": false,
+               "show": true,
+               "total": false,
+               "values": false
+            },
+            "lines": true,
+            "linewidth": 1,
+            "links": [ ],
+            "nullPointMode": "null as zero",
+            "percentage": false,
+            "pointradius": 5,
+            "points": false,
+            "renderer": "flot",
+            "seriesOverrides": [ ],
+            "spaceLength": 10,
+            "span": 6,
+            "stack": false,
+            "steppedLine": false,
+            "targets": [
+               {
+                  "expr": "sum(loki_canary_metric_test_expected{cluster=~\"$cluster\",namespace=~\"$namespace\"})",
+                  "format": null,
+                  "instant": false,
+                  "interval": "",
+                  "intervalFactor": null,
+                  "legendFormat": "",
+                  "legendLink": null,
+                  "step": null
+               }
+            ],
+            "thresholds": [ ],
+            "timeFrom": null,
+            "timeShift": null,
+            "title": "Metric Test Expected",
+            "tooltip": {
+               "shared": false,
+               "sort": 0,
+               "value_type": "individual"
+            },
+            "type": "stat",
+            "xaxis": {
+               "buckets": null,
+               "mode": "time",
+               "name": null,
+               "show": true,
+               "values": [ ]
+            },
+            "yaxes": [
+               {
+                  "format": "short",
+                  "label": null,
+                  "logBase": 1,
+                  "max": null,
+                  "min": 0,
+                  "show": true
+               },
+               {
+                  "format": "short",
+                  "label": null,
+                  "logBase": 1,
+                  "max": null,
+                  "min": null,
+                  "show": false
+               }
+            ]
+         },
+         {
+            "aliasColors": { },
+            "bars": false,
+            "dashLength": 10,
+            "dashes": false,
+            "datasource": "$datasource",
+            "fieldConfig": {
+               "defaults": {
+                  "color": {
+                     "mode": "thresholds"
+                  },
+                  "decimals": 1,
+                  "noValue": "",
+                  "thresholds": {
+                     "mode": "absolute",
+                     "steps": [ ]
+                  },
+                  "unit": "short"
+               },
+               "overrides": [ ]
+            },
+            "fill": 1,
+            "gridPos": {
+               "h": 4,
+               "w": 3,
+               "x": 3,
+               "y": 8
+            },
+            "legend": {
+               "avg": false,
+               "current": false,
+               "max": false,
+               "min": false,
+               "show": true,
+               "total": false,
+               "values": false
+            },
+            "lines": true,
+            "linewidth": 1,
+            "links": [ ],
+            "nullPointMode": "null as zero",
+            "percentage": false,
+            "pointradius": 5,
+            "points": false,
+            "renderer": "flot",
+            "seriesOverrides": [ ],
+            "spaceLength": 10,
+            "span": 6,
+            "stack": false,
+            "steppedLine": false,
+            "targets": [
+               {
+                  "expr": "sum(loki_canary_metric_test_actual{cluster=~\"$cluster\",namespace=~\"$namespace\"})",
+                  "format": null,
+                  "instant": false,
+                  "interval": "",
+                  "intervalFactor": null,
+                  "legendFormat": "",
+                  "legendLink": null,
+                  "step": null
+               }
+            ],
+            "thresholds": [ ],
+            "timeFrom": null,
+            "timeShift": null,
+            "title": "Metric Test Actual",
+            "tooltip": {
+               "shared": false,
+               "sort": 0,
+               "value_type": "individual"
+            },
+            "type": "stat",
+            "xaxis": {
+               "buckets": null,
+               "mode": "time",
+               "name": null,
+               "show": true,
+               "values": [ ]
+            },
+            "yaxes": [
+               {
+                  "format": "short",
+                  "label": null,
+                  "logBase": 1,
+                  "max": null,
+                  "min": 0,
+                  "show": true
+               },
+               {
+                  "format": "short",
+                  "label": null,
+                  "logBase": 1,
+                  "max": null,
+                  "min": null,
+                  "show": false
+               }
+            ]
+         },
+         {
+            "aliasColors": { },
+            "bars": false,
+            "dashLength": 10,
+            "dashes": false,
+            "datasource": "$datasource",
+            "fieldConfig": {
+               "defaults": {
+                  "color": {
+                     "mode": "thresholds"
+                  },
+                  "decimals": 1,
+                  "noValue": "",
+                  "thresholds": {
+                     "mode": "absolute",
+                     "steps": [ ]
+                  },
+                  "unit": "short"
+               },
+               "overrides": [ ]
+            },
+            "fill": 1,
+            "gridPos": {
+               "h": 4,
+               "w": 3,
+               "x": 6,
+               "y": 8
+            },
+            "legend": {
+               "avg": false,
+               "current": false,
+               "max": false,
+               "min": false,
+               "show": true,
+               "total": false,
+               "values": false
+            },
+            "lines": true,
+            "linewidth": 1,
+            "links": [ ],
+            "nullPointMode": "null as zero",
+            "percentage": false,
+            "pointradius": 5,
+            "points": false,
+            "renderer": "flot",
+            "seriesOverrides": [ ],
+            "spaceLength": 10,
+            "span": 6,
+            "stack": false,
+            "steppedLine": false,
+            "targets": [
+               {
+                  "expr": "sum(increase(loki_canary_websocket_missing_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))",
+                  "format": null,
+                  "instant": false,
+                  "interval": "",
+                  "intervalFactor": null,
+                  "legendFormat": "",
+                  "legendLink": null,
+                  "step": null
+               }
+            ],
+            "thresholds": [ ],
+            "timeFrom": null,
+            "timeShift": null,
+            "title": "Websocket Missing",
+            "tooltip": {
+               "shared": false,
+               "sort": 0,
+               "value_type": "individual"
+            },
+            "type": "stat",
+            "xaxis": {
+               "buckets": null,
+               "mode": "time",
+               "name": null,
+               "show": true,
+               "values": [ ]
+            },
+            "yaxes": [
+               {
+                  "format": "short",
+                  "label": null,
+                  "logBase": 1,
+                  "max": null,
+                  "min": 0,
+                  "show": true
+               },
+               {
+                  "format": "short",
+                  "label": null,
+                  "logBase": 1,
+                  "max": null,
+                  "min": null,
+                  "show": false
+               }
+            ]
+         },
+         {
+            "aliasColors": { },
+            "bars": false,
+            "dashLength": 10,
+            "dashes": false,
+            "datasource": "$datasource",
+            "fieldConfig": {
+               "defaults": {
+                  "color": {
+                     "mode": "thresholds"
+                  },
+                  "decimals": 1,
+                  "noValue": "",
+                  "thresholds": {
+                     "mode": "absolute",
+                     "steps": [ ]
+                  },
+                  "unit": "percentunit"
+               },
+               "overrides": [ ]
+            },
+            "fill": 1,
+            "gridPos": {
+               "h": 4,
+               "w": 3,
+               "x": 9,
+               "y": 8
+            },
+            "legend": {
+               "avg": false,
+               "current": false,
+               "max": false,
+               "min": false,
+               "show": true,
+               "total": false,
+               "values": false
+            },
+            "lines": true,
+            "linewidth": 1,
+            "links": [ ],
+            "nullPointMode": "null as zero",
+            "percentage": false,
+            "pointradius": 5,
+            "points": false,
+            "renderer": "flot",
+            "seriesOverrides": [ ],
+            "spaceLength": 10,
+            "span": 6,
+            "stack": false,
+            "steppedLine": false,
+            "targets": [
+               {
+                  "expr": "(sum(increase(loki_canary_websocket_missing_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__range]))/sum(increase(loki_canary_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__range])))*100",
+                  "format": null,
+                  "instant": false,
+                  "interval": "",
+                  "intervalFactor": null,
+                  "legendFormat": "",
+                  "legendLink": null,
+                  "step": null
+               }
+            ],
+            "thresholds": [ ],
+            "timeFrom": null,
+            "timeShift": null,
+            "title": "Websocket Missing %",
+            "tooltip": {
+               "shared": false,
+               "sort": 0,
+               "value_type": "individual"
+            },
+            "type": "stat",
+            "xaxis": {
+               "buckets": null,
+               "mode": "time",
+               "name": null,
+               "show": true,
+               "values": [ ]
+            },
+            "yaxes": [
+               {
+                  "format": "short",
+                  "label": null,
+                  "logBase": 1,
+                  "max": null,
+                  "min": 0,
+                  "show": true
+               },
+               {
+                  "format": "short",
+                  "label": null,
+                  "logBase": 1,
+                  "max": null,
+                  "min": null,
+                  "show": false
+               }
+            ]
+         },
+         {
+            "aliasColors": { },
+            "bars": false,
+            "dashLength": 10,
+            "dashes": false,
+            "datasource": "$datasource",
+            "fill": 1,
+            "gridPos": {
+               "h": 6,
+               "w": 12,
+               "x": 12,
+               "y": 0
+            },
+            "legend": {
+               "avg": false,
+               "current": false,
+               "max": false,
+               "min": false,
+               "show": true,
+               "total": false,
+               "values": false
+            },
+            "lines": true,
+            "linewidth": 1,
+            "links": [ ],
+            "nullPointMode": "null as zero",
+            "percentage": false,
+            "pointradius": 5,
+            "points": false,
+            "renderer": "flot",
+            "seriesOverrides": [ ],
+            "spaceLength": 10,
+            "span": 6,
+            "stack": false,
+            "steppedLine": false,
+            "targets": [
+               {
+                  "expr": "histogram_quantile(0.95, sum(rate(loki_canary_response_latency_seconds_bucket{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) by (le))",
+                  "format": "time_series",
+                  "intervalFactor": 2,
+                  "legendFormat": "p95",
+                  "legendLink": null,
+                  "step": 10
+               },
+               {
+                  "expr": "histogram_quantile(0.50, sum(rate(loki_canary_response_latency_seconds_bucket{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) by (le))",
+                  "format": "time_series",
+                  "intervalFactor": 2,
+                  "legendFormat": "p50",
+                  "legendLink": null,
+                  "step": 10
+               }
+            ],
+            "thresholds": [ ],
+            "timeFrom": null,
+            "timeShift": null,
+            "title": "Log Write to read Latency Percentiles",
+            "tooltip": {
+               "shared": false,
+               "sort": 0,
+               "value_type": "individual"
+            },
+            "type": "graph",
+            "xaxis": {
+               "buckets": null,
+               "mode": "time",
+               "name": null,
+               "show": true,
+               "values": [ ]
+            },
+            "yaxes": [
+               {
+                  "format": "short",
+                  "label": null,
+                  "logBase": 1,
+                  "max": null,
+                  "min": 0,
+                  "show": true
+               },
+               {
+                  "format": "short",
+                  "label": null,
+                  "logBase": 1,
+                  "max": null,
+                  "min": null,
+                  "show": false
+               }
+            ]
+         },
+         {
+            "cards": {
+               "cardPadding": null,
+               "cardRound": null
+            },
+            "color": {
+               "cardColor": "#b4ff00",
+               "colorScale": "sqrt",
+               "colorScheme": "interpolateReds",
+               "exponent": 0.5,
+               "mode": "spectrum"
+            },
+            "dataFormat": "timeseries",
+            "datasource": "$datasource",
+            "gridPos": {
+               "h": 6,
+               "w": 12,
+               "x": 12,
+               "y": 12
+            },
+            "heatmap": { },
+            "hideZeroBuckets": false,
+            "highlightCards": true,
+            "legend": {
+               "show": false
+            },
+            "targets": [
+               {
+                  "expr": "sum(rate(loki_canary_response_latency_seconds_bucket{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) by (le)",
+                  "format": "heatmap",
+                  "intervalFactor": 2,
+                  "legendFormat": "{{le}}",
+                  "refId": "A"
+               }
+            ],
+            "title": "Log Write to Read Latency",
+            "tooltip": {
+               "show": true,
+               "showHistogram": true
+            },
+            "type": "heatmap",
+            "xAxis": {
+               "show": true
+            },
+            "xBucketNumber": null,
+            "xBucketSize": null,
+            "yAxis": {
+               "decimals": null,
+               "format": "short",
+               "logBase": 1,
+               "max": null,
+               "min": null,
+               "show": true,
+               "splitFactor": null
+            },
+            "yBucketBound": "auto",
+            "yBucketNumber": null,
+            "yBucketSize": null
+         },
+         {
+            "aliasColors": { },
+            "bars": false,
+            "dashLength": 10,
+            "dashes": false,
+            "datasource": "$datasource",
+            "fill": 1,
+            "gridPos": {
+               "h": 6,
+               "w": 12,
+               "x": 0,
+               "y": 14
+            },
+            "legend": {
+               "avg": false,
+               "current": false,
+               "max": false,
+               "min": false,
+               "show": true,
+               "total": false,
+               "values": false
+            },
+            "lines": true,
+            "linewidth": 1,
+            "links": [ ],
+            "nullPointMode": "null as zero",
+            "percentage": false,
+            "pointradius": 5,
+            "points": false,
+            "renderer": "flot",
+            "seriesOverrides": [ ],
+            "spaceLength": 10,
+            "span": 6,
+            "stack": false,
+            "steppedLine": false,
+            "targets": [
+               {
+                  "expr": "histogram_quantile(0.99, sum(rate(loki_canary_spot_check_request_duration_seconds_bucket{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) by (le))",
+                  "format": "time_series",
+                  "intervalFactor": 2,
+                  "legendFormat": "p99",
+                  "legendLink": null,
+                  "step": 10
+               },
+               {
+                  "expr": "histogram_quantile(0.50, sum(rate(loki_canary_spot_check_request_duration_seconds_bucket{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) by (le))",
+                  "format": "time_series",
+                  "intervalFactor": 2,
+                  "legendFormat": "p95",
+                  "legendLink": null,
+                  "step": 10
+               }
+            ],
+            "thresholds": [ ],
+            "timeFrom": null,
+            "timeShift": null,
+            "title": "Spot Check Query",
+            "tooltip": {
+               "shared": false,
+               "sort": 0,
+               "value_type": "individual"
+            },
+            "type": "graph",
+            "xaxis": {
+               "buckets": null,
+               "mode": "time",
+               "name": null,
+               "show": true,
+               "values": [ ]
+            },
+            "yaxes": [
+               {
+                  "format": "short",
+                  "label": null,
+                  "logBase": 1,
+                  "max": null,
+                  "min": 0,
+                  "show": true
+               },
+               {
+                  "format": "short",
+                  "label": null,
+                  "logBase": 1,
+                  "max": null,
+                  "min": null,
+                  "show": false
+               }
+            ]
+         },
+         {
+            "aliasColors": { },
+            "bars": false,
+            "dashLength": 10,
+            "dashes": false,
+            "datasource": "$datasource",
+            "fill": 1,
+            "gridPos": {
+               "h": 6,
+               "w": 12,
+               "x": 12,
+               "y": 14
+            },
+            "legend": {
+               "avg": false,
+               "current": false,
+               "max": false,
+               "min": false,
+               "show": true,
+               "total": false,
+               "values": false
+            },
+            "lines": true,
+            "linewidth": 1,
+            "links": [ ],
+            "nullPointMode": "null as zero",
+            "percentage": false,
+            "pointradius": 5,
+            "points": false,
+            "renderer": "flot",
+            "seriesOverrides": [ ],
+            "spaceLength": 10,
+            "span": 6,
+            "stack": false,
+            "steppedLine": false,
+            "targets": [
+               {
+                  "expr": "histogram_quantile(0.99, sum(rate(loki_canary_metric_test_request_duration_seconds_bucket{cluster=~\"$cluster\",namespace=~\"$namespace\"}[15m])) by (le))",
+                  "format": "time_series",
+                  "intervalFactor": 2,
+                  "legendFormat": "p99",
+                  "legendLink": null,
+                  "step": 10
+               },
+               {
+                  "expr": "histogram_quantile(0.50, sum(rate(loki_canary_metric_test_request_duration_seconds_bucket{cluster=~\"$cluster\",namespace=~\"$namespace\"}[15m])) by (le))",
+                  "format": "time_series",
+                  "intervalFactor": 2,
+                  "legendFormat": "p95",
+                  "legendLink": null,
+                  "step": 10
+               }
+            ],
+            "thresholds": [ ],
+            "timeFrom": null,
+            "timeShift": null,
+            "title": "Metric Test Query",
+            "tooltip": {
+               "shared": false,
+               "sort": 0,
+               "value_type": "individual"
+            },
+            "type": "graph",
+            "xaxis": {
+               "buckets": null,
+               "mode": "time",
+               "name": null,
+               "show": true,
+               "values": [ ]
+            },
+            "yaxes": [
+               {
+                  "format": "short",
+                  "label": null,
+                  "logBase": 1,
+                  "max": null,
+                  "min": 0,
+                  "show": true
+               },
+               {
+                  "format": "short",
+                  "label": null,
+                  "logBase": 1,
+                  "max": null,
+                  "min": null,
+                  "show": false
+               }
+            ]
+         },
+         {
+            "aliasColors": { },
+            "bars": false,
+            "dashLength": 10,
+            "dashes": false,
+            "datasource": "$datasource",
+            "fill": 1,
+            "gridPos": {
+               "h": 6,
+               "w": 12,
+               "x": 0,
+               "y": 20
+            },
+            "legend": {
+               "avg": false,
+               "current": false,
+               "max": false,
+               "min": false,
+               "show": true,
+               "total": false,
+               "values": false
+            },
+            "lines": true,
+            "linewidth": 1,
+            "links": [ ],
+            "nullPointMode": "null as zero",
+            "percentage": false,
+            "pointradius": 5,
+            "points": false,
+            "renderer": "flot",
+            "seriesOverrides": [ ],
+            "spaceLength": 10,
+            "span": 6,
+            "stack": false,
+            "steppedLine": false,
+            "targets": [
+               {
+                  "expr": "topk(20, (sum by (cluster, pod) (increase(loki_canary_spot_check_missing_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval]))/sum by (cluster, pod) (increase(loki_canary_spot_check_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])) * 100)) > 0",
+                  "format": "time_series",
+                  "intervalFactor": 2,
+                  "legendFormat": "",
+                  "legendLink": null,
+                  "step": 10
+               }
+            ],
+            "thresholds": [ ],
+            "timeFrom": null,
+            "timeShift": null,
+            "title": "Spot Check Missing %",
+            "tooltip": {
+               "shared": false,
+               "sort": 0,
+               "value_type": "individual"
+            },
+            "type": "graph",
+            "xaxis": {
+               "buckets": null,
+               "mode": "time",
+               "name": null,
+               "show": true,
+               "values": [ ]
+            },
+            "yaxes": [
+               {
+                  "format": "short",
+                  "label": null,
+                  "logBase": 1,
+                  "max": null,
+                  "min": 0,
+                  "show": true
+               },
+               {
+                  "format": "short",
+                  "label": null,
+                  "logBase": 1,
+                  "max": null,
+                  "min": null,
+                  "show": false
+               }
+            ]
+         },
+         {
+            "aliasColors": { },
+            "bars": false,
+            "dashLength": 10,
+            "dashes": false,
+            "datasource": "$datasource",
+            "fill": 1,
+            "gridPos": {
+               "h": 6,
+               "w": 12,
+               "x": 12,
+               "y": 20
+            },
+            "legend": {
+               "avg": false,
+               "current": false,
+               "max": false,
+               "min": false,
+               "show": true,
+               "total": false,
+               "values": false
+            },
+            "lines": true,
+            "linewidth": 1,
+            "links": [ ],
+            "nullPointMode": "null as zero",
+            "percentage": false,
+            "pointradius": 5,
+            "points": false,
+            "renderer": "flot",
+            "seriesOverrides": [ ],
+            "spaceLength": 10,
+            "span": 6,
+            "stack": false,
+            "steppedLine": false,
+            "targets": [
+               {
+                  "expr": "topk(20,(sum by (cluster, pod)(increase(loki_canary_missing_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval]))/sum by (cluster, pod)(increase(loki_canary_entries_total{cluster=~\"$cluster\",namespace=~\"$namespace\"}[$__rate_interval])))*100) > 0",
+                  "format": "time_series",
+                  "intervalFactor": 2,
+                  "legendFormat": "Missing {{ cluster }} {{ pod }}",
+                  "legendLink": null,
+                  "step": 10
+               }
+            ],
+            "thresholds": [ ],
+            "timeFrom": null,
+            "timeShift": null,
+            "title": "Missing logs",
+            "tooltip": {
+               "shared": true,
+               "sort": 2,
+               "value_type": "individual"
+            },
+            "type": "graph",
+            "xaxis": {
+               "buckets": null,
+               "mode": "time",
+               "name": null,
+               "show": true,
+               "values": [ ]
+            },
+            "yaxes": [
+               {
+                  "format": "short",
+                  "label": null,
+                  "logBase": 1,
+                  "max": null,
+                  "min": 0,
+                  "show": true
+               },
+               {
+                  "format": "short",
+                  "label": null,
+                  "logBase": 1,
+                  "max": null,
+                  "min": null,
+                  "show": false
+               }
+            ]
+         }
+      ],
+      "refresh": "10s",
+      "rows": null,
+      "schemaVersion": 27,
+      "style": "dark",
+      "tags": [
+         "loki"
+      ],
+      "templating": {
+         "list": [
+            {
+               "current": {
+                  "text": "default",
+                  "value": "default"
+               },
+               "hide": 0,
+               "label": "Data Source",
+               "name": "datasource",
+               "options": [ ],
+               "query": "prometheus",
+               "refresh": 1,
+               "regex": "",
+               "type": "datasource"
+            },
+            {
+               "allValue": null,
+               "current": {
+                  "text": "prod",
+                  "value": "prod"
+               },
+               "datasource": "$datasource",
+               "hide": 0,
+               "includeAll": false,
+               "label": "cluster",
+               "multi": false,
+               "name": "cluster",
+               "options": [ ],
+               "query": "label_values(loki_build_info, cluster)",
+               "refresh": 1,
+               "regex": "",
+               "sort": 2,
+               "tagValuesQuery": "",
+               "tags": [ ],
+               "tagsQuery": "",
+               "type": "query",
+               "useTags": false
+            },
+            {
+               "allValue": null,
+               "current": {
+                  "text": "prod",
+                  "value": "prod"
+               },
+               "datasource": "$datasource",
+               "hide": 0,
+               "includeAll": false,
+               "label": "namespace",
+               "multi": false,
+               "name": "namespace",
+               "options": [ ],
+               "query": "label_values(loki_build_info{cluster=~\"$cluster\"}, namespace)",
+               "refresh": 1,
+               "regex": "",
+               "sort": 2,
+               "tagValuesQuery": "",
+               "tags": [ ],
+               "tagsQuery": "",
+               "type": "query",
+               "useTags": false
+            }
+         ]
+      },
+      "time": {
+         "from": "now-1h",
+         "to": "now"
+      },
+      "timepicker": {
+         "refresh_intervals": [
+            "5s",
+            "10s",
+            "30s",
+            "1m",
+            "5m",
+            "15m",
+            "30m",
+            "1h",
+            "2h",
+            "1d"
+         ],
+         "time_options": [
+            "5m",
+            "15m",
+            "1h",
+            "6h",
+            "12h",
+            "24h",
+            "2d",
+            "7d",
+            "30d"
+         ]
+      },
+      "timezone": "utc",
+      "title": "Loki / Canary",
+      "uid": "loki-canary",
+      "version": 0
+   }
diff --git a/loki/README.md b/loki/README.md
index e0353312..8464b0eb 100644
--- a/loki/README.md
+++ b/loki/README.md
@@ -77,3 +77,22 @@ There's been some extra changes done interactively with Grafana UI.
 * Added a `Backend Path` panel
 * Added `disk usage` to Write and Backend path panel
 * Added `total pods` to Write, Read and Backend panels
+
+
+# Loki canary
+
+This one was generated from the [mixins](https://github.com/grafana/loki/tree/main/production/loki-mixin) following these steps:
+
+* Update config (mixin-ssd.libsonnet) with:
+   ```
+        {
+      _config+:: {
+        canary+: {
+          enabled: true,
+        },
+      },
+    }
+   ```
+* Generate with `mixtool generate all mixin-ssd.libsonnet`
+* Dashboard is `loki-canary.json`
+* Then we just changed the `uid` of the dashboard to `loki-canary`