From df6aa965d1856a9cf53c6bcac7961e38f8959b47 Mon Sep 17 00:00:00 2001
From: Joseph Mckulka <16840147+jmckulk@users.noreply.github.com>
Date: Wed, 13 Sep 2023 13:56:23 -0400
Subject: [PATCH] Updates for containers config (#362)

* Update dashboard datasources to PROMETHEUS

The datasource for containers is named PROMETHEUS. Update dashboards to
use the hardcoded name.

* Add copyright to prometheus config

* Add datasource and dashboard files for containers

* Remove redundant labels from prometheus scrape

These changes were merged but don't seem to be in main
https://github.com/CrunchyData/pgmonitor/pull/300

* Add WAL statistics dashboard

* Create separate scrape configs for v4 and v5

Update the alertmanater label to match k8s conventions:

https://kubernetes.io/docs/concepts/overview/working-with-objects/common-labels/#labels
---
 .../containers/crunchy_grafana_dashboards.yml |  16 +++
 .../containers/crunchy_grafana_datasource.yml |  18 +++
 grafana/containers/pgbackrest.json            |  12 +-
 grafana/containers/pod_details.json           |  22 +--
 grafana/containers/postgresql_details.json    |  42 +++---
 grafana/containers/postgresql_overview.json   |   6 +-
 .../containers/postgresql_service_health.json |  12 +-
 grafana/containers/prometheus_alerts.json     |  22 +--
 grafana/containers/query_statistics.json      | 134 ++++++++++++++++--
 .../crunchy-prometheus.yml.containers         | 100 +++++++------
 10 files changed, 273 insertions(+), 111 deletions(-)
 create mode 100644 grafana/containers/crunchy_grafana_dashboards.yml
 create mode 100644 grafana/containers/crunchy_grafana_datasource.yml

diff --git a/grafana/containers/crunchy_grafana_dashboards.yml b/grafana/containers/crunchy_grafana_dashboards.yml
new file mode 100644
index 00000000..4845ec8e
--- /dev/null
+++ b/grafana/containers/crunchy_grafana_dashboards.yml
@@ -0,0 +1,16 @@
+###
+#
+# Copyright © 2017-2023 Crunchy Data Solutions, Inc. All Rights Reserved.
+#
+###
+apiVersion: 1
+
+providers:
+- name: 'crunchy_dashboards'
+  orgId: 1
+  folder: ''
+  type: file
+  disableDeletion: false
+  updateIntervalSeconds: 3 #how often Grafana will scan for changed dashboards
+  options:
+    path: /etc/grafana/provisioning/dashboards
diff --git a/grafana/containers/crunchy_grafana_datasource.yml b/grafana/containers/crunchy_grafana_datasource.yml
new file mode 100644
index 00000000..3a9f276e
--- /dev/null
+++ b/grafana/containers/crunchy_grafana_datasource.yml
@@ -0,0 +1,18 @@
+###
+#
+# Copyright © 2017-2023 Crunchy Data Solutions, Inc. All Rights Reserved.
+#
+###
+
+# config file version
+apiVersion: 1
+
+datasources:
+  - name: PROMETHEUS
+    type: prometheus
+    access: proxy
+    url: http://$PROM_HOST:$PROM_PORT
+    isDefault: True
+    editable: False
+    orgId: 1
+    version: 1
diff --git a/grafana/containers/pgbackrest.json b/grafana/containers/pgbackrest.json
index 981a4434..d50f5c31 100644
--- a/grafana/containers/pgbackrest.json
+++ b/grafana/containers/pgbackrest.json
@@ -68,7 +68,7 @@
   ],
   "panels": [
     {
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -139,7 +139,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "custom": {},
@@ -267,7 +267,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "custom": {},
@@ -390,7 +390,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "description": "",
       "fieldConfig": {
         "defaults": {
@@ -517,7 +517,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "description": "",
       "fieldConfig": {
         "defaults": {
@@ -637,7 +637,7 @@
       {
         "allValue": null,
         "current": {},
-        "datasource": "${DS_PROMETHEUS}",
+        "datasource": "PROMETHEUS",
         "definition": "label_values(pg_cluster)",
         "description": null,
         "error": null,
diff --git a/grafana/containers/pod_details.json b/grafana/containers/pod_details.json
index dbcccd12..8ce6d395 100644
--- a/grafana/containers/pod_details.json
+++ b/grafana/containers/pod_details.json
@@ -69,7 +69,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "color": {},
@@ -194,7 +194,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "color": {},
@@ -323,7 +323,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "custom": {},
@@ -456,7 +456,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "custom": {},
@@ -568,7 +568,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "custom": {},
@@ -675,7 +675,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "custom": {},
@@ -791,7 +791,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "custom": {},
@@ -962,7 +962,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "custom": {},
@@ -1101,7 +1101,7 @@
       {
         "allValue": null,
         "current": {},
-        "datasource": "${DS_PROMETHEUS}",
+        "datasource": "PROMETHEUS",
         "definition": "label_values(pg_cluster)",
         "description": null,
         "error": null,
@@ -1128,7 +1128,7 @@
       {
         "allValue": null,
         "current": {},
-        "datasource": "${DS_PROMETHEUS}",
+        "datasource": "PROMETHEUS",
         "definition": "label_values({pg_cluster=\"[[cluster]]\"},pod)",
         "description": null,
         "error": null,
@@ -1172,7 +1172,7 @@
     ]
   },
   "timezone": "browser",
-  "title": "POD Details",
+  "title": "Pod Details",
   "uid": "4auP6Mk7k",
   "version": 1
 }
diff --git a/grafana/containers/postgresql_details.json b/grafana/containers/postgresql_details.json
index 923c2743..548b973b 100644
--- a/grafana/containers/postgresql_details.json
+++ b/grafana/containers/postgresql_details.json
@@ -75,7 +75,7 @@
   "panels": [
     {
       "cacheTimeout": null,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -166,7 +166,7 @@
     },
     {
       "cacheTimeout": null,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -248,7 +248,7 @@
     },
     {
       "cacheTimeout": null,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -327,7 +327,7 @@
     },
     {
       "cacheTimeout": null,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "description": "",
       "fieldConfig": {
         "defaults": {
@@ -407,7 +407,7 @@
     },
     {
       "cacheTimeout": null,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -488,7 +488,7 @@
     },
     {
       "cacheTimeout": null,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -569,7 +569,7 @@
     },
     {
       "cacheTimeout": null,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "description": "",
       "fieldConfig": {
         "defaults": {
@@ -654,7 +654,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "custom": {},
@@ -769,7 +769,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "custom": {},
@@ -888,7 +888,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "custom": {},
@@ -999,7 +999,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "description": "",
       "fieldConfig": {
         "defaults": {
@@ -1124,7 +1124,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "custom": {},
@@ -1268,7 +1268,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "custom": {},
@@ -1370,7 +1370,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "custom": {},
@@ -1522,7 +1522,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "description": "",
       "fieldConfig": {
         "defaults": {
@@ -1644,7 +1644,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "custom": {},
@@ -1783,7 +1783,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "custom": {},
@@ -1930,7 +1930,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "custom": {},
@@ -2042,7 +2042,7 @@
         "allFormat": "glob",
         "allValue": null,
         "current": {},
-        "datasource": "${DS_PROMETHEUS}",
+        "datasource": "PROMETHEUS",
         "definition": "",
         "description": null,
         "error": null,
@@ -2070,7 +2070,7 @@
         "allFormat": "glob",
         "allValue": ".*",
         "current": {},
-        "datasource": "${DS_PROMETHEUS}",
+        "datasource": "PROMETHEUS",
         "definition": "label_values({pg_cluster=\"[[cluster]]\"},pod)",
         "description": null,
         "error": null,
@@ -2098,7 +2098,7 @@
         "allFormat": "glob",
         "allValue": ".*",
         "current": {},
-        "datasource": "${DS_PROMETHEUS}",
+        "datasource": "PROMETHEUS",
         "definition": "label_values({pg_cluster=\"[[cluster]]\",pod=~\"[[pod]]\"},dbname)",
         "description": null,
         "error": null,
diff --git a/grafana/containers/postgresql_overview.json b/grafana/containers/postgresql_overview.json
index 1e565b54..61c2486f 100644
--- a/grafana/containers/postgresql_overview.json
+++ b/grafana/containers/postgresql_overview.json
@@ -51,7 +51,7 @@
   "panels": [
     {
       "cacheTimeout": null,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -71,7 +71,7 @@
             },
             {
               "targetBlank": true,
-              "title": "POD Details",
+              "title": "Pod Details",
               "url": "d/4auP6Mk7k/pod-details?$__all_variables"
             },
             {
@@ -187,7 +187,7 @@
         "allFormat": "glob",
         "allValue": null,
         "current": {},
-        "datasource": "${DS_PROMETHEUS}",
+        "datasource": "PROMETHEUS",
         "definition": "label_values(pg_cluster)",
         "description": null,
         "error": null,
diff --git a/grafana/containers/postgresql_service_health.json b/grafana/containers/postgresql_service_health.json
index 7190a8d4..2bee1d0b 100644
--- a/grafana/containers/postgresql_service_health.json
+++ b/grafana/containers/postgresql_service_health.json
@@ -66,7 +66,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "custom": {},
@@ -179,7 +179,7 @@
       "cacheTimeout": null,
       "dashLength": 10,
       "dashes": false,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "custom": {},
@@ -299,7 +299,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "description": "Errors",
       "fieldConfig": {
         "defaults": {
@@ -440,7 +440,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "custom": {},
@@ -572,7 +572,7 @@
       {
         "allValue": null,
         "current": {},
-        "datasource": "${DS_PROMETHEUS}",
+        "datasource": "PROMETHEUS",
         "definition": "label_values(pg_cluster)",
         "description": null,
         "error": null,
@@ -599,7 +599,7 @@
       {
         "allValue": null,
         "current": {},
-        "datasource": "${DS_PROMETHEUS}",
+        "datasource": "PROMETHEUS",
         "definition": "label_values({pg_cluster=\"[[cluster]]\"},role)",
         "description": null,
         "error": null,
diff --git a/grafana/containers/prometheus_alerts.json b/grafana/containers/prometheus_alerts.json
index 34d11f34..ef8fb41a 100644
--- a/grafana/containers/prometheus_alerts.json
+++ b/grafana/containers/prometheus_alerts.json
@@ -65,7 +65,7 @@
   "panels": [
     {
       "collapsed": false,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -80,7 +80,7 @@
     },
     {
       "cacheTimeout": null,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "description": "",
       "fieldConfig": {
         "defaults": {
@@ -150,7 +150,7 @@
     },
     {
       "cacheTimeout": null,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "description": "",
       "fieldConfig": {
         "defaults": {
@@ -222,7 +222,7 @@
     },
     {
       "cacheTimeout": null,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "description": "",
       "fieldConfig": {
         "defaults": {
@@ -294,7 +294,7 @@
     },
     {
       "collapsed": false,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -309,7 +309,7 @@
     },
     {
       "cacheTimeout": null,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -406,7 +406,7 @@
     },
     {
       "cacheTimeout": null,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -475,7 +475,7 @@
     },
     {
       "cacheTimeout": null,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -545,7 +545,7 @@
     },
     {
       "collapsed": false,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "gridPos": {
         "h": 1,
         "w": 24,
@@ -559,7 +559,7 @@
       "type": "row"
     },
     {
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "color": {
@@ -753,7 +753,7 @@
       "type": "table"
     },
     {
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "color": {
diff --git a/grafana/containers/query_statistics.json b/grafana/containers/query_statistics.json
index 726ba8c3..a17b06ed 100644
--- a/grafana/containers/query_statistics.json
+++ b/grafana/containers/query_statistics.json
@@ -71,7 +71,7 @@
   ],
   "panels": [
     {
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "custom": {},
@@ -125,7 +125,7 @@
       "type": "stat"
     },
     {
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "custom": {},
@@ -180,7 +180,7 @@
       "type": "stat"
     },
     {
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "custom": {},
@@ -237,7 +237,7 @@
       "type": "stat"
     },
     {
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "custom": {},
@@ -298,7 +298,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "custom": {}
@@ -393,7 +393,7 @@
       }
     },
     {
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "custom": {
@@ -569,7 +569,7 @@
       "type": "table"
     },
     {
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "custom": {
@@ -745,7 +745,7 @@
       "type": "table"
     },
     {
-      "datasource": "${DS_PROMETHEUS}",
+      "datasource": "PROMETHEUS",
       "fieldConfig": {
         "defaults": {
           "custom": {
@@ -917,6 +917,116 @@
         }
       ],
       "type": "table"
+    },
+    {
+      "datasource": "PROMETHEUS",
+      "fieldConfig": {
+        "defaults": {
+          "custom": {
+            "displayMode": "auto",
+            "filterable": false,
+            "inspect": false
+          },
+          "mappings": [],
+          "thresholds": {
+            "mode": "absolute",
+            "steps": [
+              {
+                "color": "green",
+                "value": null
+              },
+              {
+                "color": "red",
+                "value": 80
+              }
+            ]
+          }
+        },
+        "overrides": [
+          {
+            "matcher": {
+              "id": "byName",
+              "options": "WAL Size"
+            },
+            "properties": [
+              {
+                "id": "unit",
+                "value": "bytes"
+              },
+              {
+                "id": "custom.displayMode",
+                "value": "auto"
+              }
+            ]
+          }
+        ]
+      },
+      "gridPos": {
+        "h": 8,
+        "w": 23,
+        "x": 0,
+        "y": 34
+      },
+      "id": 12,
+      "options": {
+        "footer": {
+          "fields": "",
+          "reducer": [
+            "sum"
+          ],
+          "show": false
+        },
+        "showHeader": true,
+        "sortBy": [
+          {
+            "desc": true,
+            "displayName": "WAL Size"
+          }
+        ]
+      },
+      "pluginVersion": "8.5.15",
+      "targets": [
+        {
+          "datasource": "PROMETHEUS",
+          "expr": "ccp_pg_stat_statements_top_wal_bytes{pg_cluster=\"[[cluster]]\", dbname=~\"[[dbname]]\", role=\"[[role]]\"}",
+          "format": "table",
+          "instant": true,
+          "interval": "",
+          "legendFormat": "",
+          "refId": "A"
+        }
+      ],
+      "title": "Query Total WAL Genterated (Bytes)",
+      "transformations": [
+        {
+          "id": "organize",
+          "options": {
+            "excludeByName": {
+              "Time": true,
+              "__name__": true,
+              "exp_type": true,
+              "job": true,
+              "role": false
+            },
+            "indexByName": {
+              "Time": 6,
+              "Value": 5,
+              "__name__": 7,
+              "dbname": 0,
+              "exp_type": 8,
+              "instance": 2,
+              "job": 9,
+              "query": 3,
+              "queryid": 4,
+              "role": 1
+            },
+            "renameByName": {
+              "Value": "WAL Size"
+            }
+          }
+        }
+      ],
+      "type": "table"
     }
   ],
   "refresh": "15m",
@@ -930,7 +1040,7 @@
       {
         "allValue": null,
         "current": {},
-        "datasource": "${DS_PROMETHEUS}",
+        "datasource": "PROMETHEUS",
         "definition": "label_values(pg_cluster)",
         "description": null,
         "error": null,
@@ -957,7 +1067,7 @@
       {
         "allValue": null,
         "current": {},
-        "datasource": "${DS_PROMETHEUS}",
+        "datasource": "PROMETHEUS",
         "definition": "label_values({pg_cluster=\"[[cluster]]\"},role)",
         "description": null,
         "error": null,
@@ -984,7 +1094,7 @@
       {
         "allValue": null,
         "current": {},
-        "datasource": "${DS_PROMETHEUS}",
+        "datasource": "PROMETHEUS",
         "definition": "label_values(ccp_database_size_bytes{pg_cluster=\"[[cluster]]\"},dbname)",
         "description": null,
         "error": null,
@@ -1011,7 +1121,7 @@
       {
         "allValue": null,
         "current": {},
-        "datasource": "${DS_PROMETHEUS}",
+        "datasource": "PROMETHEUS",
         "definition": "label_values(ccp_pg_stat_statements_total_calls_count{pg_cluster=\"[[cluster]]\", dbname=~\"[[dbname]]\"},exported_role)",
         "description": null,
         "error": null,
diff --git a/prometheus/containers/crunchy-prometheus.yml.containers b/prometheus/containers/crunchy-prometheus.yml.containers
index a391cf11..e60b0b92 100644
--- a/prometheus/containers/crunchy-prometheus.yml.containers
+++ b/prometheus/containers/crunchy-prometheus.yml.containers
@@ -1,3 +1,8 @@
+###
+#
+# Copyright © 2017-2023 Crunchy Data Solutions, Inc. All Rights Reserved.
+#
+###
 ---
 global:
   scrape_interval: 15s
@@ -8,60 +13,73 @@ scrape_configs:
 - job_name: 'crunchy-postgres-exporter'
   kubernetes_sd_configs:
   - role: pod
-
+    selectors:
+    - role: pod
+      label: postgres-operator.crunchydata.com/crunchy-postgres-exporter=true
   relabel_configs:
-  - source_labels: [__meta_kubernetes_pod_label_postgres_operator_crunchydata_com_crunchy_postgres_exporter,__meta_kubernetes_pod_label_crunchy_postgres_exporter]
-    action: keep
-    regex: true
-    separator: ""
-  - source_labels: [__meta_kubernetes_pod_container_port_number]
-    action: drop
-    regex: 5432
-  - source_labels: [__meta_kubernetes_pod_container_port_number]
-    action: drop
-    regex: 10000
-  - source_labels: [__meta_kubernetes_pod_container_port_number]
-    action: drop
-    regex: 8009
-  - source_labels: [__meta_kubernetes_pod_container_port_number]
-    action: drop
-    regex: 2022
+  # Keep exporter port and drop all others
   - source_labels: [__meta_kubernetes_pod_container_port_number]
-    action: drop
-    regex: ^$
+    action: keep
+    regex: 9187
+  # Set label for namespace
   - source_labels: [__meta_kubernetes_namespace]
-    action: replace
     target_label: kubernetes_namespace
+  # Set label for pod name
   - source_labels: [__meta_kubernetes_pod_name]
     target_label: pod
-  - source_labels: [__meta_kubernetes_pod_label_postgres_operator_crunchydata_com_cluster,__meta_kubernetes_pod_label_pg_cluster]
-    target_label: cluster
-    separator: ""
-    replacement: '$1'
-  - source_labels: [__meta_kubernetes_namespace,cluster]
+  # Convert namespace and cluster name to pg_cluster=namespace:cluster
+  - source_labels: [__meta_kubernetes_namespace,__meta_kubernetes_pod_label_postgres_operator_crunchydata_com_cluster]
     target_label: pg_cluster
     separator: ":"
     replacement: '$1$2'
+  # Convert kubernetes pod ip to ip
   - source_labels: [__meta_kubernetes_pod_ip]
     target_label: ip
-    replacement: '$1'
-  - source_labels: [__meta_kubernetes_pod_label_postgres_operator_crunchydata_com_instance,__meta_kubernetes_pod_label_deployment_name]
+  # Convert postgres-operator.crunchydata.com/instance to deployment
+  - source_labels: [__meta_kubernetes_pod_label_postgres_operator_crunchydata_com_instance]
     target_label: deployment
-    replacement: '$1'
-    separator: ""
-  - source_labels: [__meta_kubernetes_pod_label_postgres_operator_crunchydata_com_role,__meta_kubernetes_pod_label_role]
+  # Convert postgres-operator.crunchydata.com/role to role
+  - source_labels: [__meta_kubernetes_pod_label_postgres_operator_crunchydata_com_role]
     target_label: role
-    replacement: '$1'
-    separator: ""
-  - source_labels: [dbname]
-    target_label: dbname
-    replacement: '$1'
-  - source_labels: [relname]
-    target_label: relname
-    replacement: '$1'
-  - source_labels: [schemaname]
-    target_label: schemaname
-    replacement: '$1'
 
+- job_name: 'crunchy-postgres-exporter-v4'
+  kubernetes_sd_configs:
+  - role: pod
+    selectors:
+    - role: pod
+      label: crunchy-postgres-exporter=true
+
+  relabel_configs:
+  # Keep exporter port and drop all others
+  - source_labels: [__meta_kubernetes_pod_container_port_number]
+    action: keep
+    regex: 9187
+  # Set label for namespace
+  - source_labels: [__meta_kubernetes_namespace]
+    target_label: kubernetes_namespace
+  # Set label for pod name
+  - source_labels: [__meta_kubernetes_pod_name]
+    target_label: pod
+  # Convert namespace and cluster name to pg_cluster=namespace:cluster
+  - source_labels: [__meta_kubernetes_namespace,__meta_kubernetes_pod_label_pg_cluster]
+    target_label: pg_cluster
+    separator: ":"
+    replacement: '$1$2'
+  # Convert kubernetes pod ip to ip
+  - source_labels: [__meta_kubernetes_pod_ip]
+    target_label: ip
+  # Set deployment_name as deployment label
+  - source_labels: [__meta_kubernetes_pod_label_deployment_name]
+    target_label: deployment
+  # Set label for role
+  - source_labels: [__meta_kubernetes_pod_label_role]
+    target_label: role
 rule_files:
   - /etc/prometheus/alert-rules.d/*.yml
+alerting:
+  alertmanagers:
+  - kubernetes_sd_configs:
+    - role: pod
+      selectors:
+      - role: pod
+        label: app.kubernetes.io/component=crunchy-alertmanager