From 3162b6c68acbe94f77a9f71e7be66684b2e8ab25 Mon Sep 17 00:00:00 2001 From: Keith Fiske Date: Fri, 1 Sep 2023 17:53:08 -0400 Subject: [PATCH 1/4] feat: add support for new alertmanager interfaces in grafana --- changelogs/fragments/250.yml | 3 + changelogs/fragments/template.yml.example | 27 +++ grafana/common/Prometheus_Alerts.json | 189 +++++------------- grafana/common/crunchy_grafana_datasource.yml | 20 +- 4 files changed, 104 insertions(+), 135 deletions(-) create mode 100644 changelogs/fragments/250.yml create mode 100644 changelogs/fragments/template.yml.example diff --git a/changelogs/fragments/250.yml b/changelogs/fragments/250.yml new file mode 100644 index 00000000..d84185c2 --- /dev/null +++ b/changelogs/fragments/250.yml @@ -0,0 +1,3 @@ +major_changes: + - grafana - add alertmanager datasource to grafana to allow direct alert management via grafana + - grafana - update Prometheus Alerts dashboard to include new alert panel interface for active alerts diff --git a/changelogs/fragments/template.yml.example b/changelogs/fragments/template.yml.example new file mode 100644 index 00000000..ac4cd8ea --- /dev/null +++ b/changelogs/fragments/template.yml.example @@ -0,0 +1,27 @@ +--- +breaking_changes: + - area affected (roles, specific role, playbooks, docs) - This category should list all changes to features which absolutely require attention from users when upgrading, because an existing behavior is changed. This section should only appear in a initial major release (`x.0.0`) according to semantic versioning. + +major_changes: + - area affected (roles, specific role, playbooks, docs) - This category contains major changes to the project. It should only contain a few items per major version, describing high-level changes. This section should not appear in patch releases according to semantic versioning. + +minor_changes: + - area affected (roles, specific role, playbooks, docs) - This category should mention all new features not contained elsewhere. This section should not appear in patch releases according to semantic versioning. + +removed_features: + - area affected (roles, specific role, playbooks, docs) - This category should mention all features that have been removed in this release. This section should only appear in a initial major release (`x.0.0`) according to semantic versioning. + +deprecated_features: + - area affected (roles, specific role, playbooks, docs) - This category should contain all features which have been deprecated and will be removed in a future release. This section should not appear in patch releases according to semantic versioning. + +security_fixes: + - area affected (roles, specific role, playbooks, docs) - This category should mention all security relevant fixes, including CVEs if available. + +bugfixes: + - area affected (roles, specific role, playbooks, docs) - This category should be a list of all bug fixes which fix a bug that was present in a previous version. + +known_issues: + - area affected (roles, specific role, playbooks, docs) - This category should mention known issues that are currently not fixed or will not be fixed. + +trivial: + - area affected (roles, specific role, playbooks, docs) - This category will **not be shown** in the changelog. It can be used to describe changes that are not touching user-facing code, like changes in tests. This is useful if every PR is required to have a changelog fragment. diff --git a/grafana/common/Prometheus_Alerts.json b/grafana/common/Prometheus_Alerts.json index 91bffc84..3a70e2ca 100644 --- a/grafana/common/Prometheus_Alerts.json +++ b/grafana/common/Prometheus_Alerts.json @@ -3,164 +3,78 @@ "list": [ { "builtIn": 1, - "datasource": "-- Grafana --", + "datasource": { + "type": "datasource", + "uid": "grafana" + }, "enable": true, "hide": true, "iconColor": "rgba(0, 211, 255, 1)", "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, "type": "dashboard" } ] }, "editable": false, - "gnetId": null, + "fiscalYearStartMonth": 0, "graphTooltip": 0, - "id": 8, "links": [], + "liveNow": false, "panels": [ { - "columns": [], - "datasource": null, - "fontSize": "100%", + "datasource": { + "type": "prometheus", + "uid": "crunchyprometheus" + }, "gridPos": { - "h": 7, + "h": 8, "w": 24, "x": 0, "y": 0 }, - "id": 2, - "links": [], - "options": {}, - "pageSize": null, - "scroll": true, - "showHeader": true, - "sort": { - "col": 0, - "desc": true - }, - "styles": [ - { - "alias": "Time", - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "link": false, - "pattern": "Time", - "type": "date" - }, - { - "alias": "", - "colorMode": "row", - "colors": [ - "rgba(50, 172, 45, 0.97)", - "rgba(237, 129, 40, 0.89)", - "rgba(245, 54, 54, 0.9)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 0, - "link": false, - "mappingType": 1, - "pattern": "severity_num", - "thresholds": [ - "200", - "300" - ], - "type": "string", - "unit": "none" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "mappingType": 1, - "pattern": "__name__", - "thresholds": [], - "type": "hidden", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "mappingType": 1, - "pattern": "alertstate", - "thresholds": [], - "type": "hidden", - "unit": "short" + "id": 5, + "options": { + "alertInstanceLabelFilter": "", + "alertName": "", + "dashboardAlerts": false, + "datasource": "PROMETHEUS", + "groupBy": [], + "groupMode": "default", + "maxItems": 20, + "sortOrder": 1, + "stateFilter": { + "error": true, + "firing": true, + "noData": false, + "normal": false, + "pending": true }, - { - "alias": "", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "mappingType": 1, - "pattern": "Value", - "thresholds": [], - "type": "hidden", - "unit": "short" - }, - { - "alias": "", - "colorMode": null, - "colors": [ - "rgba(245, 54, 54, 0.9)", - "rgba(237, 129, 40, 0.89)", - "rgba(50, 172, 45, 0.97)" - ], - "dateFormat": "YYYY-MM-DD HH:mm:ss", - "decimals": 2, - "mappingType": 1, - "pattern": "alert_value", - "thresholds": [], - "type": "number", - "unit": "none" - } - ], - "targets": [ - { - "expr": "ALERTS{alertstate=\"firing\"}", - "format": "table", - "instant": true, - "interval": "", - "intervalFactor": 4, - "legendFormat": "", - "refId": "A" - } - ], - "timeFrom": null, + "viewMode": "list" + }, "title": "Active Alerts", - "transform": "table", - "type": "table" + "type": "alertlist" }, { "columns": [], - "datasource": null, + "datasource": { + "type": "prometheus", + "uid": "crunchyprometheus" + }, "fontSize": "100%", "gridPos": { "h": 7, "w": 24, "x": 0, - "y": 7 + "y": 8 }, "id": 3, "links": [], - "options": {}, - "pageSize": null, "scroll": true, "showHeader": true, "sort": { @@ -170,6 +84,7 @@ "styles": [ { "alias": "Time", + "align": "auto", "dateFormat": "YYYY-MM-DD HH:mm:ss", "link": false, "pattern": "Time", @@ -177,6 +92,7 @@ }, { "alias": "", + "align": "auto", "colorMode": "row", "colors": [ "rgba(50, 172, 45, 0.97)", @@ -197,7 +113,7 @@ }, { "alias": "", - "colorMode": null, + "align": "auto", "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", @@ -213,7 +129,7 @@ }, { "alias": "", - "colorMode": null, + "align": "auto", "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", @@ -229,7 +145,7 @@ }, { "alias": "", - "colorMode": null, + "align": "auto", "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", @@ -245,7 +161,7 @@ }, { "alias": "", - "colorMode": null, + "align": "auto", "colors": [ "rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", @@ -262,6 +178,10 @@ ], "targets": [ { + "datasource": { + "type": "prometheus", + "uid": "crunchyprometheus" + }, "expr": "ALERTS{alertstate=\"firing\"}", "format": "table", "instant": false, @@ -274,11 +194,11 @@ "timeFrom": "1w", "title": "Alert History (1 week)", "transform": "table", - "type": "table" + "type": "table-old" } ], "refresh": "5m", - "schemaVersion": 21, + "schemaVersion": 37, "style": "dark", "tags": [], "templating": { @@ -304,5 +224,6 @@ "timezone": "", "title": "Prometheus Alerts", "uid": "QTsttxNmk", - "version": 2 + "version": 1, + "weekStart": "" } diff --git a/grafana/common/crunchy_grafana_datasource.yml b/grafana/common/crunchy_grafana_datasource.yml index 5657a880..a9581fcf 100644 --- a/grafana/common/crunchy_grafana_datasource.yml +++ b/grafana/common/crunchy_grafana_datasource.yml @@ -7,9 +7,15 @@ # config file version apiVersion: 1 +deleteDatasources: + - name: PROMETHEUS + orgId: 1 + - name: ALERTMANAGER + orgId: 1 + datasources: - name: PROMETHEUS - uid: PDC1078F23EBDF0E5 + uid: crunchyprometheus type: prometheus access: proxy url: http://localhost:9090 @@ -17,4 +23,16 @@ datasources: editable: False orgId: 1 version: 1 + jsonData: + manageAlerts: 'true' + alertmanagerUid: 'crunchyalertmanager' + - name: ALERTMANAGER + name: 'ALERTMANAGER' + uid: crunchyalertmanager + type: alertmanager + url: http://localhost:9093 + access: proxy + jsonData: + implementation: 'prometheus' + handleGrafanaManagedAlerts: 'false' From 3b2d2d0c40908f53a94e99b59d677e5776b99431 Mon Sep 17 00:00:00 2001 From: Keith Fiske Date: Tue, 5 Sep 2023 08:54:24 -0400 Subject: [PATCH 2/4] Apply suggestions from code review Co-authored-by: Douglas J Hunley --- changelogs/fragments/250.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/changelogs/fragments/250.yml b/changelogs/fragments/250.yml index d84185c2..bbbe9995 100644 --- a/changelogs/fragments/250.yml +++ b/changelogs/fragments/250.yml @@ -1,3 +1,3 @@ major_changes: - - grafana - add alertmanager datasource to grafana to allow direct alert management via grafana - - grafana - update Prometheus Alerts dashboard to include new alert panel interface for active alerts + - grafana - add Alertmanager datasource to Grafana to allow direct alert management via Grafana + - grafana - update 'Prometheus Alerts' dashboard to include new alert panel interface for active alerts From f134e8d63ffaed3d40957b1f0a785120069dd949 Mon Sep 17 00:00:00 2001 From: Keith Fiske Date: Tue, 5 Sep 2023 14:33:27 -0400 Subject: [PATCH 3/4] docs: update documentation for alertmanager being a datasource --- grafana/common/crunchy_grafana_datasource.yml | 1 - hugo/content/grafana/_index.md | 8 +++----- 2 files changed, 3 insertions(+), 6 deletions(-) diff --git a/grafana/common/crunchy_grafana_datasource.yml b/grafana/common/crunchy_grafana_datasource.yml index a9581fcf..daec3029 100644 --- a/grafana/common/crunchy_grafana_datasource.yml +++ b/grafana/common/crunchy_grafana_datasource.yml @@ -27,7 +27,6 @@ datasources: manageAlerts: 'true' alertmanagerUid: 'crunchyalertmanager' - name: ALERTMANAGER - name: 'ALERTMANAGER' uid: crunchyalertmanager type: alertmanager url: http://localhost:9093 diff --git a/hugo/content/grafana/_index.md b/hugo/content/grafana/_index.md index 0b3391bd..3d65cd74 100644 --- a/hugo/content/grafana/_index.md +++ b/hugo/content/grafana/_index.md @@ -35,7 +35,7 @@ pgMonitor comes with several dashboards ready to be used with automatic provisio |OS Overview | Overview.json| Provides an overview that shows the up status of each system monitored by pgMonitor. | ||| |ETCD Details | ETCD_Details.json | Provides details on the status of the ETCD cluster monitored by pgMonitor. | -|Prometheus Alerts | Prometheus_Alerts.json| Provides a summary list of current and recent alerts that have fired in Prometheus. | +|Prometheus Alerts | Prometheus_Alerts.json| Provides a summary list of current and recent alerts that have fired in Prometheus. Interaction with the Alertmanager to silence alerts is possible from the Alerting menu in Grafana. | ## Installation {#installation} @@ -68,7 +68,7 @@ mkdir -p /etc/grafana/crunchy_dashboards | grafana/crunchy_grafana_datasource.yml | /etc/grafana/provisioning/datasources/datasource.yml | | grafana/crunchy_grafana_dashboards.yml | /etc/grafana/provisioning/dashboards/dashboards.yml | -Review the {{< shell >}}crunchy_grafana_datasource.yml{{< /shell >}}} file to ensure it is looking at your Prometheus database. The included file assumes Grafana and Prometheus are running on the same system. DO NOT CHANGE the datasource {{< yaml >}}name{{< /yaml >}} if you will be using the dashboards provided in this repo. They assume that name and will not work otherwise. Any other options can be changed as needed. Save the {{< shell >}}crunchy_grafana_datasource.yml{{< /shell >}} file and rename it to {{< shell >}}/etc/grafana/provisioning/datasources/datasources.yml{{< /shell >}}. Restart grafana and confirm through the web interface that the datasource was provisioned and working. +Review the {{< shell >}}crunchy_grafana_datasource.yml{{< /shell >}}} file to ensure it is looking at your Prometheus database. The included file assumes Grafana, Prometheus, and Alertmanager are running on the same system. DO NOT CHANGE the datasource {{< yaml >}}uid{{< /yaml >}} or {{< yaml >}}name{{< /yaml >}} fields if you will be using the dashboards provided in this repo. They assume those values and will not work otherwise. Any other options can be changed as needed. Save the {{< shell >}}crunchy_grafana_datasource.yml{{< /shell >}} file and rename it to {{< shell >}}/etc/grafana/provisioning/datasources/datasources.yml{{< /shell >}}. Restart grafana and confirm through the web interface that the datasource was provisioned and working. Review the {{< shell >}}crunchy_grafana_dashboards.yml{{< /shell >}} file to ensure it's looking at where you stored the provided dashboards. By default it is looking in {{< shell >}}/etc/grafana/crunchy_dashboards{{< /shell >}}. Save this file and rename it to {{< shell >}}/etc/grafana/provisioning/dashboards/dashboards.yml{{< /shell >}}. Restart grafana so it picks up the new config. @@ -76,8 +76,6 @@ Save all of the desired .json dashboard files to the {{< shell >}}/etc/grafana/c ## Upgrading {#upgrading} -If you'd like to take advantage of the new provisioning system in Grafana 5 provided by pgmonitor 2.x, we recommend either renaming or deleting your existing datasources and dashboards so there are no issues when the provisioned versions are imported. - Please review the ChangeLog for pgMonitor and take note of any changes to metric names and/or dashboards. Note that if you are using the included dashboards that are managed via the provisioning system, they will automatically be updated. If you've made any changes to configuration files and kept their default names, the package will not overwrite them and will instead make a new file with an {{< shell >}}*.rpmnew{{< /shell >}} extension. You can compare your file and the new one and incorporate any changes as needed or desired. ## Setup {#setup} @@ -125,7 +123,7 @@ Navigate to the web interface: https://<ip-address>:3000. Log in with admi Grafana provides the ability to automatically provision datasources and dashboards via configuration files instead of having to manually import them either through the web interface or the API. Note that provisioned dashboards can no longer be directly edited and saved via the web interface. See the Grafana documentation for how to edit/save provisioned dashboards: http://docs.grafana.org/administration/provisioning/#making-changes-to-a-provisioned-dashboard. If you'd like to customize these dashboards, we recommend first adding them via provisioning then saving them with a new name. You can then either manage them via the web interface or add them to the provisioning system. -The extras package takes care of putting all these files in place. If you did not use the crunchy package to install Grafana, see the additional instructions above. Once that is done, the only additional setup that needs to be done is to set the "provisioning" option in the `grafana.ini` to point to the top level directory if it hasn't been done already. If you're upgrading from Grafana 4.x to 5.x, you will have to add the "provisioning" option to the `[paths]` section of the `grafana.ini` file. Once that is done, just restart grafana and all datasources and dashboards should appear. +The extras package takes care of putting all these files in place. If you did not use the crunchy package to install Grafana, see the additional instructions above. Once that is done, the only additional setup that needs to be done is to set the "provisioning" option in the `grafana.ini` to point to the top level directory if it hasn't been done already. ```ini [paths] From e5b8fef7ec2760e4a9cbeeb8282afc478faad980 Mon Sep 17 00:00:00 2001 From: Keith Fiske Date: Wed, 6 Sep 2023 10:14:35 -0400 Subject: [PATCH 4/4] Apply suggestions from code review Co-authored-by: Douglas J Hunley --- hugo/content/grafana/_index.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/hugo/content/grafana/_index.md b/hugo/content/grafana/_index.md index 3d65cd74..ffd0726c 100644 --- a/hugo/content/grafana/_index.md +++ b/hugo/content/grafana/_index.md @@ -123,7 +123,7 @@ Navigate to the web interface: https://<ip-address>:3000. Log in with admi Grafana provides the ability to automatically provision datasources and dashboards via configuration files instead of having to manually import them either through the web interface or the API. Note that provisioned dashboards can no longer be directly edited and saved via the web interface. See the Grafana documentation for how to edit/save provisioned dashboards: http://docs.grafana.org/administration/provisioning/#making-changes-to-a-provisioned-dashboard. If you'd like to customize these dashboards, we recommend first adding them via provisioning then saving them with a new name. You can then either manage them via the web interface or add them to the provisioning system. -The extras package takes care of putting all these files in place. If you did not use the crunchy package to install Grafana, see the additional instructions above. Once that is done, the only additional setup that needs to be done is to set the "provisioning" option in the `grafana.ini` to point to the top level directory if it hasn't been done already. +The extras package takes care of putting all these files in place. If you did not use the Crunchy package to install Grafana, see the additional instructions above. Once that is done, the only additional setup that needs to be done is to set the "provisioning" option in the `grafana.ini` to point to the top level directory if it hasn't been done already. ```ini [paths]