From 787e227982abddac30d74244dd3bc5b4841a7629 Mon Sep 17 00:00:00 2001 From: Jake Hutchinson Date: Fri, 15 Sep 2023 16:09:51 +0000 Subject: [PATCH 1/2] Add os_exporter playbook, config and dashboards --- doc/source/configuration/monitoring.rst | 36 + .../ansible/deploy-os-capacity-exporter.yml | 29 + .../templates/os_capacity-clouds.yml.j2 | 10 + .../openstack/grafana_cloud_dashboard.json | 898 ++++++++++++++++++ .../openstack/grafana_project_dashboard.json | 880 +++++++++++++++++ .../config/haproxy/services.d/os_exporter.cfg | 19 + .../prometheus.yml.d/70-oscapacity.yml.j2 | 9 + .../notes/os-capacity-94006f03f16583e4.yaml | 13 + 8 files changed, 1894 insertions(+) create mode 100644 etc/kayobe/ansible/deploy-os-capacity-exporter.yml create mode 100644 etc/kayobe/ansible/templates/os_capacity-clouds.yml.j2 create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/openstack/grafana_cloud_dashboard.json create mode 100644 etc/kayobe/kolla/config/grafana/dashboards/openstack/grafana_project_dashboard.json create mode 100644 etc/kayobe/kolla/config/haproxy/services.d/os_exporter.cfg create mode 100644 etc/kayobe/kolla/config/prometheus/prometheus.yml.d/70-oscapacity.yml.j2 create mode 100644 releasenotes/notes/os-capacity-94006f03f16583e4.yaml diff --git a/doc/source/configuration/monitoring.rst b/doc/source/configuration/monitoring.rst index 0de14f8c6..880ca0032 100644 --- a/doc/source/configuration/monitoring.rst +++ b/doc/source/configuration/monitoring.rst @@ -136,3 +136,39 @@ mgrs group and list them as the endpoints for prometheus. Additionally, depending on your configuration, you may need set the ``kolla_enable_prometheus_ceph_mgr_exporter`` variable to ``true`` in order to enable the ceph mgr exporter. + +OpenStack Capacity +================== + +OpenStack Capacity allows you to see how much space you have avaliable +in your cloud. StackHPC Kayobe Config includes this exporter by default +and it's necessary that some variables are set to allow deployment. + +To successfully deploy OpenStack Capacity, you are required to specify +the OpenStack application credentials in ``kayobe/secrets.yml`` as: + +.. code-block:: yaml + + secrets_os_exporter_auth_url: + secrets_os_exporter_credential_id: + secrets_os_exporter_credential_secret: + +After defining your credentials, You may deploy OpenStack Capacity +using the ``ansible/deploy-os-capacity-exporter.yml`` Ansible playbook +via Kayobe. + +.. code-block:: console + + kayobe playbook run ansible/deploy-os-capacity-exporter.yml + +It is required that you re-configure the Prometheus, Grafana and HAProxy +services following deployment, to do this run the following Kayobe command. + +.. code-block:: console + + kayobe overcloud service reconfigure -kt grafana,prometheus,haproxy + +If you notice ``HaproxyServerDown`` or ``HaproxyBackendDown`` prometheus +alerts after deployment it's likely the os_exporter secrets have not been +set correctly, double check you have entered the correct authentication +information appropiate to your cloud and re-deploy. diff --git a/etc/kayobe/ansible/deploy-os-capacity-exporter.yml b/etc/kayobe/ansible/deploy-os-capacity-exporter.yml new file mode 100644 index 000000000..4eeb69431 --- /dev/null +++ b/etc/kayobe/ansible/deploy-os-capacity-exporter.yml @@ -0,0 +1,29 @@ +--- +- hosts: monitoring + gather_facts: false + + tasks: + - name: Create os-capacity directory + ansible.builtin.file: + path: /opt/kayobe/os-capacity/ + state: directory + + - name: Template clouds.yml + ansible.builtin.template: + src: templates/os_capacity-clouds.yml.j2 + dest: /opt/kayobe/os-capacity/clouds.yaml + + - name: Ensure os_capacity container is running + docker_container: + name: os_capacity + image: ghcr.io/stackhpc/os-capacity:master + env: + OS_CLOUD: openstack + OS_CLIENT_CONFIG_FILE: /etc/openstack/clouds.yaml + mounts: + - type: bind + source: /opt/kayobe/os-capacity/ + target: /etc/openstack/ + network_mode: host + restart_policy: unless-stopped + become: true diff --git a/etc/kayobe/ansible/templates/os_capacity-clouds.yml.j2 b/etc/kayobe/ansible/templates/os_capacity-clouds.yml.j2 new file mode 100644 index 000000000..89d66c0bc --- /dev/null +++ b/etc/kayobe/ansible/templates/os_capacity-clouds.yml.j2 @@ -0,0 +1,10 @@ +clouds: + openstack: + auth: + auth_url: "{{ secrets_os_exporter_auth_url }}" + application_credential_id: "{{ secrets_os_exporter_credential_id }}" + application_credential_secret: "{{ secrets_os_exporter_credential_secret }}" + region_name: "RegionOne" + interface: "internal" + identity_api_version: 3 + auth_type: "v3applicationcredential" diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/grafana_cloud_dashboard.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/grafana_cloud_dashboard.json new file mode 100644 index 000000000..a777c332e --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/grafana_cloud_dashboard.json @@ -0,0 +1,898 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2084495, + "links": [], + "liveNow": false, + "panels": [ + { + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 18, + "title": "Summary Flavor Capacity", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "noValue": "0", + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 4, + "w": 2.4, + "x": 0, + "y": 1 + }, + "id": 20, + "maxPerRow": 12, + "options": { + "colorMode": "value", + "graphMode": "area", + "justifyMode": "auto", + "orientation": "auto", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "textMode": "auto" + }, + "pluginVersion": "9.4.7", + "repeat": "flavors", + "repeatDirection": "h", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "openstack_free_capacity_by_flavor_total{flavor_name=~\"$flavors\"}", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "$flavors", + "type": "stat" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 5 + }, + "id": 13, + "panels": [], + "title": "Summary Hypervisor Capacity", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "decmbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 0, + "y": 6 + }, + "id": 7, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "values": [ + "value", + "percent" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(openstack_hypervisor_placement_allocatable_capacity{resource=\"MEMORY_MB\"} - on(hypervisor) openstack_hypervisor_placement_allocated{resource=\"MEMORY_MB\"})", + "hide": false, + "legendFormat": "Free Memory", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(openstack_hypervisor_placement_allocated{resource=\"MEMORY_MB\"})", + "hide": false, + "legendFormat": "Allocated Memory", + "range": true, + "refId": "B" + } + ], + "title": "Total Memory", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 8, + "y": 6 + }, + "id": 9, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "values": [ + "percent", + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(openstack_hypervisor_placement_allocatable_capacity{resource=\"VCPU\"} - on(hypervisor) openstack_hypervisor_placement_allocated{resource=\"VCPU\"})", + "hide": false, + "legendFormat": "Free VCPU", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(openstack_hypervisor_placement_allocatable_capacity{resource=\"PCPU\"} - on(hypervisor) openstack_hypervisor_placement_allocated{resource=\"PCPU\"})", + "hide": false, + "legendFormat": "Free PCPU", + "range": true, + "refId": "C" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(openstack_hypervisor_placement_allocated{resource=\"VCPU\"})", + "hide": false, + "legendFormat": "Allocated VCPU", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(openstack_hypervisor_placement_allocated{resource=\"PCPU\"})", + "hide": false, + "legendFormat": "Allocated PCPU", + "range": true, + "refId": "D" + } + ], + "title": "Total CPU", + "type": "piechart" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + } + }, + "mappings": [], + "unit": "decgbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 8, + "x": 16, + "y": 6 + }, + "id": 8, + "options": { + "legend": { + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "values": [ + "percent", + "value" + ] + }, + "pieType": "pie", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(openstack_hypervisor_placement_allocatable_capacity{resource=\"DISK_GB\"} - on(hypervisor) openstack_hypervisor_placement_allocated{resource=\"DISK_GB\"})", + "hide": false, + "legendFormat": "Free Local Disk", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "sum(openstack_hypervisor_placement_allocated{resource=\"DISK_GB\"})", + "hide": false, + "legendFormat": "Allocated Local Disk", + "range": true, + "refId": "D" + } + ], + "title": "Total Local Disk", + "type": "piechart" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 16 + }, + "id": 11, + "panels": [], + "title": "Project Usage", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 23, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decmbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 17 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "min", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "openstack_project_usage{placement_resource=\"MEMORY_MB\"}", + "legendFormat": "{{project_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Memory Used by Project", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 23, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decmbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 12, + "y": 17 + }, + "id": 16, + "options": { + "legend": { + "calcs": [ + "min", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "openstack_project_usage{placement_resource=\"VCPU\"}", + "legendFormat": "VCPU {{project_name}}", + "range": true, + "refId": "A" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "expr": "openstack_project_usage{placement_resource=\"PCPU\"}", + "hide": false, + "legendFormat": "PCPU {{project_name}}", + "range": true, + "refId": "B" + } + ], + "title": "CPUs Used by Project", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 25 + }, + "id": 15, + "panels": [], + "title": "Per Hypervisor Free Capacity", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 28, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "smooth", + "lineStyle": { + "fill": "solid" + }, + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 0, + "y": 26 + }, + "id": 6, + "options": { + "legend": { + "calcs": [ + "min", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "pluginVersion": "9.1.2", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "code", + "exemplar": false, + "expr": "openstack_free_capacity_hypervisor_by_flavor{flavor_name=~\"$flavors\"}", + "format": "time_series", + "instant": false, + "legendFormat": "{{flavor_name}} on {{hypervisor}}", + "range": true, + "refId": "Avaliable Capacity on Hypervisors" + } + ], + "title": "Avaliable Capacity for $flavors", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 21, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "decmbytes" + }, + "overrides": [] + }, + "gridPos": { + "h": 13, + "w": 12, + "x": 12, + "y": 26 + }, + "id": 4, + "options": { + "legend": { + "calcs": [ + "min", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "editorMode": "builder", + "expr": "openstack_hypervisor_placement_allocatable_capacity{resource=\"MEMORY_MB\"} - on(hypervisor) openstack_hypervisor_placement_allocated{resource=\"MEMORY_MB\"}", + "legendFormat": "{{hypervisor}}", + "range": true, + "refId": "A" + } + ], + "title": "Memory Free by Hypervisor", + "type": "timeseries" + } + ], + "refresh": "", + "revision": 1, + "schemaVersion": 38, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "description": "The prometheus datasource used for queries.", + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "DS_PROMETHEUS", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${DS_PROMETHEUS}" + }, + "definition": "label_values(openstack_free_capacity_by_flavor_total, flavor_name)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "flavors", + "options": [], + "query": { + "query": "label_values(openstack_free_capacity_by_flavor_total, flavor_name)", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-24h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Cloud Metrics", + "uid": "g__ksD67z", + "version": 1, + "weekStart": "" +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/grafana/dashboards/openstack/grafana_project_dashboard.json b/etc/kayobe/kolla/config/grafana/dashboards/openstack/grafana_project_dashboard.json new file mode 100644 index 000000000..c3a483cf9 --- /dev/null +++ b/etc/kayobe/kolla/config/grafana/dashboards/openstack/grafana_project_dashboard.json @@ -0,0 +1,880 @@ +{% raw %} +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "datasource", + "uid": "grafana" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 2084480, + "links": [], + "liveNow": false, + "panels": [ + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 11, + "panels": [], + "title": "Project Usage", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-yellow", + "value": null + }, + { + "color": "green", + "value": 1 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 1 + }, + "id": 18, + "options": { + "displayMode": "basic", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "9.4.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "max(openstack_project_usage{project_id=~\"${project_id}\"}) by (placement_resource)", + "format": "time_series", + "hide": false, + "legendFormat": "{{flavor_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Max usage", + "type": "bargauge" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 23, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 11 + }, + "id": 5, + "options": { + "legend": { + "calcs": [ + "min", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(openstack_project_usage{project_id=~\"${project_id}\"}) by (placement_resource)", + "hide": false, + "legendFormat": "{{placement_resource}}", + "range": true, + "refId": "A" + } + ], + "title": "Resources per Project", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 23, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 11 + }, + "id": 19, + "options": { + "legend": { + "calcs": [ + "min", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "openstack_project_quota{project_id=~\"${project_id}\"}", + "hide": false, + "legendFormat": "{{project_name}}:{{quota_resource}}", + "range": true, + "refId": "B" + } + ], + "title": "Project quota", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 23, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "none" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 11 + }, + "id": 17, + "options": { + "legend": { + "calcs": [ + "min", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "count(count(libvirt_domain_info_meta{project_uuid=~\"${project_id}\"}) by (domain,flavor)) by (flavor)", + "hide": false, + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Servers by flavor", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "axisSoftMax": 1, + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 23, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 0, + "y": 20 + }, + "id": 20, + "options": { + "legend": { + "calcs": [ + "min", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "sum(irate(libvirt_domain_vcpu_time_seconds_total{}[5m]) / ignoring(instance,vcpu) group_left(domain) libvirt_domain_info_virtual_cpus{}) by (domain) * on(domain) group_left(instance_name,project_name,project_uuid) libvirt_domain_info_meta{project_uuid=~\"${project_id}\"}", + "hide": false, + "legendFormat": "{{instance_name}}", + "range": true, + "refId": "B" + } + ], + "title": "CPU utilization per instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 23, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 100, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percent" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 8, + "y": 20 + }, + "id": 21, + "options": { + "legend": { + "calcs": [ + "min", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "libvirt_domain_memory_stats_used_percent * on(domain) group_left(instance_name,project_name,project_uuid) libvirt_domain_info_meta{project_uuid=~\"${project_id}\"}", + "hide": false, + "legendFormat": "{{instance_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Memory utilization per instance", + "type": "timeseries" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": true, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 23, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "max": 1, + "min": -1, + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + }, + "unit": "percentunit" + }, + "overrides": [] + }, + "gridPos": { + "h": 9, + "w": 8, + "x": 16, + "y": 20 + }, + "id": 22, + "options": { + "legend": { + "calcs": [ + "min", + "max" + ], + "displayMode": "table", + "placement": "bottom", + "showLegend": true, + "sortBy": "Max", + "sortDesc": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(libvirt_domain_block_stats_read_time_seconds_total[5m]) * on(domain) group_left(instance_name,project_name,project_uuid) libvirt_domain_info_meta{project_uuid=~\"${project_id}\"}", + "hide": false, + "legendFormat": "{{instance_name}} : read {{target_device}}", + "range": true, + "refId": "B" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "code", + "expr": "rate(libvirt_domain_block_stats_write_time_seconds_total[5m]) * on(domain) group_left(instance_name,project_name,project_uuid) libvirt_domain_info_meta{project_uuid=~\"${project_id}\"} * -1", + "hide": false, + "legendFormat": "{{instance_name}} : write {{target_device}}", + "range": true, + "refId": "C" + } + ], + "title": "Disk utilization per instance", + "type": "timeseries" + }, + { + "collapsed": false, + "gridPos": { + "h": 1, + "w": 24, + "x": 0, + "y": 29 + }, + "id": 15, + "panels": [], + "title": "Per Hypervisor Free Capacity", + "type": "row" + }, + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "description": "", + "fieldConfig": { + "defaults": { + "color": { + "mode": "thresholds" + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "semi-dark-yellow", + "value": null + }, + { + "color": "green", + "value": 4 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 30 + }, + "id": 2, + "options": { + "displayMode": "basic", + "minVizHeight": 10, + "minVizWidth": 0, + "orientation": "horizontal", + "reduceOptions": { + "calcs": [ + "lastNotNull" + ], + "fields": "", + "values": false + }, + "showUnfilled": true + }, + "pluginVersion": "9.4.7", + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "editorMode": "builder", + "expr": "openstack_free_capacity_by_flavor_total", + "format": "time_series", + "legendFormat": "{{flavor_name}}", + "range": true, + "refId": "A" + } + ], + "title": "Free Capacity by Flavor", + "type": "bargauge" + } + ], + "refresh": "", + "revision": 1, + "schemaVersion": 38, + "style": "dark", + "tags": [ + "capacity", + "azimuth" + ], + "templating": { + "list": [ + { + "current": { + "selected": false, + "text": "Prometheus", + "value": "Prometheus" + }, + "description": "The prometheus datasource used for queries.", + "hide": 0, + "includeAll": false, + "label": "datasource", + "multi": false, + "name": "datasource", + "options": [], + "query": "prometheus", + "queryValue": "", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "type": "datasource" + }, + { + "current": { + "selected": true, + "text": [ + "All" + ], + "value": [ + "$__all" + ] + }, + "datasource": { + "type": "prometheus", + "uid": "${datasource}" + }, + "definition": "openstack_project_usage", + "hide": 0, + "includeAll": true, + "label": "Project", + "multi": true, + "name": "project_id", + "options": [], + "query": { + "query": "openstack_project_usage", + "refId": "StandardVariableQuery" + }, + "refresh": 1, + "regex": "/project_name=\"(?[^\"]+)|project_id=\"(?[^\"]+)/g", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-3h", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "OpenStack Project Metrics", + "uid": "mXiuBDe7z", + "version": 2, + "weekStart": "" +} +{% endraw %} diff --git a/etc/kayobe/kolla/config/haproxy/services.d/os_exporter.cfg b/etc/kayobe/kolla/config/haproxy/services.d/os_exporter.cfg new file mode 100644 index 000000000..93ac02192 --- /dev/null +++ b/etc/kayobe/kolla/config/haproxy/services.d/os_exporter.cfg @@ -0,0 +1,19 @@ +{% raw %} +frontend os_capacity_frontend + mode http + http-request del-header X-Forwarded-Proto + option httplog + option forwardfor + http-request set-header X-Forwarded-Proto https if { ssl_fc } + bind {{ kolla_internal_vip_address }}:9000 + default_backend os_capacity_backend + +backend os_capacity_backend + mode http + +{% for host in groups['monitoring'] %} +{% set host_name = hostvars[host].ansible_facts.hostname %} +{% set host_ip = 'api' | kolla_address(host) %} + server {{ host_name }} {{ host_ip }}:9000 check inter 2000 rise 2 fall 5 +{% endfor %} +{% endraw %} diff --git a/etc/kayobe/kolla/config/prometheus/prometheus.yml.d/70-oscapacity.yml.j2 b/etc/kayobe/kolla/config/prometheus/prometheus.yml.d/70-oscapacity.yml.j2 new file mode 100644 index 000000000..1c0c25c15 --- /dev/null +++ b/etc/kayobe/kolla/config/prometheus/prometheus.yml.d/70-oscapacity.yml.j2 @@ -0,0 +1,9 @@ +{% raw %} +scrape_configs: + - job_name: os-capacity + static_configs: + - targets: + - '{{ kolla_internal_vip_address | put_address_in_context('url') }}:9000' + scrape_interval: 15m + scrape_timeout: 10m +{% endraw %} diff --git a/releasenotes/notes/os-capacity-94006f03f16583e4.yaml b/releasenotes/notes/os-capacity-94006f03f16583e4.yaml new file mode 100644 index 000000000..59b3aef40 --- /dev/null +++ b/releasenotes/notes/os-capacity-94006f03f16583e4.yaml @@ -0,0 +1,13 @@ +--- +features: + - | + This patch adds Openstack Capacity metrics and exporters to + StackHPC Kayobe Config. This includes a deployment playbook, + Prometheus scrape jobs and HAProxy configurations to support + this change. +upgrade: + - | + To deploy Openstack Capacity, you must define Openstack + application credential variables: secrets_os_exporter_auth_url, + secrets_os_exporter_credential_id and secrets_os_exporter_ + credential_secret as laid out in the 'Monitoring' documentation. From 6739b7f2867019d596a8ccd39250fe2b59cc20bc Mon Sep 17 00:00:00 2001 From: Jake Hutchinson <39007539+assumptionsandg@users.noreply.github.com> Date: Mon, 2 Oct 2023 12:48:51 +0100 Subject: [PATCH 2/2] Update releasenotes/notes/os-capacity-94006f03f16583e4.yaml Co-authored-by: Alex-Welsh <112560678+Alex-Welsh@users.noreply.github.com> --- releasenotes/notes/os-capacity-94006f03f16583e4.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/releasenotes/notes/os-capacity-94006f03f16583e4.yaml b/releasenotes/notes/os-capacity-94006f03f16583e4.yaml index 59b3aef40..e07b8a689 100644 --- a/releasenotes/notes/os-capacity-94006f03f16583e4.yaml +++ b/releasenotes/notes/os-capacity-94006f03f16583e4.yaml @@ -7,7 +7,9 @@ features: this change. upgrade: - | - To deploy Openstack Capacity, you must define Openstack - application credential variables: secrets_os_exporter_auth_url, - secrets_os_exporter_credential_id and secrets_os_exporter_ - credential_secret as laid out in the 'Monitoring' documentation. + To deploy the Openstack Capacity Grafana dashboard, you must + define Openstack application credential variables: + ``secrets_os_exporter_auth_url``, + ``secrets_os_exporter_credential_id`` and + ``secrets_os_exporter_credential_secret`` as laid out in the + 'Monitoring' documentation.