From 814fb120b84da515256648bbf76b27c9bcfa9211 Mon Sep 17 00:00:00 2001 From: wittenator <9154515+wittenator@users.noreply.github.com> Date: Wed, 12 Jun 2024 23:09:31 +0200 Subject: [PATCH 1/2] Add grafana setup --- .gitignore | 1 + docker-compose.override.yml | 15 ++- docker-compose.yml | 30 ++++- grafana/dashboards/generator_dashboard.json | 108 ++++++++++++++++++ grafana/provisioning/dashboards/default.yaml | 12 ++ .../datasources/generator_postgres.yaml | 18 +++ 6 files changed, 182 insertions(+), 2 deletions(-) create mode 100644 grafana/dashboards/generator_dashboard.json create mode 100644 grafana/provisioning/dashboards/default.yaml create mode 100644 grafana/provisioning/datasources/generator_postgres.yaml diff --git a/.gitignore b/.gitignore index 497b6f7..1bd9631 100755 --- a/.gitignore +++ b/.gitignore @@ -2,3 +2,4 @@ .mypy_cache docker-stack.yml .env +postgres/dumps diff --git a/docker-compose.override.yml b/docker-compose.override.yml index fb8579b..ba67496 100644 --- a/docker-compose.override.yml +++ b/docker-compose.override.yml @@ -1,4 +1,3 @@ -version: "3.3" services: proxy: @@ -36,6 +35,11 @@ services: ports: - "5050:5050" + db: + volumes: + - ./postgres/dumps/:/docker-entrypoint-initdb.d/ + + backend: ports: - "8888:8888" @@ -58,6 +62,15 @@ services: - traefik.http.routers.${STACK_NAME?Variable not set}-backend-http.rule=PathPrefix(`/api`) || PathPrefix(`/docs`) || PathPrefix(`/redoc`) - traefik.http.services.${STACK_NAME?Variable not set}-backend.loadbalancer.server.port=80 + grafana: + ports: + - "3000:3000" + labels: + - traefik.enable=true + - traefik.constraint-label-stack=${TRAEFIK_TAG?Variable not set} + - traefik.http.routers.${STACK_NAME?Variable not set}-grafana-http.rule=Host(`${DOMAIN?Variable not set}`) + - traefik.http.services.${STACK_NAME?Variable not set}-grafana.loadbalancer.server.port=3000 + networks: traefik-public: # For local dev, don't expect an external Traefik network diff --git a/docker-compose.yml b/docker-compose.yml index efa5f54..7b726e0 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,4 +1,3 @@ -version: "3.8" services: proxy: @@ -131,6 +130,35 @@ services: - traefik.http.routers.${STACK_NAME?Variable not set}-backend-http.rule=PathPrefix(`/api`) || PathPrefix(`/docs`) || PathPrefix(`/redoc`) - traefik.http.services.${STACK_NAME?Variable not set}-backend.loadbalancer.server.port=80 + grafana: + image: grafana/grafana-oss:latest + networks: + - ${TRAEFIK_PUBLIC_NETWORK?Variable not set} + - default + env_file: + - .env + environment: + - GF_SECURITY_ADMIN_USER=${GF_ADMIN_USER?Variable not set} + - GF_SECURITY_ADMIN_PASSWORD=${GF_ADMIN_PASSWORD?Variable not set} + # add privisioning path + - GF_PATHS_PROVISIONING=/etc/grafana/provisioning + volumes: + - ./grafana/provisioning:/etc/grafana/provisioning + - ./grafana/dashboards:/var/lib/grafana/dashboards + deploy: + labels: + - traefik.enable=true + - traefik.docker.network=${TRAEFIK_PUBLIC_NETWORK?Variable not set} + - traefik.constraint-label=${TRAEFIK_PUBLIC_TAG?Variable not set} + - traefik.http.routers.${STACK_NAME?Variable not set}-grafana-http.rule=Host(`grafana.${DOMAIN?Variable not set}`) + - traefik.http.routers.${STACK_NAME?Variable not set}-grafana-http.entrypoints=http + - traefik.http.routers.${STACK_NAME?Variable not set}-grafana-http.middlewares=${STACK_NAME?Variable not set}-https-redirect + - traefik.http.routers.${STACK_NAME?Variable not set}-grafana-https.rule=Host(`grafana.${DOMAIN?Variable not set}`) + - traefik.http.routers.${STACK_NAME?Variable not set}-grafana-https.entrypoints=https + - traefik.http.routers.${STACK_NAME?Variable not set}-grafana-https.tls=true + - traefik.http.routers.${STACK_NAME?Variable not set}-grafana-https.tls.certresolver=le + - traefik.http.services.${STACK_NAME?Variable not set}-grafana.loadbalancer.server.port=3000 + volumes: app-db-data: diff --git a/grafana/dashboards/generator_dashboard.json b/grafana/dashboards/generator_dashboard.json new file mode 100644 index 0000000..96864fb --- /dev/null +++ b/grafana/dashboards/generator_dashboard.json @@ -0,0 +1,108 @@ +{ + "__inputs": [], + "__requires": [], + "annotations": { + "list": [] + }, + "editable": false, + "gnetId": null, + "graphTooltip": 0, + "hideControls": false, + "id": null, + "links": [], + "panels": [ + { + "aliasColors": {}, + "bars": false, + "dashLength": 10, + "dashes": false, + "datasource": "TestData", + "fill": 1, + "gridPos": { + "h": 8, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 2, + "legend": { + "alignAsTable": false, + "avg": false, + "current": false, + "max": false, + "min": false, + "rightSide": false, + "show": true, + "total": false, + "values": false + }, + "lines": true, + "linewidth": 1, + "links": [], + "nullPointMode": "null", + "percentage": false, + "pointradius": 5, + "points": false, + "renderer": "flot", + "repeat": null, + "seriesOverrides": [], + "spaceLength": 10, + "stack": false, + "steppedLine": false, + "targets": [], + "thresholds": [], + "timeFrom": null, + "timeShift": null, + "title": "CPU Usage", + "tooltip": { + "shared": true, + "sort": 0, + "value_type": "individual" + }, + "type": "graph", + "xaxis": { + "buckets": null, + "mode": "time", + "name": null, + "show": true, + "values": [] + }, + "yaxes": [ + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + }, + { + "format": "short", + "label": null, + "logBase": 1, + "max": null, + "min": null, + "show": true + } + ] + } + ], + "refresh": "", + "rows": [], + "schemaVersion": 16, + "tags": ["kubernetes"], + "templating": { + "list": [] + }, + "time": { + "from": "now-6h", + "to": "now" + }, + "timepicker": { + "refresh_intervals": ["5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d"], + "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"] + }, + "timezone": "browser", + "title": "Cluster", + "version": 0 +} \ No newline at end of file diff --git a/grafana/provisioning/dashboards/default.yaml b/grafana/provisioning/dashboards/default.yaml new file mode 100644 index 0000000..cd599e3 --- /dev/null +++ b/grafana/provisioning/dashboards/default.yaml @@ -0,0 +1,12 @@ +apiVersion: 1 + +providers: + - name: "Dashboard provider" + orgId: 1 + type: file + disableDeletion: false + updateIntervalSeconds: 10 + allowUiUpdates: true + options: + path: /var/lib/grafana/dashboards + foldersFromFilesStructure: true \ No newline at end of file diff --git a/grafana/provisioning/datasources/generator_postgres.yaml b/grafana/provisioning/datasources/generator_postgres.yaml new file mode 100644 index 0000000..6c236e5 --- /dev/null +++ b/grafana/provisioning/datasources/generator_postgres.yaml @@ -0,0 +1,18 @@ +apiVersion: 1 + +datasources: + - name: Postgres + type: postgres + url: ${POSTGRES_SERVER}:5432 + user: ${POSTGRES_USER} + secureJsonData: + password: '${POSTGRES_PASSWORD}' + jsonData: + database: ${POSTGRES_DB} + sslmode: 'disable' # disable/require/verify-ca/verify-full + maxOpenConns: 100 # Grafana v5.4+ + maxIdleConns: 100 # Grafana v5.4+ + maxIdleConnsAuto: true # Grafana v9.5.1+ + connMaxLifetime: 14400 # Grafana v5.4+ + postgresVersion: 1200 # 903=9.3, 904=9.4, 905=9.5, 906=9.6, 1000=10 + timescaledb: false \ No newline at end of file From 9e1283873a9796a44d7ec5611b242728b1eac9e1 Mon Sep 17 00:00:00 2001 From: wittenator <9154515+wittenator@users.noreply.github.com> Date: Tue, 18 Jun 2024 15:04:10 +0200 Subject: [PATCH 2/2] Added initial dashboard setup --- grafana/dashboards/generator_dashboard.json | 333 +++++++++++++++----- 1 file changed, 256 insertions(+), 77 deletions(-) diff --git a/grafana/dashboards/generator_dashboard.json b/grafana/dashboards/generator_dashboard.json index 96864fb..cfae618 100644 --- a/grafana/dashboards/generator_dashboard.json +++ b/grafana/dashboards/generator_dashboard.json @@ -1,108 +1,287 @@ { - "__inputs": [], - "__requires": [], "annotations": { - "list": [] + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "type": "dashboard" + } + ] }, - "editable": false, - "gnetId": null, + "editable": true, + "fiscalYearStartMonth": 0, "graphTooltip": 0, - "hideControls": false, - "id": null, + "id": 1, "links": [], "panels": [ { - "aliasColors": {}, - "bars": false, - "dashLength": 10, - "dashes": false, - "datasource": "TestData", - "fill": 1, + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "P44368ADAD746BC27" + }, + "fieldConfig": { + "defaults": { + "color": { + "fixedColor": "blue", + "mode": "fixed" + }, + "custom": { + "axisBorderShow": false, + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "bars", + "fillOpacity": 90, + "gradientMode": "scheme", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "insertNulls": false, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, "gridPos": { - "h": 8, + "h": 9, "w": 24, "x": 0, "y": 0 }, - "id": 2, - "legend": { - "alignAsTable": false, - "avg": false, - "current": false, - "max": false, - "min": false, - "rightSide": false, - "show": true, - "total": false, - "values": false - }, - "lines": true, - "linewidth": 1, - "links": [], - "nullPointMode": "null", - "percentage": false, - "pointradius": 5, - "points": false, - "renderer": "flot", - "repeat": null, - "seriesOverrides": [], - "spaceLength": 10, - "stack": false, - "steppedLine": false, - "targets": [], - "thresholds": [], - "timeFrom": null, - "timeShift": null, - "title": "CPU Usage", - "tooltip": { - "shared": true, - "sort": 0, - "value_type": "individual" - }, - "type": "graph", - "xaxis": { - "buckets": null, - "mode": "time", - "name": null, - "show": true, - "values": [] + "id": 1, + "options": { + "legend": { + "calcs": [], + "displayMode": "hidden", + "placement": "right", + "showLegend": false + }, + "tooltip": { + "maxHeight": 600, + "mode": "single", + "sort": "none" + } }, - "yaxes": [ + "pluginVersion": "11.0.0", + "targets": [ { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "P44368ADAD746BC27" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT $__timeGroupAlias(\"timestamp\",'1w'),\n COUNT(DISTINCT name) as \"Number of Licenses\"\nFROM license\nWHERE name NOT LIKE '%test%'\nGROUP BY time", + "refId": "Count Number of Licenses over Time", + "sql": { + "columns": [ + { + "parameters": [ + { + "name": "\"timestamp\"", + "type": "functionParameter" + } + ], + "type": "function" + }, + { + "parameters": [ + { + "name": "name", + "type": "functionParameter" + } + ], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "name": "\"timestamp\"", + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + }, + "table": "license" + } + ], + "title": "Number of Licenses Created ", + "type": "timeseries" + }, + { + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "P44368ADAD746BC27" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "fillOpacity": 80, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineWidth": 1, + "stacking": { + "group": "A", + "mode": "none" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } }, + "overrides": [] + }, + "gridPos": { + "h": 8, + "w": 12, + "x": 0, + "y": 9 + }, + "id": 2, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + } + }, + "targets": [ { - "format": "short", - "label": null, - "logBase": 1, - "max": null, - "min": null, - "show": true + "datasource": { + "type": "grafana-postgresql-datasource", + "uid": "P44368ADAD746BC27" + }, + "editorMode": "code", + "format": "table", + "rawQuery": true, + "rawSql": "SELECT COUNT(name)\nFROM license\nGROUP BY name", + "refId": "A", + "sql": { + "columns": [ + { + "parameters": [], + "type": "function" + } + ], + "groupBy": [ + { + "property": { + "type": "string" + }, + "type": "groupBy" + } + ], + "limit": 50 + } } - ] + ], + "title": "Number of Edits of a License", + "type": "histogram" } ], "refresh": "", - "rows": [], - "schemaVersion": 16, - "tags": ["kubernetes"], + "schemaVersion": 39, + "tags": [ + "kubernetes" + ], "templating": { "list": [] }, "time": { - "from": "now-6h", + "from": "2024-02-29T23:00:00.000Z", "to": "now" }, + "timeRangeUpdatedDuringEditOrView": false, "timepicker": { - "refresh_intervals": ["5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d"], - "time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"] + "refresh_intervals": [ + "5s", + "10s", + "30s", + "1m", + "5m", + "15m", + "30m", + "1h", + "2h", + "1d" + ], + "time_options": [ + "5m", + "15m", + "1h", + "6h", + "12h", + "24h", + "2d", + "7d", + "30d" + ] }, "timezone": "browser", "title": "Cluster", - "version": 0 + "uid": "fdp5e17m7wkqoc", + "version": 4, + "weekStart": "" } \ No newline at end of file