From 8bac9193786307444337a895a5b525d7e706be52 Mon Sep 17 00:00:00 2001 From: Alex Gartner Date: Fri, 26 Jul 2024 13:53:32 -0700 Subject: [PATCH] refactor: move localnet monitoring to profile --- Makefile | 42 ++-- .../localnet/docker-compose-monitoring.yml | 43 ---- contrib/localnet/docker-compose.yml | 91 ++++++++ .../grafana/dashboards/log_browser.json | 212 ++++++++++++++++++ contrib/localnet/grafana/datasource.yaml | 6 +- contrib/localnet/grafana/get-tss-address.sh | 9 - contrib/localnet/promtail/config.yaml | 19 ++ 7 files changed, 343 insertions(+), 79 deletions(-) delete mode 100644 contrib/localnet/docker-compose-monitoring.yml create mode 100644 contrib/localnet/grafana/dashboards/log_browser.json delete mode 100755 contrib/localnet/grafana/get-tss-address.sh create mode 100644 contrib/localnet/promtail/config.yaml diff --git a/Makefile b/Makefile index 0cbdff1614..a739e22460 100644 --- a/Makefile +++ b/Makefile @@ -4,6 +4,9 @@ VERSION := $(shell ./version.sh) COMMIT := $(shell [ -z "${COMMIT_ID}" ] && git log -1 --format='%H' || echo ${COMMIT_ID} ) BUILDTIME := $(shell date -u +"%Y%m%d.%H%M%S" ) DOCKER ?= docker +# allow setting of DOCKER_COMPOSE_ARGS to pass additional args to docker compose +# useful for setting profiles +DOCKER_COMPOSE ?= $(DOCKER) compose $(COMPOSE_ARGS) DOCKER_BUF := $(DOCKER) run --rm -v $(CURDIR):/workspace --workdir /workspace bufbuild/buf GOFLAGS:="" @@ -209,11 +212,11 @@ start-localnet: zetanode start-localnet-skip-build start-localnet-skip-build: @echo "--> Starting localnet" export LOCALNET_MODE=setup-only && \ - cd contrib/localnet/ && $(DOCKER) compose -f docker-compose.yml up -d + cd contrib/localnet/ && $(DOCKER_COMPOSE) -f docker-compose.yml up -d # stop-localnet should include all profiles so other containers are also removed stop-localnet: - cd contrib/localnet/ && $(DOCKER) compose --profile all down --remove-orphans + cd contrib/localnet/ && $(DOCKER_COMPOSE) --profile all down --remove-orphans ############################################################################### ### E2E tests ### @@ -236,37 +239,37 @@ solana: start-e2e-test: zetanode @echo "--> Starting e2e test" - cd contrib/localnet/ && $(DOCKER) compose up -d + cd contrib/localnet/ && $(DOCKER_COMPOSE) up -d start-e2e-admin-test: zetanode @echo "--> Starting e2e admin test" export E2E_ARGS="--skip-regular --test-admin" && \ - cd contrib/localnet/ && $(DOCKER) compose --profile eth2 -f docker-compose.yml up -d + cd contrib/localnet/ && $(DOCKER_COMPOSE) --profile eth2 -f docker-compose.yml up -d start-e2e-performance-test: zetanode @echo "--> Starting e2e performance test" export E2E_ARGS="--test-performance" && \ - cd contrib/localnet/ && $(DOCKER) compose -f docker-compose.yml up -d + cd contrib/localnet/ && $(DOCKER_COMPOSE) -f docker-compose.yml up -d start-e2e-import-mainnet-test: zetanode @echo "--> Starting e2e import-data test" export ZETACORED_IMPORT_GENESIS_DATA=true && \ export ZETACORED_START_PERIOD=15m && \ - cd contrib/localnet/ && ./scripts/import-data.sh mainnet && $(DOCKER) compose -f docker-compose.yml up -d + cd contrib/localnet/ && ./scripts/import-data.sh mainnet && $(DOCKER_COMPOSE) -f docker-compose.yml up -d start-stress-test: zetanode @echo "--> Starting stress test" - cd contrib/localnet/ && $(DOCKER) compose --profile stress -f docker-compose.yml up -d + cd contrib/localnet/ && $(DOCKER_COMPOSE) --profile stress -f docker-compose.yml up -d start-tss-migration-test: zetanode @echo "--> Starting migration test" export E2E_ARGS="--test-tss-migration" && \ - cd contrib/localnet/ && $(DOCKER) compose up -d + cd contrib/localnet/ && $(DOCKER_COMPOSE) up -d start-solana-test: zetanode solana @echo "--> Starting solana test" export E2E_ARGS="--skip-regular --test-solana" && \ - cd contrib/localnet/ && $(DOCKER) compose --profile solana -f docker-compose.yml up -d + cd contrib/localnet/ && $(DOCKER_COMPOSE) --profile solana -f docker-compose.yml up -d ############################################################################### ### Upgrade Tests ### @@ -290,13 +293,13 @@ start-upgrade-test: zetanode-upgrade @echo "--> Starting upgrade test" export LOCALNET_MODE=upgrade && \ export UPGRADE_HEIGHT=225 && \ - cd contrib/localnet/ && $(DOCKER) compose --profile upgrade -f docker-compose.yml -f docker-compose-upgrade.yml up -d + cd contrib/localnet/ && $(DOCKER_COMPOSE) --profile upgrade -f docker-compose.yml -f docker-compose-upgrade.yml up -d start-upgrade-test-light: zetanode-upgrade @echo "--> Starting light upgrade test (no ZetaChain state populating before upgrade)" export LOCALNET_MODE=upgrade && \ export UPGRADE_HEIGHT=90 && \ - cd contrib/localnet/ && $(DOCKER) compose --profile upgrade -f docker-compose.yml -f docker-compose-upgrade.yml up -d + cd contrib/localnet/ && $(DOCKER_COMPOSE) --profile upgrade -f docker-compose.yml -f docker-compose-upgrade.yml up -d start-upgrade-test-admin: zetanode-upgrade @@ -304,7 +307,7 @@ start-upgrade-test-admin: zetanode-upgrade export LOCALNET_MODE=upgrade && \ export UPGRADE_HEIGHT=90 && \ export E2E_ARGS="--skip-regular --test-admin" && \ - cd contrib/localnet/ && $(DOCKER) compose --profile upgrade -f docker-compose.yml -f docker-compose-upgrade.yml up -d + cd contrib/localnet/ && $(DOCKER_COMPOSE) --profile upgrade -f docker-compose.yml -f docker-compose-upgrade.yml up -d start-upgrade-import-mainnet-test: zetanode-upgrade @echo "--> Starting import-data upgrade test" @@ -312,20 +315,7 @@ start-upgrade-import-mainnet-test: zetanode-upgrade export ZETACORED_IMPORT_GENESIS_DATA=true && \ export ZETACORED_START_PERIOD=15m && \ export UPGRADE_HEIGHT=225 && \ - cd contrib/localnet/ && ./scripts/import-data.sh mainnet && $(DOCKER) compose --profile upgrade -f docker-compose.yml -f docker-compose-upgrade.yml up -d - -############################################################################### -### Monitoring ### -############################################################################### - -start-monitoring: - @echo "Starting monitoring services" - cd contrib/localnet/grafana/ && ./get-tss-address.sh - cd contrib/localnet/ && $(DOCKER) compose -f docker-compose-monitoring.yml up -d - -stop-monitoring: - @echo "Stopping monitoring services" - cd contrib/localnet/ && $(DOCKER) compose -f docker-compose-monitoring.yml down + cd contrib/localnet/ && ./scripts/import-data.sh mainnet && $(DOCKER_COMPOSE) --profile upgrade -f docker-compose.yml -f docker-compose-upgrade.yml up -d ############################################################################### ### GoReleaser ### diff --git a/contrib/localnet/docker-compose-monitoring.yml b/contrib/localnet/docker-compose-monitoring.yml deleted file mode 100644 index 89ddd0d7c3..0000000000 --- a/contrib/localnet/docker-compose-monitoring.yml +++ /dev/null @@ -1,43 +0,0 @@ -services: - grafana: - image: grafana/grafana:latest - container_name: grafana - hostname: grafana - volumes: - - ./grafana/datasource.yaml:/etc/grafana/provisioning/datasources/datasource.yaml - - ./grafana/dashboards/:/etc/grafana/provisioning/dashboards - ports: - - "3000:3000" - networks: - mynetwork: - ipv4_address: 172.20.0.30 - - prometheus: - image: prom/prometheus:latest - container_name: prometheus - hostname: prometheus - volumes: - - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml - networks: - mynetwork: - ipv4_address: 172.20.0.31 - - zetachain-exporter: - image: zetachain/zetachain-exporter:latest - container_name: zetachain-exporter - hostname: zetachain-exporter - volumes: - - ./grafana/addresses.txt:/app/addresses.txt - ports: - - "9015:9015" - networks: - mynetwork: - ipv4_address: 172.20.0.32 - environment: - - GETH=http://eth:8545 - -networks: - mynetwork: - ipam: - config: - - subnet: 172.20.0.0/24 \ No newline at end of file diff --git a/contrib/localnet/docker-compose.yml b/contrib/localnet/docker-compose.yml index 928a9fe5a7..6eece58fce 100644 --- a/contrib/localnet/docker-compose.yml +++ b/contrib/localnet/docker-compose.yml @@ -291,6 +291,97 @@ services: - UPGRADE_HEIGHT=${UPGRADE_HEIGHT-} volumes: - ssh:/root/.ssh + + grafana: + image: grafana/grafana:9.1.7 + container_name: grafana + hostname: grafana + profiles: + - monitoring + - all + volumes: + - ./grafana/datasource.yaml:/etc/grafana/provisioning/datasources/datasource.yaml + - ./grafana/dashboards/:/etc/grafana/provisioning/dashboards + - grafana_storage:/var/lib/grafana + ports: + - "3000:3000" + networks: + mynetwork: + ipv4_address: 172.20.0.30 + depends_on: + - prometheus + + prometheus: + image: prom/prometheus:v2.53.1 + container_name: prometheus + hostname: prometheus + profiles: + - monitoring + - all + volumes: + - ./prometheus/prometheus.yml:/etc/prometheus/prometheus.yml + networks: + mynetwork: + ipv4_address: 172.20.0.31 + depends_on: + zetacore0: + condition: service_healthy + + zetachain-exporter: + image: zetachain/zetachain-exporter:v2.0.6 + container_name: zetachain-exporter + hostname: zetachain-exporter + profiles: + - monitoring + - all + ports: + - "9015:9015" + networks: + mynetwork: + ipv4_address: 172.20.0.32 + environment: + - ETH_EVM_URL=http://eth:8545 + - ZETA_EVM_URL=http://zetacore0:8545 + - ZETA_LCD_URL=http://zetacore0:1317 + - ZETA_TENDERMINT_URL=http://zetacore0:26657 + - ETH_CHAIN_ID=1337 + - ZETA_CHAIN_ID=7001 + - REFRESH_INTERVAL=15 + - FAST_REFRESH_INTERVAL=1 + depends_on: + zetacore0: + condition: service_healthy + eth: + condition: service_started + + loki: + container_name: loki + image: grafana/loki:3.1.0 + hostname: loki + profiles: + - monitoring + - all + networks: + mynetwork: + ipv4_address: 172.20.0.33 + command: -config.file=/etc/loki/local-config.yaml + + promtail: + container_name: promtail + image: grafana/promtail:2.9.9 + hostname: promtail + profiles: + - monitoring + - all + networks: + mynetwork: + ipv4_address: 172.20.0.34 + volumes: + - ./promtail/config.yaml:/etc/promtail/config.yaml:ro + - /var/run/docker.sock:/var/run/docker.sock + command: -config.file=/etc/promtail/config.yaml + volumes: ssh: preparams: + grafana_storage: diff --git a/contrib/localnet/grafana/dashboards/log_browser.json b/contrib/localnet/grafana/dashboards/log_browser.json new file mode 100644 index 0000000000..27851fe69d --- /dev/null +++ b/contrib/localnet/grafana/dashboards/log_browser.json @@ -0,0 +1,212 @@ +{ + "annotations": { + "list": [ + { + "builtIn": 1, + "datasource": { + "type": "grafana", + "uid": "-- Grafana --" + }, + "enable": true, + "hide": true, + "iconColor": "rgba(0, 211, 255, 1)", + "name": "Annotations & Alerts", + "target": { + "limit": 100, + "matchAny": false, + "tags": [], + "type": "dashboard" + }, + "type": "dashboard" + } + ] + }, + "description": "Easily browse logs for containers and filter by time", + "editable": true, + "fiscalYearStartMonth": 0, + "graphTooltip": 0, + "id": 5, + "links": [], + "liveNow": false, + "panels": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "fieldConfig": { + "defaults": { + "color": { + "mode": "palette-classic" + }, + "custom": { + "axisCenteredZero": false, + "axisColorMode": "text", + "axisLabel": "", + "axisPlacement": "auto", + "barAlignment": 0, + "drawStyle": "line", + "fillOpacity": 0, + "gradientMode": "none", + "hideFrom": { + "legend": false, + "tooltip": false, + "viz": false + }, + "lineInterpolation": "linear", + "lineWidth": 1, + "pointSize": 5, + "scaleDistribution": { + "type": "linear" + }, + "showPoints": "auto", + "spanNulls": false, + "stacking": { + "group": "A", + "mode": "none" + }, + "thresholdsStyle": { + "mode": "off" + } + }, + "mappings": [], + "thresholds": { + "mode": "absolute", + "steps": [ + { + "color": "green", + "value": null + }, + { + "color": "red", + "value": 80 + } + ] + } + }, + "overrides": [] + }, + "gridPos": { + "h": 6, + "w": 24, + "x": 0, + "y": 0 + }, + "id": 5, + "options": { + "legend": { + "calcs": [], + "displayMode": "list", + "placement": "bottom", + "showLegend": true + }, + "tooltip": { + "mode": "single", + "sort": "none" + } + }, + "targets": [ + { + "datasource": { + "type": "prometheus", + "uid": "PBFA97CFB590B2093" + }, + "editorMode": "builder", + "expr": "zetaclient_pending_txs_total", + "legendFormat": "__auto", + "range": true, + "refId": "A" + } + ], + "title": "Pending TXs total", + "type": "timeseries" + }, + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "gridPos": { + "h": 10, + "w": 24, + "x": 0, + "y": 6 + }, + "id": 2, + "options": { + "dedupStrategy": "signature", + "enableLogDetails": true, + "prettifyLogMessage": false, + "showCommonLabels": false, + "showLabels": false, + "showTime": true, + "sortOrder": "Descending", + "wrapLogMessage": false + }, + "pluginVersion": "9.1.7", + "repeat": "log_containers", + "repeatDirection": "v", + "targets": [ + { + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "editorMode": "builder", + "expr": "{container=\"$log_containers\"}", + "queryType": "range", + "refId": "A" + } + ], + "title": "${log_containers}", + "type": "logs" + } + ], + "refresh": "5s", + "schemaVersion": 37, + "style": "dark", + "tags": [], + "templating": { + "list": [ + { + "current": { + "selected": true, + "text": [ + "zetaclient0", + "zetacore0" + ], + "value": [ + "zetaclient0", + "zetacore0" + ] + }, + "datasource": { + "type": "loki", + "uid": "P8E80F9AEF21F6940" + }, + "definition": "label_values(container)", + "hide": 0, + "includeAll": true, + "multi": true, + "name": "log_containers", + "options": [], + "query": "label_values(container)", + "refresh": 1, + "regex": "", + "skipUrlSync": false, + "sort": 0, + "type": "query" + } + ] + }, + "time": { + "from": "now-30m", + "to": "now" + }, + "timepicker": {}, + "timezone": "", + "title": "Log Browser", + "uid": "QD_fGXuSk", + "version": 6, + "weekStart": "" + } \ No newline at end of file diff --git a/contrib/localnet/grafana/datasource.yaml b/contrib/localnet/grafana/datasource.yaml index 59cd333a94..3c0d5e3680 100644 --- a/contrib/localnet/grafana/datasource.yaml +++ b/contrib/localnet/grafana/datasource.yaml @@ -13,4 +13,8 @@ datasources: prometheusVersion: 2.48.1 cacheLevel: 'High' disableRecordingRules: false - incrementalQueryOverlapWindow: 10m \ No newline at end of file + incrementalQueryOverlapWindow: 10m + - name: Loki + type: loki + access: proxy + url: http://loki:3100 \ No newline at end of file diff --git a/contrib/localnet/grafana/get-tss-address.sh b/contrib/localnet/grafana/get-tss-address.sh deleted file mode 100755 index eae4b031af..0000000000 --- a/contrib/localnet/grafana/get-tss-address.sh +++ /dev/null @@ -1,9 +0,0 @@ -#!/bin/bash - -/usr/sbin/sshd - -#retrieve value of ETH TSS Address from localnet -ethTSS_address=$(curl 'http://localhost:1317/zeta-chain/observer/get_tss_address' | jq -r '.eth') - -#write value of ETH TSS Address to addresses.txt file -printf "ethTSS:$ethTSS_address\n" > addresses.txt \ No newline at end of file diff --git a/contrib/localnet/promtail/config.yaml b/contrib/localnet/promtail/config.yaml new file mode 100644 index 0000000000..82a6922caf --- /dev/null +++ b/contrib/localnet/promtail/config.yaml @@ -0,0 +1,19 @@ +server: + http_listen_port: 9080 + grpc_listen_port: 0 + +positions: + filename: /tmp/positions.yaml + +clients: + - url: http://loki:3100/loki/api/v1/push + +scrape_configs: + - job_name: flog_scrape + docker_sd_configs: + - host: unix:///var/run/docker.sock + refresh_interval: 5s + relabel_configs: + - source_labels: ['__meta_docker_container_name'] + regex: '/(.*)' + target_label: 'container'